Class: RKelly::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/rkelly/tokenizer.rb

Constant Summary collapse

KEYWORDS = 
Hash[%w{
  break case catch continue default delete do else finally for function
  if in instanceof new return switch this throw try typeof var void while
  with

  const true false null debugger
}.map {|kw| [kw, kw.upcase.to_sym]
RESERVED =

These 6 are always reserved in ECMAScript 5.1 Some others are only reserved in strict mode, but RKelly doesn’t differenciate between strict and non-strict mode code. www.ecma-international.org/ecma-262/5.1/#sec-7.6.1.2 developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Reserved_Words


      
        
LITERALS =
          
        

        
{
  # Punctuators
  '=='  => :EQEQ,
  '!='  => :NE,
  '===' => :STREQ,
  '!==' => :STRNEQ,
  '<='  => :LE,
  '>='  => :GE,
  '||'  => :OR,
  '&&'  => :AND,
  '++'  => :PLUSPLUS,
  '--'  => :MINUSMINUS,
  '<<'  => :LSHIFT,
  '<<=' => :LSHIFTEQUAL,
  '>>'  => :RSHIFT,
  '>>=' => :RSHIFTEQUAL,
  '>>>' => :URSHIFT,
  '>>>='=> :URSHIFTEQUAL,
  '&='  => :ANDEQUAL,
  '%='  => :MODEQUAL,
  '^='  => :XOREQUAL,
  '|='  => :OREQUAL,
  '+='  => :PLUSEQUAL,
  '-='  => :MINUSEQUAL,
  '*='  => :MULTEQUAL,
  '/='  => :DIVEQUAL,
}

      
        
KEYWORDS_THAT_IMPLY_DIVISION =
          

  

    

Some keywords can be followed by regular expressions (eg, return and throw). Others can be followed by division.



  





  



        

        
{
  'this' => true,
  'true' => true,
  'false' => true,
  'null' => true,
}

      
        
KEYWORDS_THAT_IMPLY_REGEX =
          
        

        
KEYWORDS.reject {|k,v| KEYWORDS_THAT_IMPLY_DIVISION[k] }

      
        
SINGLE_CHARS_THAT_IMPLY_DIVISION =
          
        

        
{
  ')' => true,
  ']' => true,
  '}' => true,
}

      
        
BYTESIZE_METHOD =
          

  

    

Determine the method to use to measure String length in bytes, because StringScanner#pos can only be set in bytes.


  • 

    In Ruby 1.8 String#length returns always the string length in bytes.
    

  • 

    In Ruby 1.9+ String#length returns string length in characters and we need to use String#bytesize instead.
    




  





  



        

        
"".respond_to?(:bytesize) ? :bytesize : :length

      
        
WHITESPACE_REGEX =
          

  

    

JavaScript whitespace can consist of any Unicode space separator characters.


  • 

    In Ruby 1.9+ we can just use the [[:space:]] character class and match them all.
    

  • 

    In Ruby 1.8 we need a regex that identifies the specific bytes in UTF-8 text.
    




  





  



        

        
"".respond_to?(:encoding) ? /[[:space:]]+/m : %r{
  (
    \xC2\xA0     |   # no-break space
    \xE1\x9A\x80 |   # ogham space mark
    \xE2\x80\x80 |   # en quad
    \xE2\x80\x81 |   # em quad
    \xE2\x80\x82 |   # en space
    \xE2\x80\x83 |   # em space
    \xE2\x80\x84 |   # three-per-em space
    \xE2\x80\x85 |   # four-pre-em süace
    \xE2\x80\x86 |   # six-per-em space
    \xE2\x80\x87 |   # figure space
    \xE2\x80\x88 |   # punctuation space
    \xE2\x80\x89 |   # thin space
    \xE2\x80\x8A |   # hair space
    \xE2\x80\xA8 |   # line separator
    \xE2\x80\xA9 |   # paragraph separator
    \xE2\x80\xAF |   # narrow no-break space
    \xE2\x81\x9F |   # medium mathematical space
    \xE3\x80\x80     # ideographic space
  )+
}mx

      
        
WORD_CHARS =
          
        

        
(('a'..'z').to_a + ('A'..'Z').to_a + ['_', '$']).freeze

      
        
DIGITS =
          
        

        
('0'..'9').to_a.freeze

      
    

  







  
    

      Instance Method Summary
      collapse
    


    
  



  
Constructor Details

  
    

  

  
    #initialize(&block)  ⇒ Tokenizer 
  

  

  




  

    

Returns a new instance of Tokenizer.



  





  



  

    
      


116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187

    		
    
      
# File 'lib/rkelly/tokenizer.rb', line 116

def initialize(&block)
  @lexemes = Hash.new {|hash, key| hash[key] = [] }

  token(:COMMENT, /\/(?:\*(?:.)*?\*\/|\/[^\n]*)/m, ['/'])
  token(:STRING, /"(?:[^"\\]*(?:\\.[^"\\]*)*)"|'(?:[^'\\]*(?:\\.[^'\\]*)*)'/m, ["'", '"'])

  # Matcher for basic ASCII whitespace.
  # (Unicode whitespace is handled separately in #match_lexeme)
  #
  # Can't use just "\s" in regex, because in Ruby 1.8 this
  # doesn't include the vertical tab "\v" character
  token(:S, /[ \t\r\n\f\v]*/m, [" ", "\t", "\r", "\n", "\f", "\v"])

  # A regexp to match floating point literals (but not integer literals).

  token(:NUMBER, /\d+\.\d*(?:[eE][-+]?\d+)?|\d+(?:\.\d*)?[eE][-+]?\d+|\.\d+(?:[eE][-+]?\d+)?/m, DIGITS+['.']) do |type, value|
    value.gsub!(/\.(\D)/, '.0\1') if value =~ /\.\w/
    #value.gsub!(/\.$/, '.0') if value.end_with? '.'
    #value.gsub!(/^\./, '0.') if value.start_with? '.'
    [type, value.to_f]
  end
  token(:NUMBER, /0[xX][\da-fA-F]+|0[oO][0-7]+|0[0-7]*|\d+/, DIGITS) do |type, value|
    [type, value.to_i(0)]
  end

  token(:RAW_IDENT, /([_\$A-Za-z][_\$0-9A-Za-z]*)/, WORD_CHARS) do |type,value|
    if KEYWORDS[value]
      [KEYWORDS[value], value]
    elsif RESERVED[value]
      [:RESERVED, value]
    else
      [:IDENT, value]
    end
  end

  # To distinguish regular expressions from comments, we require that
  # regular expressions start with a non * character (ie, not look like
  # /*foo*/). Note that we can't depend on the length of the match to
  # correctly distinguish, since `/**/i` is longer if matched as a regular
  # expression than as matched as a comment.
  # Incidentally, we're also not matching empty regular expressions
  # (eg, // and //g). Here we could depend on match length and priority to
  # determine that these are actually comments, but it turns out to be
  # easier to not match them in the first place.
  token(:REGEXP, %r{
         /                  (?# beginning )

         (?:
           [^\r\n\[/\\]+      (?# any char except \r \n [ / \ )
           |
           \\ [^\r\n]         (?# escape sequence )
           |
           \[ (?:[^\]\\]|\\.)* \]   (?# [...] can contain any char including / )
                                    (?# only \ and ] have to be escaped here )
         )+

         /[gimuy]*          (?# ending + modifiers )
  }x, ['/'])

  literal_chars = LITERALS.keys.map {|k| k.slice(0,1) }.uniq
  literal_regex = Regexp.new(LITERALS.keys.sort_by { |x|
      x.length
    }.reverse.map { |x| "#{x.gsub(/([|+*^])/, '\\\\\1')}" }.join('|'))
  token(:LITERALS, literal_regex, literal_chars) do |type, value|
    [LITERALS[value], value]
  end

  symbols = ('!'..'/').to_a + (':'..'@').to_a + ('['..'^').to_a + ['`'] + ('{'..'~').to_a
  token(:SINGLE_CHAR, /./, symbols) do |type, value|
    [value, value]
  end
end

    		
  





  




  

    
Instance Method Details


    
      

  

  
    #raw_tokens(string)  ⇒ Object 
  

  

  




  

    
      


193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210

    		
    
      
# File 'lib/rkelly/tokenizer.rb', line 193

def raw_tokens(string)
  scanner = StringScanner.new(string)
  tokens = []
  range = CharRange::EMPTY
  accepting_regexp = true
  while !scanner.eos?
    token = match_lexeme(scanner, accepting_regexp)

    if token.name != :S
      accepting_regexp = followable_by_regex(token)
    end

    scanner.pos += token.value.send(BYTESIZE_METHOD)
    token.range = range = range.next(token.value)
    tokens << token
  end
  tokens
end

    		
  





    
      

  

  
    #tokenize(string)  ⇒ Object 
  

  

  




  

    
      


189
190
191

    		
    
      
# File 'lib/rkelly/tokenizer.rb', line 189

def tokenize(string)
  raw_tokens(string).map { |x| x.to_racc_token }
end