Class: MongoRecord::SQL::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/mongo_record/sql.rb

Overview

A simple tokenizer for SQL.

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(sql) ⇒ Tokenizer

Returns a new instance of Tokenizer.



24
25
26
27
28
29
 
# File 'lib/mongo_record/sql.rb', line 24

def initialize(sql)
  @sql = sql
  @length = sql.length
  @pos = 0
  @extra_tokens = []
end

Instance Attribute Details

#sql ⇒ Object (readonly)

Returns the value of attribute sql.



22
23
24
 
# File 'lib/mongo_record/sql.rb', line 22

def sql
  @sql
end

Instance Method Details

#add_extra_token(tok) ⇒ Object

Push tok onto the stack.



32
33
34
 
# File 'lib/mongo_record/sql.rb', line 32

def add_extra_token(tok)
  @extra_tokens.push(tok)
end

#identifier_char?(c) ⇒ Boolean

Return true if the next character is a legal starting identifier character.

Returns:

  • (Boolean) 


81
82
83
 
# File 'lib/mongo_record/sql.rb', line 81

def identifier_char?(c)
  c =~ /[\.a-zA-Z0-9_]/ ? true : false
end

#more? ⇒ Boolean

Return true if there are more non-whitespace characters.

Returns:

  • (Boolean) 


45
46
47
48
 
# File 'lib/mongo_record/sql.rb', line 45

def more?
  skip_whitespace
  @pos < @length
end

#next_string(c) ⇒ Object

Return the next string without its surrounding quotes. Assumes we have already seen a quote character.



52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/mongo_record/sql.rb', line 52

def next_string(c)
  q = c
  @pos += 1
  t = ''
  while @pos < @length
    c = @sql[@pos, 1]
    case c
    when q
      if @pos + 1 < @length && @sql[@pos + 1, 1] == q # double quote
        t += q
        @pos += 1
      else
        @pos += 1
        return t
      end
    when '\\'
      @pos += 1
      return t if @pos >= @length
      t << @sql[@pos, 1]
    else
      t << c
    end
    @pos += 1
  end
  raise "unterminated string in SQL: #{@sql}"
end

#next_token ⇒ Object

Return the next token, or nil if there are no more.



91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
 
# File 'lib/mongo_record/sql.rb', line 91

def next_token
  return @extra_tokens.pop unless @extra_tokens.empty?

  skip_whitespace
  c = @sql[@pos, 1]
  return next_string(c) if quote?(c)

  first_is_identifier_char = identifier_char?(c)
  t = c
  @pos += 1
  while @pos < @length
    c = @sql[@pos, 1]
    break if c == ' '

    this_is_identifier_char = identifier_char?(c)
    break if first_is_identifier_char != this_is_identifier_char && @length > 0
    break if !this_is_identifier_char && quote?(c)

    t << c
    @pos += 1
  end

  case t
  when ''
    nil
  when /^\d+$/
    t.to_i
  else
    t
  end
end

#quote?(c) ⇒ Boolean

Return true if c is a single or double quote character.

Returns:

  • (Boolean) 


86
87
88
 
# File 'lib/mongo_record/sql.rb', line 86

def quote?(c)
  c == '"' || c == "'"
end

#skip_whitespace ⇒ Object

Skips whitespace, setting @pos to the position of the next non-whitespace character. If there is none, @pos will == @length.



38
39
40
41
42
 
# File 'lib/mongo_record/sql.rb', line 38

def skip_whitespace
  while @pos < @length && [" ", "\n", "\r", "\t"].include?(@sql[@pos,1])
    @pos += 1
  end
end