Class: CqlRuby::CqlLexer

Inherits:
Object
  • Object
show all
Defined in:
lib/cql_ruby/cql_lexer.rb

Overview

This is derived from java based a semi-trivial subclass for java.io.StreamTokenizer that:

* Includes a render() method
* Knows about the multi-character tokens "<=", ">=" and "<>"
* Recognises a set of keywords as tokens in their own right
* Includes some primitive debugging-output facilities

It’s used only by CQLParser

Constant Summary collapse

TT_EOF = 
1
TT_EOL = 
2
TT_NUMBER = 
3
TT_WORD = 
4
TT_LE =

The “<=” relation

1000
TT_GE =

The “>=” relation

1001
TT_NE =

The “<>” relation

1002
TT_EQEQ =

The “==” relation

1003
TT_AND =

The “and” boolean

1004
TT_OR =

The “or” boolean

1005
TT_NOT =

The “not” boolean

1006
TT_PROX =

The “prox” boolean

1007
TT_SORTBY =

The “sortby” operator

1008
@@keywords = 
{ "and" => CqlLexer::TT_AND, "or" => CqlLexer::TT_OR, "not" => CqlLexer::TT_NOT,
"prox" => CqlLexer::TT_PROX, "sortby" => CqlLexer::TT_SORTBY}
@@symbol_tokens = 
@@keywords.values + [CqlLexer::TT_WORD,CqlLexer::TT_NUMBER,'"']
@@ordinary_chars = 
/[=<>\/()]/
@@re_any_token_start = 
/[\w(\)<>\/="*]/
@@re_string_end = 
/[ \t(\)<>\/="]/

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(s = "", debug = false) ⇒ CqlLexer

Returns a new instance of CqlLexer.



35
36
37
38
39
 
# File 'lib/cql_ruby/cql_lexer.rb', line 35

def initialize( s="", debug=false )
  debug
  @simple_tokens = tokenize( s )
  @index = -1
end

Instance Attribute Details

#index ⇒ Object

Returns the value of attribute index.



11
12
13
 
# File 'lib/cql_ruby/cql_lexer.rb', line 11

def index
  @index
end

#simple_tokens ⇒ Object

Returns the value of attribute simple_tokens.



11
12
13
 
# File 'lib/cql_ruby/cql_lexer.rb', line 11

def simple_tokens
  @simple_tokens
end

#token_type ⇒ Object

Returns the value of attribute token_type.



11
12
13
 
# File 'lib/cql_ruby/cql_lexer.rb', line 11

def token_type
  @token_type
end

#tokenizer ⇒ Object

Returns the value of attribute tokenizer.



11
12
13
 
# File 'lib/cql_ruby/cql_lexer.rb', line 11

def tokenizer
  @tokenizer
end

#value ⇒ Object

Returns the value of attribute value.



11
12
13
 
# File 'lib/cql_ruby/cql_lexer.rb', line 11

def value
  @value
end

Class Method Details

.symbol_tokens ⇒ Object 



41
42
43
 
# File 'lib/cql_ruby/cql_lexer.rb', line 41

def CqlLexer.symbol_tokens
  @@symbol_tokens
end

Instance Method Details

#and? ⇒ Boolean

Returns:

  • (Boolean) 


208
209
210
 
# File 'lib/cql_ruby/cql_lexer.rb', line 208

def and?
  @token_type == CqlLexer::TT_AND
end

#double_equal? ⇒ Boolean

Returns:

  • (Boolean) 


199
200
201
 
# File 'lib/cql_ruby/cql_lexer.rb', line 199

def double_equal?
  @token_type == CqlLexer::TT_EQEQ
end

#eof? ⇒ Boolean

Returns:

  • (Boolean) 


181
182
183
 
# File 'lib/cql_ruby/cql_lexer.rb', line 181

def eof?
  @token_type == CqlLexer::TT_EOF
end

#eol? ⇒ Boolean

Returns:

  • (Boolean) 


184
185
186
 
# File 'lib/cql_ruby/cql_lexer.rb', line 184

def eol?
  @token_type == CqlLexer::TT_EOL
end

#find_quoted_string_end(s, position) ⇒ Object

Raises:



49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
 
# File 'lib/cql_ruby/cql_lexer.rb', line 49

def find_quoted_string_end( s, position )

  is_backslashed = false
  for i in position+1..s.length
    if s[i..i] == '\\'
      is_backslashed = ! is_backslashed
    else
      if s[i..i] == '"' and not is_backslashed
        return i + 1
      end
      is_backslashed = false
    end
  end 
  raise CqlException,  "unterminated quoted string at position #{position} in '#{s}'"
  return s.length
end

#find_string_end(s, position) ⇒ Object 



45
46
47
 
# File 'lib/cql_ruby/cql_lexer.rb', line 45

def find_string_end( s, position )
  s.index( @@re_string_end, position) || s.length
end

#greater_than_or_equal? ⇒ Boolean

Returns:

  • (Boolean) 


196
197
198
 
# File 'lib/cql_ruby/cql_lexer.rb', line 196

def greater_than_or_equal?
  @token_type == CqlLexer::TT_GE
end

#less_than_or_equal? ⇒ Boolean

Returns:

  • (Boolean) 


193
194
195
 
# File 'lib/cql_ruby/cql_lexer.rb', line 193

def less_than_or_equal?
  @token_type == CqlLexer::TT_LE
end

#next_token ⇒ Object 



96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
 
# File 'lib/cql_ruby/cql_lexer.rb', line 96

def next_token
  underlying_next_token
  return CqlLexer::TT_EOF if eof? or eol?
  
  if @token_type == '<'
    underlying_next_token
    if @token_type == '=' 
      @token_type = CqlLexer::TT_LE
    elsif @token_type == ">"
      @token_type = CqlLexer::TT_NE
    else
      push_back
      @token_type = '<'
    end
  elsif @token_type == '>'
    underlying_next_token
    if @token_type == '=' 
      @token_type = CqlLexer::TT_GE
    else
      push_back
      @token_type = '>'
    end
  elsif @token_type == '='
    underlying_next_token
    if @token_type == '=' 
      @token_type = CqlLexer::TT_EQEQ
    else
      push_back
      @token_type = '='
    end
  end
  @token_type
end

#not? ⇒ Boolean

Returns:

  • (Boolean) 


205
206
207
 
# File 'lib/cql_ruby/cql_lexer.rb', line 205

def not?
  @token_type == CqlLexer::TT_NOT
end

#not_equal? ⇒ Boolean

Returns:

  • (Boolean) 


202
203
204
 
# File 'lib/cql_ruby/cql_lexer.rb', line 202

def not_equal?
  @token_type == CqlLexer::TT_NE
end

#number? ⇒ Boolean

Returns:

  • (Boolean) 


187
188
189
 
# File 'lib/cql_ruby/cql_lexer.rb', line 187

def number?
  @token_type == CqlLexer::TT_NUMBER
end

#or? ⇒ Boolean

Returns:

  • (Boolean) 


211
212
213
 
# File 'lib/cql_ruby/cql_lexer.rb', line 211

def or?
  @token_type == CqlLexer::TT_OR
end

#prox? ⇒ Boolean

Returns:

  • (Boolean) 


214
215
216
 
# File 'lib/cql_ruby/cql_lexer.rb', line 214

def prox?
  @token_type == CqlLexer::TT_PROX
end

#push_back ⇒ Object 



152
153
154
 
# File 'lib/cql_ruby/cql_lexer.rb', line 152

def push_back
  @index -= 1
end

#render(token = nil, quote_chars = true) ⇒ Object 



156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
 
# File 'lib/cql_ruby/cql_lexer.rb', line 156

def render( token=nil, quote_chars=true )
  token = @token_type unless token
  case token
    when CqlLexer::TT_EOF then "EOF"
    when CqlLexer::TT_NUMBER then @value
    when CqlLexer::TT_WORD then "word:#{@value}"
    when "'" then "string:\"#{@value}\""
    when CqlLexer::TT_LE then "<="
    when CqlLexer::TT_GE then ">="
    when CqlLexer::TT_NE then "<>"
    when CqlLexer::TT_EQEQ then "=="
    when CqlLexer::TT_EOF then "EOF"
  else
    if @@keywords.has_value?( @value )
      @value
    else
      if quote_chars 
        "'#{token}'"
      else
        token
      end
    end
  end
end

#sortby? ⇒ Boolean

Returns:

  • (Boolean) 


217
218
219
 
# File 'lib/cql_ruby/cql_lexer.rb', line 217

def sortby?
  @token_type == CqlLexer::TT_SORTBY
end

#tokenize(cql_string) ⇒ Object 



66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
 
# File 'lib/cql_ruby/cql_lexer.rb', line 66

def tokenize( cql_string )
  position = 0
  previous_backslash = false
  length = cql_string.length
  
  @tokens = []
  while position < length
    token_begin = cql_string.index( @@re_any_token_start, position )
    break unless token_begin
    

    case cql_string[token_begin..token_begin]
      when @@ordinary_chars
        token_end = token_begin+1
      when /[\w*]/
        token_end = find_string_end( cql_string, token_begin )
      when '"'
        token_end = find_quoted_string_end( cql_string, token_begin )
      else
        token_end = token_begin+1
    end
    
    @tokens << cql_string[token_begin..token_end-1]
    position = token_end
  end
  
  # puts "tokens=#{@tokens.inspect}"
  @tokens
end

#underlying_next_token ⇒ Object 



130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
 
# File 'lib/cql_ruby/cql_lexer.rb', line 130

def underlying_next_token
  @index += 1
  if @index >= @simple_tokens.length
    @token_type = CqlLexer::TT_EOF
    @value = nil
    return
  end
  @value = @simple_tokens[ @index ]
  if /^[0-9]+$/ =~ @value
    @token_type = CqlLexer::TT_NUMBER
  elsif @value.length > 1
    if @value.slice(0..0) == '"'
      @token_type = '"'
      @value = @value.slice(1..-2)
    else
      @token_type = @@keywords[ @value.downcase ] || CqlLexer::TT_WORD
    end
  else
    @token_type = @value
  end
end

#word? ⇒ Boolean

Returns:

  • (Boolean) 


190
191
192
 
# File 'lib/cql_ruby/cql_lexer.rb', line 190

def word?
  @token_type == CqlLexer::TT_WORD
end