Class: Koara::TokenManager

Inherits:
Object
  • Object
show all
Defined in:
lib/koara/token_manager.rb

Constant Summary collapse

EOF = 
0
ASTERISK = 
1
BACKSLASH = 
2
BACKTICK = 
3
CHAR_SEQUENCE = 
4
COLON = 
5
DASH = 
6
DIGITS = 
7
DOT = 
8
EOL = 
9
EQ = 
10
ESCAPED_CHAR = 
11
GT = 
12
IMAGE_LABEL = 
13
LBRACK = 
14
LPAREN = 
15
LT = 
16
RBRACK = 
17
RPAREN = 
18
SPACE = 
19
TAB = 
20
UNDERSCORE = 
21

Instance Method Summary collapse

Constructor Details

#initialize(stream) ⇒ TokenManager

Returns a new instance of TokenManager.



31
32
33
34
35
36
37
 
# File 'lib/koara/token_manager.rb', line 31

def initialize(stream)
  @jj_rounds = Array.new(8, 0)
  @jj_state_set = Array.new(16, 0)
  @jj_next_states = [2, 3, 5]
  @cs = stream
  @round = 0
end

Instance Method Details

#check_n_add(state) ⇒ Object 



313
314
315
316
317
318
319
 
# File 'lib/koara/token_manager.rb', line 313

def check_n_add(state)
  if @jj_rounds[state] != @round
    @jj_state_set[@jj_new_state_cnt] = state
    @jj_new_state_cnt += 1
    @jj_rounds[state] = @round
  end
end

#check_n_add_states(start, ending) ⇒ Object 



305
306
307
308
309
310
311
 
# File 'lib/koara/token_manager.rb', line 305

def check_n_add_states(start, ending)
  loop do
    check_n_add(@jj_next_states[start])
    break if start == ending
    start += 1
  end
end

#fill_token ⇒ Object 



66
67
68
 
# File 'lib/koara/token_manager.rb', line 66

def fill_token
  Token.new(@matched_kind, @cs.begin_line, @cs.begin_column, @cs.end_line, @cs.end_column, @cs.image)
end

#get_next_token ⇒ Object 



39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
 
# File 'lib/koara/token_manager.rb', line 39

def get_next_token
  #   begin
  while true
    begin
      @cur_char = @cs.begin_token
    rescue
      @matched_kind = 0
      @matched_pos = -1
      return fill_token
    end

    @matched_kind = 2147483647
    @matched_pos = 0
    cur_pos = move_string_literal_dfa0

    if @matched_kind != 2147483647
      if (@matched_pos + 1) < cur_pos
        @cs.backup(cur_pos - @matched_pos - 1)
      end
      return fill_token
    end
  end
  #    rescue => err
  #      return nil
  #    end
end

#move_nfa(start_state, cur_pos) ⇒ Object 



174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
 
# File 'lib/koara/token_manager.rb', line 174

def move_nfa(start_state, cur_pos)
  starts_at = 0
  @jj_new_state_cnt = 8
  i = 1
  @jj_state_set[0] = start_state
  kind = 0x7fffffff

  while true
    if (@round += 1) == 0x7fffffff
      @round = 0x80000001
    end
    if @cur_char.ord < 64
      l = 1 << @cur_char.ord
      loop do
        i-=1

        case @jj_state_set[i]
          when 6
            if (0x880098feffffd9ff & l) != 0
              kind = 4 if kind > 4
              check_n_add(0)
            elsif ((0x3ff000000000000 & l) != 0)
              kind = 7 if kind > 7
              check_n_add(1)
            elsif (0x2400 & l) != 0
              kind = 9 if kind > 9
            elsif (0x100000200 & l) != 0
              check_n_add_states(0, 2)
            end
            if @cur_char.ord == 13
              @jj_state_set[@jj_new_state_cnt+=1] = 4
            end
          when 8
            if ((0x2400 & l) != 0)
              kind = 9 if kind > 9
            elsif (0x100000200 & l) != 0
              check_n_add_states(0, 2)
            end
            if @cur_char.ord == 13
              @jj_state_set[@jj_new_state_cnt+=1] = 4
            end
          when 0
            if (0x880098feffffd9ff & l) != 0
              kind = 4
              check_n_add(0)
            end
          when 1
            if (0x3ff000000000000 & l) != 0
              kind = 7 if kind > 7
              check_n_add(1)
            end
          when 2
            if (0x100000200 & l) != 0
              check_n_add_states(0, 2)
            end
          when 3
            if (0x2400 & l) != 0 && kind > 9
              kind = 9
            end
          when 4
            if @cur_char.ord == 10 && kind > 9
              kind = 9
            end
          when 5
            if @cur_char.ord == 13
              @jj_state_set[@jj_new_state_cnt+=1] = 4
            end
          when 7
            if (0x77ff670000000000 & l) != 0 && kind > 11
              kind = 11
            end
        end
        break if (i == starts_at)
      end
    elsif @cur_char.ord < 128
      l = 1 << (@cur_char.ord & 077)
      loop do
        i -= 1
        case @jj_state_set[i]
          when 6
            if l != 0
              kind = 4 if kind > 4
              check_n_add(0)
            elsif @cur_char == 92
              @jj_state_set[@jj_new_state_cnt+=1] = 7
            end
          when 0
            if (0xfffffffe47ffffff & l) != 0
              kind = 4
              check_n_add(0)
            end
          when 7
            if (0x1b8000000 & l) != 0 && kind > 11
              kind = 11
            end
        end
        break if (i == starts_at)
      end
    else
      loop do
        i-=1
        case @jj_state_set[i]
          when 6
            kind = 4 if kind > 4
            check_n_add(0)
          when 0
            kind = 4 if kind > 4
            check_n_add(0)
        end
        break if i == starts_at
      end
    end
    if kind != 0x7fffffff
      @matched_kind = kind
      @matched_pos = cur_pos
      kind = 0x7fffffff
    end
    cur_pos += 1

    if (i = @jj_new_state_cnt) == (starts_at = 8 - (@jj_new_state_cnt = starts_at))
      return cur_pos
    end

    begin
      @cur_char = @cs.read_char
    rescue => error
      return cur_pos
    end
  end
end

#move_string_literal_dfa0 ⇒ Object 



70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
 
# File 'lib/koara/token_manager.rb', line 70

def move_string_literal_dfa0
  case @cur_char.ord
    when 9
      return start_nfa_with_states(0, TAB, 8)
    when 32
      return start_nfa_with_states(0, SPACE, 8)
    when 40
      return stop_at_pos(0, LPAREN)
    when 41
      return stop_at_pos(0, RPAREN)
    when 42
      return stop_at_pos(0, ASTERISK)
    when 45
      return stop_at_pos(0, DASH)
    when 46
      return stop_at_pos(0, DOT)
    when 58
      return stop_at_pos(0, COLON)
    when 60
      return stop_at_pos(0, LT)
    when 61
      return stop_at_pos(0, EQ)
    when 62
      return stop_at_pos(0, GT)
    when 73
      return move_string_literal_dfa1(0x2000)
    when 91
      return stop_at_pos(0, LBRACK)
    when 92
      return start_nfa_with_states(0, BACKSLASH, 7)
    when 93
      return stop_at_pos(0, RBRACK)
    when 95
      return stop_at_pos(0, UNDERSCORE)
    when 96
      return stop_at_pos(0, BACKTICK)
    when 105
      return move_string_literal_dfa1(0x2000)
    else
      return move_nfa(6, 0)
  end
end

#move_string_literal_dfa1(active) ⇒ Object 



130
131
132
133
134
135
136
 
# File 'lib/koara/token_manager.rb', line 130

def move_string_literal_dfa1(active)
  @cur_char = @cs.read_char
  if @cur_char.ord == 77 || @cur_char.ord == 109
    return move_string_literal_dfa2(active, 0x2000)
  end
  start_nfa(0, active)
end

#move_string_literal_dfa2(old, active) ⇒ Object 



138
139
140
141
142
143
144
 
# File 'lib/koara/token_manager.rb', line 138

def move_string_literal_dfa2(old, active)
  @cur_char = @cs.read_char
  if @cur_char.ord == 65 || @cur_char.ord == 97
    return move_string_literal_dfa3(active, 0x2000)
  end
  start_nfa(1, active)
end

#move_string_literal_dfa3(old, active) ⇒ Object 



146
147
148
149
150
151
152
 
# File 'lib/koara/token_manager.rb', line 146

def move_string_literal_dfa3(old, active)
  @cur_char = @cs.read_char
  if @cur_char.ord == 71 || @cur_char.ord == 103
    return move_string_literal_dfa4(active, 0x2000)
  end
  start_nfa(2, active)
end

#move_string_literal_dfa4(old, active) ⇒ Object 



154
155
156
157
158
159
160
 
# File 'lib/koara/token_manager.rb', line 154

def move_string_literal_dfa4(old, active)
  @cur_char = @cs.read_char
  if @cur_char.ord == 69 || @cur_char.ord == 101
    return move_string_literal_dfa5(active, 0x2000)
  end
  start_nfa(3, active)
end

#move_string_literal_dfa5(old, active) ⇒ Object 



162
163
164
165
166
167
168
 
# File 'lib/koara/token_manager.rb', line 162

def move_string_literal_dfa5(old, active)
  @cur_char = @cs.read_char()
  if @cur_char.ord == 58 && ((active & 0x2000) != 0)
    return stop_at_pos(5, 13)
  end
  start_nfa(4, active)
end

#start_nfa(pos, active) ⇒ Object 



170
171
172
 
# File 'lib/koara/token_manager.rb', line 170

def start_nfa(pos, active)
  move_nfa(stop_string_literal_dfa(pos, active), pos + 1)
end

#start_nfa_with_states(pos, kind, state) ⇒ Object 



113
114
115
116
117
118
119
120
121
122
 
# File 'lib/koara/token_manager.rb', line 113

def start_nfa_with_states(pos, kind, state)
  @matched_kind = kind
  @matched_pos = pos
  begin
    @cur_char = @cs.read_char
  rescue
    return pos + 1
  end
  move_nfa(state, pos + 1)
end

#stop_at_pos(pos, kind) ⇒ Object 



124
125
126
127
128
 
# File 'lib/koara/token_manager.rb', line 124

def stop_at_pos(pos, kind)
  @matched_kind = kind
  @matched_pos = pos
  pos + 1
end

#stop_string_literal_dfa(pos, active) ⇒ Object 



321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
 
# File 'lib/koara/token_manager.rb', line 321

def stop_string_literal_dfa(pos, active)
  if pos == 0
    if (active & 0x2000) != 0
      @matched_kind = 4
      return 0
    elsif (active & 0x180000) != 0
      return 8
    elsif (active & 0x4) != 0
      return 7
    end
  elsif pos == 1 && (active & 0x2000) != 0
    @matched_kind = 4
    @matched_pos = 1
    return 0
  elsif pos == 2 && (active & 0x2000) != 0
    @matched_kind = 4
    @matched_pos = 2
    return 0
  elsif pos == 3 && (active & 0x2000) != 0
    @matched_kind = 4
    @matched_pos = 3
    return 0
  elsif pos == 4 && (active & 0x2000) != 0
    @matched_kind = 4
    @matched_pos = 4
    return 0
  end
  -1
end