74
75
76
77
78
79
80
81
82
83
84
|
# File 'lib/graphlyte/lexer.rb', line 74
def tokenize!
while source_uncompleted?
self.lexeme_start_p = current_position
token = next_token
tokens << token if token
end
tokens << Lexing::Token.new(:EOF, nil, after_source_end_location)
end
|