Class: CTokenizer::Lexer

Inherits:
StringScanner
  • Object
show all
Includes:
CTokenizer
Defined in:
lib/caphir/ctokenizer.rb

Constant Summary

Constants included from CTokenizer

CP_RESERVED, C_RESERVED, EOF_TOKEN

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from CTokenizer

#collect, #each, #error, error, line_count, #parse_error, #to_a, #token_error, #unmatched_error, #warning

Constructor Details

#initialize(str, file = nil, line = 1) ⇒ Lexer

Returns a new instance of Lexer.



123
124
125
126
127
128
 
# File 'lib/caphir/ctokenizer.rb', line 123

def initialize(str, file=nil, line=1)
	str.freeze
	super(str, false) # DO NOT dup str
	@file = file
	@line = line
end

Instance Attribute Details

#file ⇒ Object

the current file (may be nil)



131
132
133
 
# File 'lib/caphir/ctokenizer.rb', line 131

def file
  @file
end

#line ⇒ Object

the current line number



134
135
136
 
# File 'lib/caphir/ctokenizer.rb', line 134

def line
  @line
end

Instance Method Details

#scan(re) ⇒ Object 



140
141
142
143
144
145
 
# File 'lib/caphir/ctokenizer.rb', line 140

def scan(re)
	if m = super(re)
		@line += CTokenizer.line_count(m)
	end
	m
end

#scan_super ⇒ Object 



138
 
# File 'lib/caphir/ctokenizer.rb', line 138

alias scan_super scan

#shift ⇒ Object

Returns the next token and moves the position in the input stream.



148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
 
# File 'lib/caphir/ctokenizer.rb', line 148

def shift
	# don't need \A in regexp's cause StringScanner does this automatically.
	# try to put most common tokens first
	t = case
		#when m = scan_super(Expression::SPACE)
		#	@line += CTokenizer.line_count(m)
		#	[:SPACE, m]
		when m = scan_super(Expression::SPACE_1)
			[:SPACE, m]
		when m = scan_super(Expression::IDENTIFIER)
			[:IDENTIFIER, m]
		when m = scan_super(Expression::COMMENT)
			@line += CTokenizer.line_count(m)
			[:COMMENT, m]
		# SYMBOL should come after CONSTANT
		when m = scan_super(Expression::CONSTANT)
			[:CONSTANT, m]
		when m = scan_super(Expression::SYMBOL)
			[:SYMBOL, m]
		when m = scan_super(Expression::NEWLINE)
			#@line += CTokenizer.line_count(m)
			@line += 1
			[:NEWLINE, m]
		when m = scan_super(Expression::STRING)
			[:STRING, m]
		when m = scan_super(Expression::SPACE_2)
			@line += 1
			[:SPACE, m]
		when m = scan_super(Expression::CHARACTER)
			[:CHARACTER, m]
		when eos?
			EOF_TOKEN # end of file, \Z don't work with StringScanner
		else
			[:UNKNOWN, getch]
	end # case
	m.freeze
	t.freeze
end