6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
|
# File 'lib/rvm/languages/math/tokenizer.rb', line 6
def Tokenizer.tokenize str
s = StringScanner.new str
tokens = []
state = :number
paren_deepth = 0
while not s.eos?
s.skip(/\s*/)
if state == :number
if s.scan(/[+-]+/)
t = s.matched.gsub('+','').gsub('--','')
if t == '-'
tokens << [t, :function]
end
elsif s.scan(/[_a-z][_a-z0-9]*/i)
r = s.matched
if s.scan(/\(/)
tokens << [r, :function]
if s.scan(/\)/)
tokens << [s.matched, :paren_close]
else
paren_deepth += 1
end
else
tokens << [r, :ident]
state = :opperator
end
elsif s.scan(/((\d+(\.\d+)?)|(\.\d+))(e-?\d+)?/i)
tokens << [s.matched, :number]
state = :opperator
elsif s.scan(/\(/)
paren_deepth += 1
tokens << [s.matched, :paren_open]
else
raise "Unknown literal in term at positin #{s.pos}."
end
else
if s.scan(/\)/)
paren_deepth -= 1
raise "Unmatched parenthes at #{s.pos}" if paren_deepth < 0
tokens << [s.matched, :paren_close]
elsif s.scan(/,/)
tokens << [s.matched, :function_sep]
state = :number
elsif s.scan(/[*+\/^=-]/)
tokens << [s.matched, :opperator]
state = :number
elsif s.scan(/;/)
if paren_deepth == 0
tokens << [s.matched, :opperator]
state = :number
else
raise "Expression sepperator enclosed in parenthes at #{s.pos}"
end
else
raise "Unknown literal in term at positin #{s.pos}."
end
end
end
tokens
end
|