Class: Nydp::Tokeniser

Inherits:
Object
  • Object
show all
Defined in:
lib/nydp/tokeniser.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(reader) ⇒ Tokeniser

Returns a new instance of Tokeniser.



7
8
9
10
11
 
# File 'lib/nydp/tokeniser.rb', line 7

def initialize reader
  @reader = reader
  @scanner = StringScanner.new("")
  @state = :lisp
end

Instance Attribute Details

#finished ⇒ Object

Returns the value of attribute finished.



5
6
7
 
# File 'lib/nydp/tokeniser.rb', line 5

def finished
  @finished
end

#state ⇒ Object

Returns the value of attribute state.



5
6
7
 
# File 'lib/nydp/tokeniser.rb', line 5

def state
  @state
end

Instance Method Details

#close_delimiter?(scanner, delim) ⇒ Boolean

Returns:

  • (Boolean) 


23
24
25
26
 
# File 'lib/nydp/tokeniser.rb', line 23

def close_delimiter? scanner, delim
  return (no_more? ? '' : nil) if (delim == :eof)
  scanner.scan(delim)
end

#next_string_fragment(open_delimiter, close_delimiter) ⇒ Object 



28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
 
# File 'lib/nydp/tokeniser.rb', line 28

def next_string_fragment open_delimiter, close_delimiter
  s = @scanner
  rep = "#{open_delimiter}"
  string = ""
  while (!no_more?)
    if esc = s.scan(/\\/)
      rep    << esc
      ch = s.getch
      case ch
        when "n" ; string << "\n"
        when "t" ; string << "\t"
        else       string << ch
      end
      rep << ch
    elsif closer = close_delimiter?(s, close_delimiter)
      rep << closer
      return StringFragmentCloseToken.new(string, rep)
    elsif start_interpolation = s.scan(/~/)
      rep << start_interpolation
      return StringFragmentToken.new(string, rep)
    else
      ch = s.getch
      string << ch
      rep    << ch
    end
  end

  return StringFragmentCloseToken.new(string, rep) if close_delimiter == :eof
end

#next_token ⇒ Object 



58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/nydp/tokeniser.rb', line 58

def next_token
  s = @scanner
  tok = nil
  while !tok
    if no_more?
      @finished = true
      return nil
    elsif comment = s.scan(/;.*$/)
      tok = [:comment, comment[1..-1].strip]
    elsif open_str = s.scan(/"/)
      tok = [:string_open_delim, open_str]
    elsif list_prefix = s.scan(/[^\s()]*\(/)
      tok = [:left_paren, list_prefix[0...-1]]
    elsif list_prefix = s.scan(/[^\s()]*\{/)
      tok = [:left_brace, list_prefix[0...-1]]
    elsif s.scan(/\)/)
      tok = [:right_paren]
    elsif number = s.scan(/[-+]?[0-9]*\.[0-9]+([eE][-+]?[0-9]+)?/)
      tok = [:number, number.to_f]
    elsif integer = s.scan(/[-+]?[0-9]+/)
      tok = [:number, integer.to_i]
    elsif atom = s.scan(/[^\s()"{}]+/)
      tok = [:symbol, atom]
    else
      s.getch
    end
  end
  tok
end

#no_more? ⇒ Boolean

Returns:

  • (Boolean) 


13
14
15
16
17
18
19
20
21
 
# File 'lib/nydp/tokeniser.rb', line 13

def no_more?
  if @scanner.eos?
    nextline = @reader.nextline
    return true if nextline.nil?
    @scanner << nextline
  end

  false
end