Class: Nydp::Tokeniser

Inherits:
Object show all
Defined in:
lib/nydp/tokeniser.rb

Constant Summary collapse

BACKSLASH = 
/\\/.freeze
COMMENT = 
/;;?.*$/.freeze
QUOTE = 
/"/.freeze
PIPE = 
/\|/.freeze
LIST_PFX = 
/[^\s()]*\(/.freeze
BRACE_PFX = 
/[^\s()\}\{]*\{/.freeze
RPAREN = 
/\)/.freeze
RBRACE = 
/\}/.freeze
FLOAT = 
/[-+]?[0-9]*\.[0-9]+([eE][-+]?[0-9]+)?/.freeze
INTEGER = 
/[-+]?[0-9]+/.freeze
ATOM_PIPE = 
/[^\s()"{}\|]+\|/.freeze
ATOM = 
/[^\s()"{}\|]+/.freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(reader) ⇒ Tokeniser

Returns a new instance of Tokeniser.



20
21
22
23
 
# File 'lib/nydp/tokeniser.rb', line 20

def initialize reader
  @reader = reader
  @scanner = StringScanner.new("")
end

Instance Attribute Details

#finished ⇒ Object

Returns the value of attribute finished.



18
19
20
 
# File 'lib/nydp/tokeniser.rb', line 18

def finished
  @finished
end

#state ⇒ Object

Returns the value of attribute state.



18
19
20
 
# File 'lib/nydp/tokeniser.rb', line 18

def state
  @state
end

Instance Method Details

#close_delimiter?(scanner, delim) ⇒ Boolean

Returns:

  • (Boolean) 


35
36
37
38
 
# File 'lib/nydp/tokeniser.rb', line 35

def close_delimiter? scanner, delim
  return (no_more? ? '' : nil) if (delim == :eof)
  scanner.scan(delim)
end

#next_string_fragment(open_delimiter, close_delimiter, interpolation_sign, interpolation_escapes = { }) ⇒ Object 



40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/nydp/tokeniser.rb', line 40

def next_string_fragment open_delimiter, close_delimiter, interpolation_sign, interpolation_escapes={ }
  s = @scanner
  rep = open_delimiter.to_s
  string = ""
  while (!no_more?)
    if esc = s.scan(BACKSLASH)
      rep    << esc
      ch = s.getch
      case ch
        when "n" ; string << "\n"
        when "t" ; string << "\t"
        else       string << ch
      end
      rep << ch
    elsif closer = close_delimiter?(s, close_delimiter)
      rep << closer
      return StringFragmentCloseToken.new(string, rep)
    elsif interpolation_sign
      escaped = false
      interpolation_escapes.each do |esc, repl|
        if !escaped && (escape = s.scan esc)
          string << ((repl == true) ? escape : repl)
          rep    << escape
          escaped = true
        end
      end
      if !escaped && (start_interpolation = s.scan(interpolation_sign))
        rep << start_interpolation
        return StringFragmentToken.new(string, rep)
      else
        ch = s.getch
        string << ch
        rep    << ch
      end
    else
      ch = s.getch
      string << ch
      rep    << ch
    end
  end

  return StringFragmentCloseToken.new(string, rep) if close_delimiter == :eof
end

#next_token ⇒ Object 



84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
 
# File 'lib/nydp/tokeniser.rb', line 84

def next_token
  s = @scanner
  tok = nil
  while !tok
    if no_more?
      @finished = true
      return nil
    elsif comment = s.scan(COMMENT)
      tok = [:comment, comment.gsub(/^;;?\s*/, '').strip]
    elsif open_str = s.scan(QUOTE)
      tok = [:string_open_delim, open_str]
    elsif open_sym = s.scan(PIPE)
      tok = [:sym_open_delim, open_sym]
    elsif list_prefix = s.scan(LIST_PFX)
      tok = [:left_paren, list_prefix[0...-1]]
    elsif list_prefix = s.scan(BRACE_PFX)
      tok = [:left_brace, list_prefix[0...-1]]
    elsif s.scan(RPAREN)
      tok = [:right_paren]
    elsif s.scan(RBRACE)
      tok = [:right_brace]
    elsif number = s.scan(FLOAT)
      tok = [:number, number.to_f]
    elsif integer = s.scan(INTEGER)
      tok = [:number, integer.to_i]
    elsif atom = s.scan(ATOM_PIPE)
      atom = atom[0...-1]
      rest = next_string_fragment("|", PIPE, nil) || Nydp::StringFragmentToken.new("", "")
      tok = [:symbol, "#{atom}#{rest.string}"]
    elsif atom = s.scan(ATOM)
      tok = [:symbol, atom]
    else
      s.getch
    end
  end
  tok
end

#no_more? ⇒ Boolean

Returns:

  • (Boolean) 


25
26
27
28
29
30
31
32
33
 
# File 'lib/nydp/tokeniser.rb', line 25

def no_more?
  if @scanner.eos?
    nextline = @reader.nextline
    return true if nextline.nil?
    @scanner << nextline
  end

  false
end