Class: Layo::Lexer

Inherits:
Object
  • Object
show all
Includes:
Peekable
Defined in:
lib/layo/lexer.rb

Instance Attribute Summary collapse

Attributes included from Peekable

#peek_index

Instance Method Summary collapse

Methods included from Peekable

#next, #peek, #reset_peek, #unpeek

Constructor Details

#initialize(io = nil) ⇒ Lexer

Returns a new instance of Lexer.



10
11
12
 
# File 'lib/layo/lexer.rb', line 10

def initialize(io = nil)
  self.input = io unless io.nil?
end

Instance Attribute Details

#input ⇒ Object

Input stream. Must be an instance of IO class (File, StringIO)



6
7
8
 
# File 'lib/layo/lexer.rb', line 6

def input
  @input
end

#line_no ⇒ Object (readonly)

Current line number (1-based) and position (0-based) of cursor



8
9
10
 
# File 'lib/layo/lexer.rb', line 8

def line_no
  @line_no
end

#pos ⇒ Object (readonly)

Current line number (1-based) and position (0-based) of cursor



8
9
10
 
# File 'lib/layo/lexer.rb', line 8

def pos
  @pos
end

Instance Method Details

#buffer_empty? ⇒ Boolean

Returns true if input buffer is fully processed

Returns:

  • (Boolean) 


163
164
165
 
# File 'lib/layo/lexer.rb', line 163

def buffer_empty?
  @line_no.zero? || @pos > @line.length - 1
end

#escape_string(str) ⇒ Object

Performs substitution of escape characters in string



142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
 
# File 'lib/layo/lexer.rb', line 142

def escape_string(str)
  replacement = {
    ':)' => "\n", ':>' => "\t", ':o' => "\a", ':"' => '"', '::' => ':'
  }
  str
    .gsub(/:[\)>o":]/, replacement)
    .gsub(/:\(([0-9a-fA-F]+)\)/) do |match|
      $1.to_i(16).chr(Encoding::UTF_8)
    end
    .gsub(/:\[(.+?)\]/) do |match|
      code = Unicode::DATA[$1]
      if code
        code.chr(Encoding::UTF_8)
      else
        $stderr.puts("Unknown Unicode normative name: #{$1}")
        match
      end
    end
end

#lexeme_delimiter?(pos) ⇒ Boolean

Tells whether there is a lexeme delimiter at position pos in current line

Returns:

  • (Boolean) 


31
32
33
34
35
 
# File 'lib/layo/lexer.rb', line 31

def lexeme_delimiter?(pos)
  @line[pos] == '!' || @line[pos] == ',' ||
  @line[pos] == "\n" || space?(@line[pos]) ||
  @line[pos] == '…' || @line[pos, 3] == '...'
end

#next_item ⇒ Object

Reads and returns next lexeme



38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
 
# File 'lib/layo/lexer.rb', line 38

def next_item
  return @last_lexeme if @last_lexeme[0].nil?
  while true
    @line = next_line if buffer_empty?
    if @line.nil?
      lexeme = [nil, @line_no, 1]
      break
    end

    # Skip whitespaces
    while space?(@line[@pos])
      @pos += 1
    end

    # Skip triple dot characters (join lines)
    if @line[@pos, 4] == "...\n" || @line[@pos, 2] == "…\n"
      line_no, pos = @line_no, @pos + 1
      @line, @pos = next_line, 0
      if @line.nil? || @line.strip.empty?
        raise SyntaxError.new(line_no, pos, 'Line continuation may not be followed by an empty line')
      end
      next
    end

    # Skip one line comments
    if @line[@pos, 3] == 'BTW'
      @pos = @line.length - 1
    end
    # and multiline ones
    if @last_lexeme[0] == "\n" && @line[@pos, 4] == 'OBTW'
      tldr_found, line_no, pos = false, @line_no, @pos + 1
      while true
        @line = next_line
        break if @line.nil?
        m = @line.chomp.match(/(^|\s+)TLDR\s*(,|$)/)
        unless m.nil?
          tldr_found = true
          @pos = m.end(0)
          break
        end
      end
      unless tldr_found
        raise SyntaxError.new(line_no, pos, 'Unterminated multiline comment')
      end
      next
    end

    if @line[@pos] == "\n" || @line[@pos] == '!'
      # Handle newline and bang separately
      lexeme = [@line[@pos], @line_no, @pos + 1]
      @pos += 1
    elsif @line[@pos] == ','
      # Comma is a virtual newline
      lexeme = ["\n", @line_no, @pos + 1]
      @pos += 1
    elsif @line[@pos] == '"'
      # Strings begin with "
      # Need to handle empty strings separately
      if @line[@pos + 1] == '"'
        string = '""'
      else
        m = @line.match(/([^:](?:::)*)"/, @pos + 1)
        string = @line[@pos..m.end(0) - 1] unless m.nil?
      end
      # String must be followed by an allowed lexeme delimiter
      if string.nil? || !lexeme_delimiter?(@pos + string.length)
        raise SyntaxError.new(@line_no, @pos + 1, 'Unterminated string constant')
      end
      lexeme = [%Q["#{escape_string(string[1..-2])}"], @line_no, @pos + 1]
      @pos = @pos + string.length
    else
      # Grab as much characters as we can until meeting lexeme delimiter
      # Treat what we grabbed as a lexeme
      seq, pos = '', @pos + 1
      until lexeme_delimiter?(@pos)
        seq += @line[@pos]
        @pos += 1
      end
      lexeme = [seq, @line_no, pos]
    end

    break
  end
  @last_lexeme = lexeme
end

#next_line ⇒ Object

Reads and returns next line from input stream. Converts newline character to n returns nil upon reaching EOF



127
128
129
130
131
132
133
134
135
136
137
138
139
 
# File 'lib/layo/lexer.rb', line 127

def next_line
  return nil if @input.eof?
  line, ch, @pos, @line_no = '', '', 0, @line_no + 1
  until ch == "\r" || ch == "\n" || ch.nil?
    ch = @input.getc
    line += ch unless ch.nil?
  end
  if ch == "\r"
    ch = @input.getc
    @input.ungetc(ch) unless ch == "\n" || ch.nil?
  end
  line.chomp << "\n"
end

#reset ⇒ Object

Resets this lexer instance



21
22
23
24
 
# File 'lib/layo/lexer.rb', line 21

def reset
  @line_no, @last_lexeme = 0, ["\n"]
  super
end

#space?(char) ⇒ Boolean

Returns:

  • (Boolean) 


26
27
28
 
# File 'lib/layo/lexer.rb', line 26

def space?(char)
  char == ' ' || char == "\t"
end