Class: Lox::Scanner

Inherits:
Object
  • Object
show all
Defined in:
lib/loxby/scanner.rb

Overview

‘Lox::Scanner` converts a string to a series of tokens using a giant `case` statement.

Constant Summary collapse

EXPRESSIONS =

Custom character classes for certain tokens.

Lox.config.scanner.expressions.to_h
KEYWORDS =

Map of keywords to token types. Right now, all keywords have their own token type.

Lox.config.scanner.keywords.map { [_1, _1.to_sym] }.to_h

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(source, interpreter) ⇒ Scanner

Returns a new instance of Scanner.



21
22
23
24
25
26
27
28
29
 
# File 'lib/loxby/scanner.rb', line 21

def initialize(source, interpreter)
  @source = source
  @tokens = []
  @interpreter = interpreter
  # Variables for scanning
  @start = 0
  @current = 0
  @line = 1
end

Instance Attribute Details

#line ⇒ Object

Returns the value of attribute line.



19
20
21
 
# File 'lib/loxby/scanner.rb', line 19

def line
  @line
end

Instance Method Details

#add_token(type, literal = nil) ⇒ Object

Emit a token.



155
156
157
158
 
# File 'lib/loxby/scanner.rb', line 155

def add_token(type, literal = nil)
  text = @source[@start...@current]
  @tokens << Lox::Token.new(type, text, literal, @line)
end

#advance_character ⇒ Object

Move the pointer ahead one character and return it.



148
149
150
151
152
 
# File 'lib/loxby/scanner.rb', line 148

def advance_character
  character = @source[@current]
  @current += 1
  character
end

#end_of_source? ⇒ Boolean

Returns:

  • (Boolean) 


168
 
# File 'lib/loxby/scanner.rb', line 168

def end_of_source? = @current >= @source.size

#match(expected) ⇒ Object

Move the pointer ahead if character matches expected character; error otherwise.



161
162
163
164
165
166
 
# File 'lib/loxby/scanner.rb', line 161

def match(expected)
  return false unless @source[@current] == expected || end_of_source?

  @current += 1
  true
end

#peek ⇒ Object

1-character lookahead



171
172
173
 
# File 'lib/loxby/scanner.rb', line 171

def peek
  end_of_source? ? "\0" : @source[@current]
end

#peek_next ⇒ Object

2-character lookahead



176
177
178
 
# File 'lib/loxby/scanner.rb', line 176

def peek_next
  (@current + 1) > @source.size ? "\0" : @source[@current + 1]
end

#scan_block_comment ⇒ Object

rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity



90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
 
# File 'lib/loxby/scanner.rb', line 90

def scan_block_comment # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
  advance_character until (peek == '*' && peek_next == '/') || (peek == '/' && peek_next == '*') || end_of_source?

  if end_of_source? || peek_next == "\0" # If 0 or 1 characters are left
    @interpreter.error(line, 'Unterminated block comment.')
    return
  elsif peek == '/' && peek_next == '*'
    # Nested block comment. Skip opening characters
    match '/'
    match '*'
    scan_block_comment # Skip nested comment
    advance_character until (peek == '*' && peek_next == '/') || (peek == '/' && peek_next == '*') || end_of_source?
  end

  # Skip closing characters
  match '*'
  match '/'
end

#scan_identifier ⇒ Object 



141
142
143
144
145
 
# File 'lib/loxby/scanner.rb', line 141

def scan_identifier
  advance_character while peek =~ Regexp.union(EXPRESSIONS[:identifier], /\d/)
  text = @source[@start...@current]
  add_token(KEYWORDS[text] || :identifier)
end

#scan_number ⇒ Object 



128
129
130
131
132
133
134
135
136
137
138
139
 
# File 'lib/loxby/scanner.rb', line 128

def scan_number
  advance_character while peek =~ EXPRESSIONS[:number_literal]

  # Check for decimal
  if peek == '.' && peek_next =~ EXPRESSIONS[:number_literal]
    # Consume decimal point
    advance_character
    advance_character while peek =~ EXPRESSIONS[:number_literal]
  end

  add_token :number, @source[@start...@current].to_f
end

#scan_string ⇒ Object

rubocop:disable Metrics/MethodLength



109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
 
# File 'lib/loxby/scanner.rb', line 109

def scan_string # rubocop:disable Metrics/MethodLength
  until peek == '"' || end_of_source?
    @line += 1 if peek == "\n" # Multiline strings are valid
    advance_character
  end

  if end_of_source?
    @interpreter.error(line, 'Unterminated string.')
    return
  end

  # Skip closing "
  advance_character

  # Trim quotes around literal
  value = @source[(@start + 1)...(@current - 1)]
  add_token :string, value
end

#scan_token ⇒ Object

Consume enough characters for the next token.



46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/loxby/scanner.rb', line 46

def scan_token # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
  character = advance_character

  case character
  # '/' is division, '//' is comment, '/* ... */'
  # is block comment. Needs special care.
  when '/'
    if match('/') # comment line
      advance_character until peek == "\n" || end_of_source?
    elsif match('*') # block comment
      scan_block_comment
    else
      add_token :slash
    end
  # Single-character tokens
  when Regexp.union(Lox::Token::SINGLE_TOKENS.keys)
    add_token Lox::Token::SINGLE_TOKENS[character]
  # 1-2 character tokens
  when '!'
    add_token match('=') ? :bang_equal : :bang
  when '='
    add_token match('=') ? :equal_equal : :equal
  when '<'
    add_token match('=') ? :less_equal : :less
  when '>'
    add_token match('=') ? :greater_equal : :greater
  # Whitespace
  when "\n"
    @line += 1
  when EXPRESSIONS[:whitespace]
  # Literals
  when '"'
    scan_string
  when EXPRESSIONS[:number_literal]
    scan_number
  # Keywords and identifiers
  when EXPRESSIONS[:identifier]
    scan_identifier
  else
    # Unknown character
    @interpreter.error(@line, 'Unexpected character.')
  end
end

#scan_tokens ⇒ Object

Process text source into a list of tokens.



33
34
35
36
37
38
39
40
41
42
 
# File 'lib/loxby/scanner.rb', line 33

def scan_tokens
  until end_of_source?
    # Beginnning of next lexeme
    @start = @current
    scan_token
  end

  # Implicitly return @tokens
  @tokens << Lox::Token.new(:eof, '', nil, @line)
end