Module: RDoc::Parser::RubyTools

Included in:
Ruby
Defined in:
lib/rdoc/parser/ruby_tools.rb

Overview

Collection of methods for writing parsers

Instance Method Summary collapse

Instance Method Details

#add_token_listener(obj) ⇒ Object

Adds a token listener obj, but you should probably use token_listener



10
11
12
13
 
# File 'lib/rdoc/parser/ruby_tools.rb', line 10

def add_token_listener(obj)
  @token_listeners ||= []
  @token_listeners << obj
end

#get_tk ⇒ Object

Fetches the next token from the scanner



18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/rdoc/parser/ruby_tools.rb', line 18

def get_tk
  tk = nil

  if @tokens.empty? then
    if @scanner_point >= @scanner.size
      return
    else
      tk = @scanner[@scanner_point]
      @scanner_point += 1
      @read.push tk[:text]
    end
  else
    @read.push @unget_read.shift
    tk = @tokens.shift
  end

  if tk.nil? || :on___end__ == tk[:kind]
    return
  end

  # inform any listeners of our shiny new token
  @token_listeners&.each do |obj|
    obj.add_token(tk)
  end

  tk
end

#get_tk_until(*tokens) ⇒ Object

Reads and returns all tokens up to one of tokens. Leaves the matched token in the token list.



50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/rdoc/parser/ruby_tools.rb', line 50

def get_tk_until(*tokens)
  read = []

  loop do
    tk = get_tk

    case tk
    when *tokens then
      unget_tk tk
      break
    end

    read << tk
  end

  read
end

#get_tkread ⇒ Object

Retrieves a String representation of the read tokens



71
72
73
74
75
 
# File 'lib/rdoc/parser/ruby_tools.rb', line 71

def get_tkread
  read = @read.join("")
  @read = []
  read
end

#peek_read ⇒ Object

Peek equivalent for get_tkread



80
81
82
 
# File 'lib/rdoc/parser/ruby_tools.rb', line 80

def peek_read
  @read.join('')
end

#peek_tk ⇒ Object

Peek at the next token, but don’t remove it from the stream



87
88
89
90
 
# File 'lib/rdoc/parser/ruby_tools.rb', line 87

def peek_tk
  unget_tk(tk = get_tk)
  tk
end

#remove_token_listener(obj) ⇒ Object

Removes the token listener obj



95
96
97
 
# File 'lib/rdoc/parser/ruby_tools.rb', line 95

def remove_token_listener(obj)
  @token_listeners.delete(obj)
end

#reset ⇒ Object

Resets the tools



102
103
104
105
106
107
108
 
# File 'lib/rdoc/parser/ruby_tools.rb', line 102

def reset
  @read       = []
  @tokens     = []
  @unget_read = []
  @nest = 0
  @scanner_point = 0
end

#skip_tkspace ⇒ Object

Skips whitespace tokens including newlines



113
114
115
116
117
118
119
120
121
122
 
# File 'lib/rdoc/parser/ruby_tools.rb', line 113

def skip_tkspace
  tokens = []

  while (tk = get_tk) and (:on_sp == tk[:kind] or :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind]) do
    tokens.push(tk)
  end

  unget_tk(tk)
  tokens
end

#skip_tkspace_without_nl ⇒ Object

Skips whitespace tokens excluding newlines



127
128
129
130
131
132
133
134
135
136
 
# File 'lib/rdoc/parser/ruby_tools.rb', line 127

def skip_tkspace_without_nl
  tokens = []

  while (tk = get_tk) and :on_sp == tk[:kind] do
    tokens.push(tk)
  end

  unget_tk(tk)
  tokens
end

#token_listener(obj) ⇒ Object

Has obj listen to tokens



141
142
143
144
145
146
 
# File 'lib/rdoc/parser/ruby_tools.rb', line 141

def token_listener(obj)
  add_token_listener obj
  yield
ensure
  remove_token_listener obj
end

#unget_tk(tk) ⇒ Object

Returns tk to the scanner



151
152
153
154
155
156
157
158
159
160
161
 
# File 'lib/rdoc/parser/ruby_tools.rb', line 151

def unget_tk(tk)
  @tokens.unshift tk
  @unget_read.unshift @read.pop

  # Remove this token from any listeners
  @token_listeners.each do |obj|
    obj.pop_token
  end if @token_listeners

  nil
end