Module: RDoc::Parser::RubyTools

Includes:
RubyToken
Included in:
Ruby
Defined in:
lib/rdoc/parser/ruby_tools.rb

Overview

frozen_string_literal: false

Collection of methods for writing parsers against RDoc::RubyLex and RDoc::RubyToken

Constant Summary

Constants included from RubyToken

RubyToken::EXPR_ARG, RubyToken::EXPR_BEG, RubyToken::EXPR_CLASS, RubyToken::EXPR_DOT, RubyToken::EXPR_END, RubyToken::EXPR_FNAME, RubyToken::EXPR_MID, RubyToken::NEWLINE_TOKEN, RubyToken::Symbol, RubyToken::TkReading2Token, RubyToken::TkSymbol2Token, RubyToken::TkToken2Reading, RubyToken::TokenDefinitions

Instance Method Summary collapse

Methods included from RubyToken

#Token, def_token, #set_token_position

Instance Method Details

#add_token_listener(obj) ⇒ Object

Adds a token listener obj, but you should probably use token_listener



13
14
15
16
 
# File 'lib/rdoc/parser/ruby_tools.rb', line 13

def add_token_listener(obj)
  @token_listeners ||= []
  @token_listeners << obj
end

#get_tk ⇒ Object

Fetches the next token from the scanner



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
 
# File 'lib/rdoc/parser/ruby_tools.rb', line 21

def get_tk
  tk = nil

  if @tokens.empty? then
    tk = @scanner.token
    @read.push @scanner.get_readed
    puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG
  else
    @read.push @unget_read.shift
    tk = @tokens.shift
    puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG
  end

  tk = nil if TkEND_OF_SCRIPT === tk

  if TkSYMBEG === tk then
    set_token_position tk.line_no, tk.char_no

    case tk1 = get_tk
    when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then
      if tk1.respond_to?(:name) then
        tk = Token(TkSYMBOL).set_text(":" + tk1.name)
      else
        tk = Token(TkSYMBOL).set_text(":" + tk1.text)
      end

      # remove the identifier we just read to replace it with a symbol
      @token_listeners.each do |obj|
        obj.pop_token
      end if @token_listeners
    else
      tk = tk1
    end
  end

  # inform any listeners of our shiny new token
  @token_listeners.each do |obj|
    obj.add_token(tk)
  end if @token_listeners

  tk
end

#get_tk_until(*tokens) ⇒ Object

Reads and returns all tokens up to one of tokens. Leaves the matched token in the token list.



68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
 
# File 'lib/rdoc/parser/ruby_tools.rb', line 68

def get_tk_until(*tokens)
  read = []

  loop do
    tk = get_tk

    case tk
    when *tokens then
      unget_tk tk
      break
    end

    read << tk
  end

  read
end

#get_tkread ⇒ Object

Retrieves a String representation of the read tokens



89
90
91
92
93
 
# File 'lib/rdoc/parser/ruby_tools.rb', line 89

def get_tkread
  read = @read.join("")
  @read = []
  read
end

#peek_read ⇒ Object

Peek equivalent for get_tkread



98
99
100
 
# File 'lib/rdoc/parser/ruby_tools.rb', line 98

def peek_read
  @read.join('')
end

#peek_tk ⇒ Object

Peek at the next token, but don’t remove it from the stream



105
106
107
108
 
# File 'lib/rdoc/parser/ruby_tools.rb', line 105

def peek_tk
  unget_tk(tk = get_tk)
  tk
end

#remove_token_listener(obj) ⇒ Object

Removes the token listener obj



113
114
115
 
# File 'lib/rdoc/parser/ruby_tools.rb', line 113

def remove_token_listener(obj)
  @token_listeners.delete(obj)
end

#reset ⇒ Object

Resets the tools



120
121
122
123
124
125
 
# File 'lib/rdoc/parser/ruby_tools.rb', line 120

def reset
  @read       = []
  @tokens     = []
  @unget_read = []
  @nest = 0
end

#skip_tkspace(skip_nl = true) ⇒ Object

Skips whitespace tokens including newlines if skip_nl is true



130
131
132
133
134
135
136
137
138
139
 
# File 'lib/rdoc/parser/ruby_tools.rb', line 130

def skip_tkspace(skip_nl = true) # HACK dup
  tokens = []

  while TkSPACE === (tk = get_tk) or (skip_nl and TkNL === tk) do
    tokens.push tk
  end

  unget_tk tk
  tokens
end

#token_listener(obj) ⇒ Object

Has obj listen to tokens



144
145
146
147
148
149
 
# File 'lib/rdoc/parser/ruby_tools.rb', line 144

def token_listener(obj)
  add_token_listener obj
  yield
ensure
  remove_token_listener obj
end

#unget_tk(tk) ⇒ Object

Returns tk to the scanner



154
155
156
157
158
159
160
161
162
163
164
 
# File 'lib/rdoc/parser/ruby_tools.rb', line 154

def unget_tk(tk)
  @tokens.unshift tk
  @unget_read.unshift @read.pop

  # Remove this token from any listeners
  @token_listeners.each do |obj|
    obj.pop_token
  end if @token_listeners

  nil
end