Class: RDoc::Parser::RipperStateLex

Inherits:
Object
  • Object
show all
Defined in:
lib/rdoc/parser/ripper_state_lex.rb

Overview

Wrapper for Ripper lex states

Defined Under Namespace

Classes: InnerStateLex, Token

Constant Summary collapse

RIPPER_HAS_LEX_STATE =

TODO: Remove this constants after Ruby 2.4 EOL

Ripper::Filter.method_defined?(:state)
EXPR_NONE = 
0
EXPR_BEG = 
1
EXPR_END = 
2
EXPR_ENDARG = 
4
EXPR_ENDFN = 
8
EXPR_ARG = 
16
EXPR_CMDARG = 
32
EXPR_MID = 
64
EXPR_FNAME = 
128
EXPR_DOT = 
256
EXPR_CLASS = 
512
EXPR_LABEL = 
1024
EXPR_LABELED = 
2048
EXPR_FITEM = 
4096
EXPR_VALUE = 
EXPR_BEG
EXPR_BEG_ANY = 
(EXPR_BEG | EXPR_MID | EXPR_CLASS)
EXPR_ARG_ANY = 
(EXPR_ARG | EXPR_CMDARG)
EXPR_END_ANY = 
(EXPR_END | EXPR_ENDARG | EXPR_ENDFN)

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(code) ⇒ RipperStateLex

New lexer for code.



576
577
578
579
580
581
 
# File 'lib/rdoc/parser/ripper_state_lex.rb', line 576

def initialize(code)
  @buf = []
  @heredoc_queue = []
  @inner_lex = InnerStateLex.new(code)
  @tokens = @inner_lex.parse([])
end

Class Method Details

.end?(token) ⇒ Boolean

Returns true if lex state will be END after token.

Returns:

  • (Boolean) 


597
598
599
 
# File 'lib/rdoc/parser/ripper_state_lex.rb', line 597

def self.end?(token)
  (token[:state] & EXPR_END)
end

.parse(code) ⇒ Object

Returns tokens parsed from code.



584
585
586
587
588
589
590
591
592
593
594
 
# File 'lib/rdoc/parser/ripper_state_lex.rb', line 584

def self.parse(code)
  lex = self.new(code)
  tokens = []
  begin
    while tk = lex.get_squashed_tk
      tokens.push tk
    end
  rescue StopIteration
  end
  tokens
end

Instance Method Details

#get_squashed_tk ⇒ Object 



322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
 
# File 'lib/rdoc/parser/ripper_state_lex.rb', line 322

def get_squashed_tk
  if @buf.empty?
    tk = @tokens.shift
  else
    tk = @buf.shift
  end
  return nil if tk.nil?
  case tk[:kind]
  when :on_symbeg then
    tk = get_symbol_tk(tk)
  when :on_tstring_beg then
    tk = get_string_tk(tk)
  when :on_backtick then
    if (tk[:state] & (EXPR_FNAME | EXPR_ENDFN)) != 0
      @inner_lex.lex_state = EXPR_ARG unless RIPPER_HAS_LEX_STATE
      tk[:kind] = :on_ident
      tk[:state] = Ripper::Lexer.const_defined?(:State) ? Ripper::Lexer::State.new(EXPR_ARG) : EXPR_ARG
    else
      tk = get_string_tk(tk)
    end
  when :on_regexp_beg then
    tk = get_regexp_tk(tk)
  when :on_embdoc_beg then
    tk = get_embdoc_tk(tk)
  when :on_heredoc_beg then
    @heredoc_queue << retrieve_heredoc_info(tk)
    @inner_lex.lex_state = EXPR_END unless RIPPER_HAS_LEX_STATE
  when :on_nl, :on_ignored_nl, :on_comment, :on_heredoc_end then
    if !@heredoc_queue.empty?
      get_heredoc_tk(*@heredoc_queue.shift)
    elsif tk[:text].nil? # :on_ignored_nl sometimes gives nil
      tk[:text] = ''
    end
  when :on_words_beg then
    tk = get_words_tk(tk)
  when :on_qwords_beg then
    tk = get_words_tk(tk)
  when :on_symbols_beg then
    tk = get_words_tk(tk)
  when :on_qsymbols_beg then
    tk = get_words_tk(tk)
  when :on_op then
    if '&.' == tk[:text]
      tk[:kind] = :on_period
    else
      tk = get_op_tk(tk)
    end
  end
  tk
end