Class: Brice::Colours::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/brice/colours.rb

Overview

Tokenize an inspection string.

Constant Summary collapse

EVENT_MAP =
{
# on_CHAR:            :unknown,
# on___end__:         :unknown,
# on_backref:         :unknown,
# on_backtick:        :unknown,
  on_comma:           :comma,
  on_comment:         :unknown,
  on_const:           :class,
# on_cvar:            :unknown,
# on_embdoc:          :unknown,
# on_embdoc_beg:      :unknown,
# on_embdoc_end:      :unknown,
# on_embexpr_beg:     :unknown,
# on_embexpr_end:     :unknown,
# on_embvar:          :unknown,
  on_float:           :number,
# on_gvar:            :unknown,
# on_heredoc_beg:     :unknown,
# on_heredoc_end:     :unknown,
  on_ident:           :symbol,
# on_ignored_nl:      :unknown,
  on_imaginary:       :number,
  on_int:             :number,
# on_ivar:            :unknown,
  on_kw:              :keyword,
  on_label:           :unknown,
  on_lbrace:          :open_hash,
  on_lbracket:        :open_array,
  on_lparen:          :unknown,
# on_nl:              :unknown,
  on_op:              :refers,
  on_period:          :comma,
# on_qsymbols_beg:    :unknown,
# on_qwords_beg:      :unknown,
  on_rational:        :number,
  on_rbrace:          :close_hash,
  on_rbracket:        :close_array,
  on_regexp_beg:      :unknown,
  on_regexp_end:      :unknown,
  on_rparen:          :unknown,
  on_semicolon:       :comma,
  on_sp:              :whitespace,
  on_symbeg:          :symbol_prefix,
# on_symbols_beg:     :unknown,
# on_tlambda:         :unknown,
# on_tlambeg:         :unknown,
  on_tstring_beg:     :open_string,
  on_tstring_content: :string,
  on_tstring_end:     :close_string,
# on_words_beg:       :unknown,
# on_words_sep:       :unknown
}
OBJECT_RE =
%r{
  \A
  ( \#< )
  ( .+ )
  ( > )
  \z
}x
OBJECT_CLASS_RE =
%r{
  \A
  (?: \w | :: )+
}x
OBJECT_ADDR_RE =
%r{
  \A
  ( : )
  ( 0x [\hx]+ )
  (?= \s | \z )
}x
IVAR_RE =
%r{
  \A
  ( @ )
  ( .+ )
  \z
}x
RANGE_RE =
%r{
  \A
  \.+
  \z
}x

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(&block) ⇒ Tokenizer

Returns a new instance of Tokenizer.


330
331
332
# File 'lib/brice/colours.rb', line 330

def initialize(&block)
  @block = block or raise ArgumentError, 'no block given'
end

Instance Attribute Details

#blockObject (readonly)

Returns the value of attribute block


334
335
336
# File 'lib/brice/colours.rb', line 334

def block
  @block
end

Class Method Details

.tokenize(str, &block) ⇒ Object


326
327
328
# File 'lib/brice/colours.rb', line 326

def self.tokenize(str, &block)
  new(&block).tokenize(str)
end

Instance Method Details

#tokenize(str) ⇒ Object


336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
# File 'lib/brice/colours.rb', line 336

def tokenize(str)
  return if str.empty?
  return if enc_event(str)

  lex, prev = Ripper.lex(str), nil

  len = lex[-1][0][-1] + lex[-1][-1].bytesize
  str, rest = str.byteslice(0, len), str.byteslice(len .. -1)

  return block[:unknown, rest] if str.empty?

  lex.each { |_, event, tok|
    sym_event(event, tok, prev) ||
    obj_event(event, tok)       ||
    rng_event(event, tok)       ||
    var_event(event, tok)       ||
    map_event(event, tok)

    prev = event
  }

  tokenize(rest)
end