Class: Spellr::LineTokenizer

Inherits:
StringScanner
  • Object
show all
Includes:
TokenRegexps
Defined in:
lib/spellr/line_tokenizer.rb

Constant Summary

Constants included from TokenRegexps

TokenRegexps::AFTER_KEY_SKIPS, TokenRegexps::ALPHA_SEP_RE, TokenRegexps::BACKSLASH_ESCAPE_RE, TokenRegexps::HEX_RE, TokenRegexps::KEY_DATA_URL, TokenRegexps::KEY_GTM_RE, TokenRegexps::KEY_HYPERWALLET_RE, TokenRegexps::KEY_PATTERNS_RE, TokenRegexps::KEY_SENDGRID_RE, TokenRegexps::KEY_SHA1, TokenRegexps::KEY_SHA512, TokenRegexps::LEFTOVER_NON_WORD_BITS_RE, TokenRegexps::LOWER_CASE_RE, TokenRegexps::NOT_EVEN_NON_WORDS_RE, TokenRegexps::NUM_SEP_RE, TokenRegexps::OTHER_CASE_RE, TokenRegexps::POSSIBLE_KEY_RE, TokenRegexps::REPEATED_SINGLE_LETTERS_RE, TokenRegexps::SEQUENTIAL_LETTERS_RE, TokenRegexps::SHELL_COLOR_ESCAPE_RE, TokenRegexps::SKIPS, TokenRegexps::SPELLR_DISABLE_RE, TokenRegexps::SPELLR_ENABLE_RE, TokenRegexps::SPELLR_LINE_DISABLE_RE, TokenRegexps::TERM_RE, TokenRegexps::THREE_CHUNK_RE, TokenRegexps::TITLE_CASE_RE, TokenRegexps::UPPER_CASE_RE, TokenRegexps::URL_ENCODED_ENTITIES_RE, TokenRegexps::URL_FRAGMENT, TokenRegexps::URL_HOSTNAME, TokenRegexps::URL_IP_ADDRESS, TokenRegexps::URL_PATH, TokenRegexps::URL_PORT, TokenRegexps::URL_QUERY, TokenRegexps::URL_QUERY_PART, TokenRegexps::URL_RE, TokenRegexps::URL_REST, TokenRegexps::URL_SCHEME, TokenRegexps::URL_USERINFO

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from TokenRegexps

#min_alpha_re

Constructor Details

#initialize(line, skip_key: false) ⇒ LineTokenizer

Returns a new instance of LineTokenizer.



18
19
20
21
22
23
24
 
# File 'lib/spellr/line_tokenizer.rb', line 18

def initialize(line, skip_key: false)
  @line = line
  @skip_key = skip_key
  @disabled = false

  super(@line.to_s)
end

Instance Attribute Details

#line ⇒ Object (readonly)

Returns the value of attribute line.



12
13
14
 
# File 'lib/spellr/line_tokenizer.rb', line 12

def line
  @line
end

#skip_key ⇒ Object (readonly) Also known as: skip_key?

Returns the value of attribute skip_key.



12
13
14
 
# File 'lib/spellr/line_tokenizer.rb', line 12

def skip_key
  @skip_key
end

Instance Method Details

#each_term ⇒ Object 



31
32
33
34
35
36
37
38
 
# File 'lib/spellr/line_tokenizer.rb', line 31

def each_term
  until eos?
    term = next_term
    next if !term || @disabled

    yield term
  end
end

#each_token(skip_term_proc: nil) ⇒ Object 



40
41
42
43
44
45
46
47
48
 
# File 'lib/spellr/line_tokenizer.rb', line 40

def each_token(skip_term_proc: nil)
  until eos?
    term = next_term
    next unless term
    next if @disabled || skip_term_proc&.call(term)

    yield Token.new(term, line: line, location: column_location(term))
  end
end

#string=(line) ⇒ Object 



26
27
28
29
 
# File 'lib/spellr/line_tokenizer.rb', line 26

def string=(line)
  @line = line
  super(@line.to_s)
end