Class: TwitterCldr::Tokenizers::DateTokenizer
- Inherits:
-
Object
- Object
- TwitterCldr::Tokenizers::DateTokenizer
- Defined in:
- lib/twitter_cldr/tokenizers/calendars/date_tokenizer.rb
Instance Attribute Summary collapse
-
#data_reader ⇒ Object
readonly
Returns the value of attribute data_reader.
Instance Method Summary collapse
-
#initialize(data_reader) ⇒ DateTokenizer
constructor
A new instance of DateTokenizer.
- #tokenize(pattern) ⇒ Object
- #tokenizer ⇒ Object
Constructor Details
#initialize(data_reader) ⇒ DateTokenizer
Returns a new instance of DateTokenizer.
12 13 14 |
# File 'lib/twitter_cldr/tokenizers/calendars/date_tokenizer.rb', line 12 def initialize(data_reader) @data_reader = data_reader end |
Instance Attribute Details
#data_reader ⇒ Object (readonly)
Returns the value of attribute data_reader.
10 11 12 |
# File 'lib/twitter_cldr/tokenizers/calendars/date_tokenizer.rb', line 10 def data_reader @data_reader end |
Instance Method Details
#tokenize(pattern) ⇒ Object
16 17 18 |
# File 'lib/twitter_cldr/tokenizers/calendars/date_tokenizer.rb', line 16 def tokenize(pattern) PatternTokenizer.new(data_reader, tokenizer).tokenize(pattern) end |
#tokenizer ⇒ Object
20 21 22 23 24 25 26 |
# File 'lib/twitter_cldr/tokenizers/calendars/date_tokenizer.rb', line 20 def tokenizer @tokenizer ||= Tokenizer.new([ TokenRecognizer.new(:composite, /^\#\{[^\}]+\}/, /^\#\{([^\}]+)\}/), TokenRecognizer.new(:pattern, /^(?:G{1,5}|y+|Y+|Q{1,4}|q{1,5}|M{1,5}|L{1,5}|d{1,2}|F{1}|E{1,5}|e{1,5}|c{1,5})/), TokenRecognizer.new(:plaintext, //) ], /(\s*\'[\w\s-]+\'\s*|G{1,5}|y+|Y+|Q{1,4}|q{1,5}|M{1,5}|L{1,5}|d{1,2}|F{1}|E{1,5}|e{1,5}|c{1,5}|\#\{[^\}]+\})/) end |