Class: Tml::Tokenizers::Data

Inherits:
Object
  • Object
show all
Defined in:
lib/tml/tokenizers/data.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(text, context = {}, opts = {}) ⇒ Data

Returns a new instance of Data.



64
65
66
67
68
69
70
 
# File 'lib/tml/tokenizers/data.rb', line 64

def initialize(text, context={}, opts={})
  self.text = text
  self.context = context
  self.opts = opts
  self.tokens = []
  tokenize
end

Instance Attribute Details

#context ⇒ Object

Returns the value of attribute context.



54
55
56
 
# File 'lib/tml/tokenizers/data.rb', line 54

def context
  @context
end

#opts ⇒ Object

Returns the value of attribute opts.



54
55
56
 
# File 'lib/tml/tokenizers/data.rb', line 54

def opts
  @opts
end

#text ⇒ Object

Returns the value of attribute text.



54
55
56
 
# File 'lib/tml/tokenizers/data.rb', line 54

def text
  @text
end

#tokens ⇒ Object

Returns the value of attribute tokens.



54
55
56
 
# File 'lib/tml/tokenizers/data.rb', line 54

def tokens
  @tokens
end

Class Method Details

.required?(label) ⇒ Boolean

Returns:

  • (Boolean) 


60
61
62
 
# File 'lib/tml/tokenizers/data.rb', line 60

def self.required?(label)
  label.index('{')
end

.supported_tokens ⇒ Object 



56
57
58
 
# File 'lib/tml/tokenizers/data.rb', line 56

def self.supported_tokens
  [Tml::Tokens::Data, Tml::Tokens::Method, Tml::Tokens::Transform, Tml::Tokens::Map]
end

Instance Method Details

#substitute(language, options = {}) ⇒ Object 



85
86
87
88
89
90
91
92
93
94
 
# File 'lib/tml/tokenizers/data.rb', line 85

def substitute(language, options = {})
  label = self.text

  tokens.each do |token|
    next unless token_allowed?(token)
    label = token.substitute(label, context, language, options)
  end

  label
end

#token_allowed?(token) ⇒ Boolean

Returns:

  • (Boolean) 


80
81
82
83
 
# File 'lib/tml/tokenizers/data.rb', line 80

def token_allowed?(token)
  return true unless opts[:allowed_tokens]
  not opts[:allowed_tokens][token.name].nil?
end

#tokenize ⇒ Object 



72
73
74
75
76
77
78
 
# File 'lib/tml/tokenizers/data.rb', line 72

def tokenize
  self.tokens = []
  self.class.supported_tokens.each do |klass|
    self.tokens << klass.parse(self.text)
  end
  self.tokens.flatten!.uniq!
end