Class: TinyClassifier::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/tiny-classifier/tokenizer.rb

Constant Summary collapse

TOKENIZERS = 
[:none, :mecab]

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(params = nil) ⇒ Tokenizer

Returns a new instance of Tokenizer.



22
23
24
25
26
27
 
# File 'lib/tiny-classifier/tokenizer.rb', line 22

def initialize(params = nil)
  if params
    @type = params[:type]
  end
  @type ||= :none
end

Instance Attribute Details

#type ⇒ Object

Returns the value of attribute type.



20
21
22
 
# File 'lib/tiny-classifier/tokenizer.rb', line 20

def type
  @type
end

Instance Method Details

#tokenize(input) ⇒ Object 



29
30
31
32
33
34
35
36
 
# File 'lib/tiny-classifier/tokenizer.rb', line 29

def tokenize(input)
  case @type.to_s.downcase.to_sym
  when :mecab
    tokenize_by_mecab(input)
  else
    input
  end
end