Module: Transformers::ConvertSlowTokenizer

Defined in:
lib/transformers/convert_slow_tokenizer.rb

Defined Under Namespace

Classes: BertConverter, Converter

Constant Summary collapse

SLOW_TO_FAST_CONVERTERS = 
{
  "BertTokenizer" => BertConverter,
  "DistilBertTokenizer" => BertConverter
}

Class Method Summary collapse

Class Method Details

.convert_slow_tokenizer(transformer_tokenizer) ⇒ Object 



75
76
77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/transformers/convert_slow_tokenizer.rb', line 75

def self.convert_slow_tokenizer(transformer_tokenizer)
  tokenizer_class_name = transformer_tokenizer.class.name.split("::").last

  if !SLOW_TO_FAST_CONVERTERS.include?(tokenizer_class_name)
    raise ArgumentError,
      "An instance of tokenizer class #{tokenizer_class_name} cannot be converted in a Fast tokenizer instance." +
      " No converter was found. Currently available slow->fast convertors:" +
      " #{SLOW_TO_FAST_CONVERTERS.keys}"
  end

  converter_class = SLOW_TO_FAST_CONVERTERS.fetch(tokenizer_class_name)

  converter_class.new(transformer_tokenizer).converted
end