Class: Transformers::XlmRoberta::XLMRobertaTokenizerFast

Inherits:
PreTrainedTokenizerFast show all
Defined in:
lib/transformers/models/xlm_roberta/tokenization_xlm_roberta_fast.rb

Constant Summary collapse

VOCAB_FILES_NAMES = 
{vocab_file: "sentencepiece.bpe.model", tokenizer_file: "tokenizer.json"}

Constants included from SpecialTokensMixin

SpecialTokensMixin::SPECIAL_TOKENS_ATTRIBUTES

Instance Attribute Summary

Attributes inherited from PreTrainedTokenizerBase

#init_kwargs, #model_max_length

Instance Method Summary collapse

Methods inherited from PreTrainedTokenizerFast

#_convert_token_to_id_with_added_voc, #backend_tokenizer, #convert_ids_to_tokens, #convert_tokens_to_ids, #convert_tokens_to_string, #get_vocab, #is_fast, #vocab

Methods inherited from PreTrainedTokenizerBase

#_eventual_warn_about_too_long_sequence, _from_pretrained, #call, from_pretrained

Methods included from ClassAttribute

#class_attribute

Methods included from SpecialTokensMixin

#bos_token_id, #cls_token_id, #eos_token_id, #pad_token_id, #sep_token_id, #special_tokens_map, #unk_token_id

Constructor Details

#initialize(vocab_file: nil, tokenizer_file: nil, bos_token: "<s>", eos_token: "</s>", sep_token: "</s>", cls_token: "<s>", unk_token: "<unk>", pad_token: "<pad>", mask_token: "<mask>", **kwargs) ⇒ XLMRobertaTokenizerFast

self.slow_tokenizer_class = XLMRobertaTokenizer



24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/transformers/models/xlm_roberta/tokenization_xlm_roberta_fast.rb', line 24

def initialize(
  vocab_file: nil,
  tokenizer_file: nil,
  bos_token: "<s>",
  eos_token: "</s>",
  sep_token: "</s>",
  cls_token: "<s>",
  unk_token: "<unk>",
  pad_token: "<pad>",
  mask_token: "<mask>",
  **kwargs
)
  # Mask token behave like a normal word, i.e. include the space before it
  mask_token = mask_token.is_a?(String) ? Tokenizers::AddedToken.new(mask_token, lstrip: true, rstrip: false) : mask_token

  super(vocab_file, tokenizer_file: tokenizer_file, bos_token: bos_token, eos_token: eos_token, sep_token: sep_token, cls_token: cls_token, unk_token: unk_token, pad_token: pad_token, mask_token: mask_token, **kwargs)

  @vocab_file = vocab_file
end

Instance Method Details

#build_inputs_with_special_tokens(token_ids_0, token_ids_1: nil) ⇒ Object 



48
49
50
51
52
53
54
55
 
# File 'lib/transformers/models/xlm_roberta/tokenization_xlm_roberta_fast.rb', line 48

def build_inputs_with_special_tokens(token_ids_0, token_ids_1: nil)
  if token_ids_1.nil?
    return [@cls_token_id] + token_ids_0 + [@sep_token_id]
  end
  cls = [@cls_token_id]
  sep = [@sep_token_id]
  cls + token_ids_0 + sep + sep + token_ids_1 + sep
end

#can_save_slow_tokenizer ⇒ Object 



44
45
46
 
# File 'lib/transformers/models/xlm_roberta/tokenization_xlm_roberta_fast.rb', line 44

def can_save_slow_tokenizer
  @vocab_file ? File.exist?(@vocab_file) : false
end

#create_token_type_ids_from_sequences(token_ids_0, token_ids_1: nil) ⇒ Object 



57
58
59
60
61
62
63
64
65
 
# File 'lib/transformers/models/xlm_roberta/tokenization_xlm_roberta_fast.rb', line 57

def create_token_type_ids_from_sequences(token_ids_0, token_ids_1: nil)
  sep = [@sep_token_id]
  cls = [@cls_token_id]

  if token_ids_1.nil?
    return (cls + token_ids_0 + sep).length * [0]
  end
  (cls + token_ids_0 + sep + sep + token_ids_1 + sep).length * [0]
end