Module: Transformers::SpecialTokensMixin

Included in:
PreTrainedTokenizerBase
Defined in:
lib/transformers/tokenization_utils_base.rb

Constant Summary collapse

SPECIAL_TOKENS_ATTRIBUTES = 
[
  :bos_token,
  :eos_token,
  :unk_token,
  :sep_token,
  :pad_token,
  :cls_token,
  :mask_token,
  :additional_special_tokens
]

Instance Method Summary collapse

Instance Method Details

#bos_token_id ⇒ Object 



143
144
145
146
147
148
 
# File 'lib/transformers/tokenization_utils_base.rb', line 143

def bos_token_id
  if @bos_token.nil?
    return nil
  end
  convert_tokens_to_ids(@bos_token)
end

#cls_token_id ⇒ Object 



178
179
180
181
182
183
 
# File 'lib/transformers/tokenization_utils_base.rb', line 178

def cls_token_id
  if @cls_token.nil?
    return nil
  end
  convert_tokens_to_ids(@cls_token)
end

#eos_token_id ⇒ Object 



150
151
152
153
154
155
 
# File 'lib/transformers/tokenization_utils_base.rb', line 150

def eos_token_id
  if @eos_token.nil?
    return nil
  end
  convert_tokens_to_ids(@eos_token)
end

#initialize(**kwargs) ⇒ Object 



137
138
139
140
141
 
# File 'lib/transformers/tokenization_utils_base.rb', line 137

def initialize(**kwargs)
  SPECIAL_TOKENS_ATTRIBUTES.each do |k|
    instance_variable_set("@#{k}", kwargs[k])
  end
end

#pad_token_id ⇒ Object 



171
172
173
174
175
176
 
# File 'lib/transformers/tokenization_utils_base.rb', line 171

def pad_token_id
  if @pad_token.nil?
    return nil
  end
  convert_tokens_to_ids(@pad_token)
end

#sep_token_id ⇒ Object 



164
165
166
167
168
169
 
# File 'lib/transformers/tokenization_utils_base.rb', line 164

def sep_token_id
  if @sep_token.nil?
    return nil
  end
  convert_tokens_to_ids(@sep_token)
end

#special_tokens_map ⇒ Object 



185
186
187
188
189
190
191
192
193
194
 
# File 'lib/transformers/tokenization_utils_base.rb', line 185

def special_tokens_map
  set_attr = {}
  SPECIAL_TOKENS_ATTRIBUTES.each do |attr|
    attr_value = send(attr)
    if attr_value
      set_attr[attr] = attr_value
    end
  end
  set_attr
end

#unk_token_id ⇒ Object 



157
158
159
160
161
162
 
# File 'lib/transformers/tokenization_utils_base.rb', line 157

def unk_token_id
  if @unk_token.nil?
    return nil
  end
  convert_tokens_to_ids(@unk_token)
end