Class: Tokenizers::Encoding

Inherits:
Object
  • Object
show all
Defined in:
lib/tokenizers/encoding.rb

Instance Method Summary collapse

Instance Method Details

#char_to_token(char_pos, sequence_index = 0) ⇒ Object 



11
12
13
 
# File 'lib/tokenizers/encoding.rb', line 11

def char_to_token(char_pos, sequence_index = 0)
  _char_to_token(char_pos, sequence_index)
end

#char_to_word(char_pos, sequence_index = 0) ⇒ Object 



15
16
17
 
# File 'lib/tokenizers/encoding.rb', line 15

def char_to_word(char_pos, sequence_index = 0)
  _char_to_word(word_index, sequence_index)
end

#word_to_chars(word_index, sequence_index = 0) ⇒ Object 



7
8
9
 
# File 'lib/tokenizers/encoding.rb', line 7

def word_to_chars(word_index, sequence_index = 0)
  _word_to_chars(word_index, sequence_index)
end

#word_to_tokens(word_index, sequence_index = 0) ⇒ Object 



3
4
5
 
# File 'lib/tokenizers/encoding.rb', line 3

def word_to_tokens(word_index, sequence_index = 0)
  _word_to_tokens(word_index, sequence_index)
end