Documentation for tokenizers (0.5.3)

Alphabetic Index

File Listing

README

Namespace Listing A-Z

A

AddedToken (Tokenizers)

B

BPE (Tokenizers::Models)
BPEDecoder (Tokenizers::Decoders)
BertNormalizer (Tokenizers::Normalizers)
BpeTrainer (Tokenizers::Trainers)
ByteLevel (Tokenizers::Processors)
ByteLevel (Tokenizers::PreTokenizers)

C

CTC (Tokenizers::Decoders)
CharBPETokenizer (Tokenizers)

D

Decoders (Tokenizers)
Digits (Tokenizers::PreTokenizers)

E

Encoding (Tokenizers)
Error (Tokenizers)

F

FromPretrained (Tokenizers)

M

Metaspace (Tokenizers::Decoders)
Metaspace (Tokenizers::PreTokenizers)
Models (Tokenizers)

N

Normalizers (Tokenizers)

P

PreTokenizers (Tokenizers)
Prepend (Tokenizers::Normalizers)
Processors (Tokenizers)
Punctuation (Tokenizers::PreTokenizers)

R

RobertaProcessing (Tokenizers::Processors)

S

Split (Tokenizers::PreTokenizers)
Strip (Tokenizers::Normalizers)
Strip (Tokenizers::Decoders)

T

TemplateProcessing (Tokenizers::Processors)
Tokenizer (Tokenizers)
Tokenizers
Trainers (Tokenizers)

U

Unigram (Tokenizers::Models)
UnigramTrainer (Tokenizers::Trainers)

W

WordLevel (Tokenizers::Models)
WordLevelTrainer (Tokenizers::Trainers)
WordPiece (Tokenizers::Models)
WordPiece (Tokenizers::Decoders)
WordPieceTrainer (Tokenizers::Trainers)