View Source Tokenizers.Native (Tokenizers v0.1.2)

Link to this section Summary

Link to this section Functions

Link to this function

decode(tokenizer, ids, skip_special_tokens)

View Source
Link to this function

decode_batch(tokenizer, ids, skip_special_tokens)

View Source
Link to this function

encode(tokenizer, input, add_special_tokens)

View Source
Link to this function

encode_batch(tokenizer, input, add_special_tokens)

View Source
Link to this function

from_pretrained(identifier)

View Source
Link to this function

get_attention_mask(encoding)

View Source
Link to this function

get_model_details(model)

View Source
Link to this function

get_special_tokens_mask(encoding)

View Source
Link to this function

get_vocab(tokenizer, with_added_tokens)

View Source
Link to this function

get_vocab_size(tokenizer, with_added_tokens)

View Source
Link to this function

id_to_token(tokenizer, id)

View Source
Link to this function

pad(encoding, target_length, pad_id, pad_type_id, pad_token, direction)

View Source
Link to this function

save(tokenizer, path, pretty)

View Source
Link to this function

token_to_id(tokenizer, token)

View Source
Link to this function

truncate(encoding, max_len, stride, direction)

View Source