Obscura.Tiktoken.Encoding (obscura v0.1.0)

Copy Markdown View Source

A tiktoken-compatible byte-pair encoding.

Summary

Types

special_selector()
t()

Functions

decode(encoding, tokens, opts \\ [])
decode!(encoding, tokens, opts \\ [])
decode_bytes(encoding, tokens)
decode_bytes!(encoding, tokens)
decode_single_token_bytes(encoding, token)
decode_tokens_bytes(encoding, tokens)
decode_with_offsets(encoding, tokens)
encode(encoding, text, opts \\ [])
encode!(encoding, text, opts \\ [])
encode_ordinary(encoding, text)
encode_ordinary!(encoding, text)
encode_single_piece(encoding, text_or_bytes)
encode_single_token(encoding, text_or_bytes)
n_vocab(encoding)
new(attrs)
special_token?(encoding, token)
token_byte_values(encoding)

Types

special_selector()

@type special_selector() :: :all | [String.t()] | MapSet.t(String.t())

t()

@type t() :: %Obscura.Tiktoken.Encoding{
  decoder: %{required(non_neg_integer()) => binary()},
  eot_token: non_neg_integer() | nil,
  explicit_n_vocab: non_neg_integer() | nil,
  max_token_value: non_neg_integer(),
  mergeable_ranks: %{required(binary()) => non_neg_integer()},
  name: String.t(),
  pat_str: String.t(),
  pattern: Regex.t(),
  sorted_token_bytes: [binary()],
  special_token_values: MapSet.t(non_neg_integer()),
  special_tokens: %{required(String.t()) => non_neg_integer()},
  special_tokens_decoder: %{required(non_neg_integer()) => binary()}
}

Functions

decode(encoding, tokens, opts \\ [])

@spec decode(t(), [non_neg_integer()], keyword()) ::
  {:ok, String.t()} | {:error, term()}

decode!(encoding, tokens, opts \\ [])

@spec decode!(t(), [non_neg_integer()], keyword()) :: String.t()

decode_bytes(encoding, tokens)

@spec decode_bytes(t(), [non_neg_integer()]) :: {:ok, binary()} | {:error, term()}

decode_bytes!(encoding, tokens)

@spec decode_bytes!(t(), [non_neg_integer()]) :: binary()

decode_single_token_bytes(encoding, token)

@spec decode_single_token_bytes(t(), non_neg_integer()) ::
  {:ok, binary()} | {:error, term()}

decode_tokens_bytes(encoding, tokens)

@spec decode_tokens_bytes(t(), [non_neg_integer()]) ::
  {:ok, [binary()]} | {:error, term()}

decode_with_offsets(encoding, tokens)

@spec decode_with_offsets(t(), [non_neg_integer()]) ::
  {:ok, {String.t(), [non_neg_integer()]}} | {:error, term()}

encode(encoding, text, opts \\ [])

@spec encode(t(), String.t(), keyword()) ::
  {:ok, [non_neg_integer()]} | {:error, term()}

encode!(encoding, text, opts \\ [])

@spec encode!(t(), String.t(), keyword()) :: [non_neg_integer()]

encode_ordinary(encoding, text)

@spec encode_ordinary(t(), String.t()) ::
  {:ok, [non_neg_integer()]} | {:error, term()}

encode_ordinary!(encoding, text)

@spec encode_ordinary!(t(), String.t()) :: [non_neg_integer()]

encode_single_piece(encoding, text_or_bytes)

@spec encode_single_piece(t(), String.t() | binary()) ::
  {:ok, [non_neg_integer()]} | {:error, term()}

encode_single_token(encoding, text_or_bytes)

@spec encode_single_token(t(), String.t() | binary()) ::
  {:ok, non_neg_integer()} | {:error, term()}

n_vocab(encoding)

@spec n_vocab(t()) :: non_neg_integer()

new(attrs)

@spec new(keyword()) :: {:ok, t()} | {:error, term()}

special_token?(encoding, token)

@spec special_token?(t(), non_neg_integer()) :: boolean()

token_byte_values(encoding)

@spec token_byte_values(t()) :: [binary()]