erltoken (erltoken v0.1.5)

Summary

Types

name/0

token/0

Functions

count(Name, Text)

count_nif(Name, Text)

decode(Name, Tokens)

decode_nif(Name, Tokens)

encode(Name, Text)

encode_nif(Name, Text)

encode_with_special_tokens(Name, Text)

encode_with_special_tokens_nif(Name, Text)

encoding_for_model(Model)

encoding_for_model_nif(Model)

estimate_cost_usd_micro(Model, InputTokens, OutputTokens)

estimate_cost_usd_micro_nif(Model, InputTokens, OutputTokens)

fits(Name, Text, MaxTokens)

list_encodings()

list_encodings_nif()

remaining(Name, Text, ContextWindow)

trim_to_token_limit(Name, Text, MaxTokens)

Types

name/0

-type name() :: binary().

token/0

-type token() :: non_neg_integer().

Functions

count(Name, Text)

-spec count(name(), binary()) -> {ok, non_neg_integer()} | {error, term()}.

count_nif(Name, Text)

decode(Name, Tokens)

-spec decode(name(), [token()]) -> {ok, binary()} | {error, term()}.

decode_nif(Name, Tokens)

encode(Name, Text)

-spec encode(name(), binary()) -> {ok, [token()]} | {error, term()}.

encode_nif(Name, Text)

encode_with_special_tokens(Name, Text)

-spec encode_with_special_tokens(name(), binary()) -> {ok, [token()]} | {error, term()}.

encode_with_special_tokens_nif(Name, Text)

encoding_for_model(Model)

-spec encoding_for_model(binary()) -> {ok, binary()} | {error, unknown_model | invalid_utf8}.

encoding_for_model_nif(Model)

estimate_cost_usd_micro(Model, InputTokens, OutputTokens)

-spec estimate_cost_usd_micro(binary(), non_neg_integer(), non_neg_integer()) ->
                                 {ok, non_neg_integer()} | {error, term()}.

estimate_cost_usd_micro_nif(Model, InputTokens, OutputTokens)

fits(Name, Text, MaxTokens)

-spec fits(name(), binary(), non_neg_integer()) -> {ok, boolean()} | {error, term()}.

list_encodings()

-spec list_encodings() -> [binary()].

list_encodings_nif()

remaining(Name, Text, ContextWindow)

-spec remaining(name(), binary(), non_neg_integer()) -> {ok, non_neg_integer()} | {error, term()}.

trim_to_token_limit(Name, Text, MaxTokens)

-spec trim_to_token_limit(name(), binary(), non_neg_integer()) -> {ok, binary()} | {error, term()}.