erllama_model_backend behaviour (erllama v0.11.0)

View Source

Behaviour describing the operations the erllama_model gen_statem needs from a backing inference engine.

Two backends ship in v0.2:

erllama_model_stub — deterministic phash2-based stubs; used

  by tests that don't have a GGUF on disk.

erllama_model_llama — real llama.cpp via the NIF.

Future backends (mock for fault injection, remote for distributed inference, etc.) can plug in via this same surface.

Summary

Types

chat_message()
chat_request()
chat_tool()
sampler_opts()
sampler_ref()
seq_id()
state()
step_op()
step_result()

Callbacks

abort_handle/1
apply_adapters/2
apply_chat_template/2
clear_sampler/1
configure_sampler/2
decode_one/2
detokenize/2
detokenize/3
embed/2
extra_metadata/1
init(Config)
kv_pack/2
kv_pack/3
kv_unpack/2
kv_unpack/3
load_adapter/2
prefill/2
reset_context/1
sampler_free/1
sampler_new/2
seq_clear/1
seq_cp/3
seq_rm/2
seq_rm_last/2
seq_rm_last/3
set_grammar/2
step/2
terminate/1
thinking_signature/3
tokenize/2
unload_adapter/2
verify/4
vocab_info/1

Types

chat_message()

-type chat_message() :: #{role := binary(), content := binary() | [map()]}.

chat_request()

-type chat_request() ::
          #{messages := [chat_message()],
            system => binary() | undefined,
            tools => [chat_tool()] | undefined}.

chat_tool()

-type chat_tool() :: #{name := binary(), description => binary(), schema => map()}.

sampler_opts()

-type sampler_opts() ::
          #{grammar => binary(),
            grammar_lazy => boolean(),
            trigger_patterns => [binary()],
            trigger_tokens => [erllama:token_id()],
            grammar_prefill => binary(),
            repetition_penalty => float(),
            frequency_penalty => float(),
            presence_penalty => float(),
            penalty_last_n => integer(),
            top_k => non_neg_integer(),
            top_p => float(),
            min_p => float(),
            typical_p => float(),
            top_n_sigma => float(),
            xtc_probability => float(),
            xtc_threshold => float(),
            dynatemp_range => float(),
            dynatemp_exponent => float(),
            min_keep => pos_integer(),
            dry_multiplier => float(),
            dry_base => float(),
            dry_allowed_length => integer(),
            dry_penalty_last_n => integer(),
            dry_sequence_breakers => [binary()],
            mirostat => 0 | 1 | 2,
            mirostat_tau => float(),
            mirostat_eta => float(),
            logit_bias => [{erllama:token_id(), number()}],
            ignore_eos => boolean(),
            infill => boolean(),
            logprobs => non_neg_integer(),
            temperature => float(),
            seed => non_neg_integer()}.

sampler_ref()

-type sampler_ref() :: term().

seq_id()

-type seq_id() :: non_neg_integer().

state()

-type state() :: term().

step_op()

-type step_op() :: {prefill, [erllama:token_id()]} | {decode, sampler_ref()}.

step_result()

-type step_result() ::
          prefilled |
          {token, erllama:token_id(), 0 | 1} |
          {token, erllama:token_id(), 0 | 1, {float(), [{erllama:token_id(), float()}]}} |
          {thinking_token, erllama:token_id()} |
          thinking_end.

Callbacks

abort_handle/1

(optional) 
-callback abort_handle(state()) -> {ok, term()} | undefined.

apply_adapters/2

(optional) 
-callback apply_adapters(state(), [{term(), float()}]) -> {ok, state()} | {error, term()}.

apply_chat_template/2

(optional) 
-callback apply_chat_template(state(), Request :: chat_request()) ->
                                 {ok, [erllama:token_id()]} | {error, term()}.

clear_sampler/1

(optional) 
-callback clear_sampler(state()) -> {ok, state()} | {error, term()}.

configure_sampler/2

(optional) 
-callback configure_sampler(state(), sampler_opts()) -> {ok, state()} | {error, term()}.

decode_one/2

-callback decode_one(state(), ContextTokens :: [erllama:token_id()]) ->
                        {ok, erllama:token_id()} | {eog, erllama:token_id()} | {error, term()}.

detokenize/2

-callback detokenize(state(), [erllama:token_id()]) -> binary() | {error, term()}.

detokenize/3

(optional) 
-callback detokenize(state(), [erllama:token_id()], map()) -> binary() | {error, term()}.

embed/2

(optional) 
-callback embed(state(), [erllama:token_id()]) -> {ok, [float()]} | {error, term()}.

extra_metadata/1

(optional) 
-callback extra_metadata(state()) ->
                            #{model_size_bytes => non_neg_integer(),
                              total_layers => non_neg_integer(),
                              n_gpu_layers => integer(),
                              family => map()}.

init(Config)

-callback init(Config :: map()) -> {ok, state()} | {error, term()}.

kv_pack/2

-callback kv_pack(state(), Tokens :: [erllama:token_id()]) -> binary() | {error, term()}.

kv_pack/3

(optional) 
-callback kv_pack(state(), Tokens :: [erllama:token_id()], seq_id()) -> binary() | {error, term()}.

kv_unpack/2

-callback kv_unpack(state(), Bin :: binary()) -> ok | {error, term()}.

kv_unpack/3

(optional) 
-callback kv_unpack(state(), Bin :: binary(), seq_id()) -> ok | {error, term()}.

load_adapter/2

(optional) 
-callback load_adapter(state(), Path :: iodata()) -> {ok, term(), state()} | {error, term()}.

prefill/2

-callback prefill(state(), [erllama:token_id()]) -> ok | {error, term()}.

reset_context/1

(optional) 
-callback reset_context(state()) -> {ok, state()} | {error, term()}.

sampler_free/1

(optional) 
-callback sampler_free(sampler_ref()) -> ok | {error, term()}.

sampler_new/2

(optional) 
-callback sampler_new(state(), sampler_opts()) -> {ok, sampler_ref()} | {error, term()}.

seq_clear/1

(optional) 
-callback seq_clear(state()) -> ok | {error, term()}.

seq_cp/3

(optional) 
-callback seq_cp(state(), SrcSeq :: seq_id(), DstSeq :: seq_id()) -> ok | {error, term()}.

seq_rm/2

(optional) 
-callback seq_rm(state(), seq_id()) -> ok | {error, term()}.

seq_rm_last/2

(optional) 
-callback seq_rm_last(state(), NTokens :: pos_integer()) -> ok | {error, term()}.

seq_rm_last/3

(optional) 
-callback seq_rm_last(state(), seq_id(), NTokens :: pos_integer()) -> ok | {error, term()}.

set_grammar/2

(optional) 
-callback set_grammar(state(), Grammar :: binary() | undefined) -> {ok, state()} | {error, term()}.

step/2

(optional) 
-callback step(state(), [{seq_id(), step_op()}]) -> {ok, [{seq_id(), step_result()}]} | {error, term()}.

terminate/1

-callback terminate(state()) -> ok.

thinking_signature/3

(optional) 
-callback thinking_signature(state(), seq_id(), Bytes :: binary()) -> binary().

tokenize/2

-callback tokenize(state(), Text :: binary()) -> [erllama:token_id()] | {error, term()}.

unload_adapter/2

(optional) 
-callback unload_adapter(state(), Handle :: term()) -> {ok, state()} | {error, term()}.

verify/4

(optional) 
-callback verify(state(),
                 PrefixTokens :: [erllama:token_id()],
                 Candidates :: [erllama:token_id()],
                 K :: pos_integer()) ->
                    {ok,
                     AcceptedCount :: non_neg_integer(),
                     NextToken :: erllama:token_id() | eos,
                     NewState :: state()} |
                    {error, term()}.

vocab_info/1

(optional) 
-callback vocab_info(state()) -> {ok, map()} | {error, term()}.