@spec chat(LlamaCppEx.Model.t(), [LlamaCppEx.Chat.message()], keyword()) ::
  {:ok, String.t()} | {:error, term()}

Applies the chat template and generates a response.

Options

Accepts all options from generate/3 plus the chat-templating options from LlamaCppEx.Chat.apply_template/3 (:add_assistant, :enable_thinking, :chat_template_kwargs).

Examples

{:ok, reply} = LlamaCppEx.chat(model, [
  %{role: "system", content: "You are helpful."},
  %{role: "user", content: "What is Elixir?"}
], max_tokens: 200)

@spec chat_completion(
  LlamaCppEx.Model.t() | GenServer.server(),
  [LlamaCppEx.Chat.message()],
  keyword()
) ::
  {:ok, LlamaCppEx.ChatCompletion.t()} | {:error, term()}

Generates an OpenAI-compatible chat completion response.

Applies the chat template, runs generation, and returns a %ChatCompletion{} struct with choices, usage counts, and finish reason.

Options

Accepts all options from generate/3 plus the chat-templating options from LlamaCppEx.Chat.apply_template/3 (:add_assistant, :enable_thinking, :chat_template_kwargs).

Examples

{:ok, completion} = LlamaCppEx.chat_completion(model, [
  %{role: "user", content: "What is Elixir?"}
], max_tokens: 200)

completion.choices |> hd() |> Map.get(:message) |> Map.get(:content)

Errors

Returns {:error, reason} when templating, tokenization, or generation fails, and {:error, :timeout} when generation does not complete within :timeout.

Server routing

When given a running LlamaCppEx.Server (pid or name) instead of a %Model{}, the request is served by the batching server: chat templating and tokenization happen in the caller, and the multi-turn prompt benefits from the server's prefix cache. Server-only request options like :session and :cache_prompt are forwarded.

@spec devices() :: [map()]

Lists the available ggml backend devices (GPUs, integrated GPUs, accelerators, and the CPU).

Each entry is a map with :index (ggml device order), :gpu_index (0-based among GPU/IGPU devices, matching :tensor_split's index space, or nil for non-GPU devices), :name, :description, :type (:gpu, :igpu, :cpu, :accel, or :other), :backend (e.g. "CUDA", "Metal"), :memory_total, and :memory_free (bytes). On Metal, unified memory means a single device.

Examples

:ok = LlamaCppEx.init()
LlamaCppEx.devices()
#=> [%{gpu_index: 0, type: :gpu, name: "NVIDIA RTX 4090",
#      memory_total: 24_000_000_000, memory_free: 23_500_000_000, ...}, ...]

@spec embed(LlamaCppEx.Model.t(), String.t(), keyword()) ::
  {:ok, LlamaCppEx.Embedding.t()} | {:error, String.t()}

Computes an embedding for a single text.

See LlamaCppEx.Embedding.embed/3 for options.

@spec embed_batch(LlamaCppEx.Model.t(), [String.t()], keyword()) ::
  {:ok, [LlamaCppEx.Embedding.t()]} | {:error, String.t()}

Computes embeddings for multiple texts.

See LlamaCppEx.Embedding.embed_batch/3 for options.

@spec generate(LlamaCppEx.Model.t(), String.t(), keyword()) ::
  {:ok, String.t()} | {:error, term()}

Generates text from a prompt.

Creates a temporary context and sampler, tokenizes the prompt, runs generation, and returns the generated text.

Options

• :max_tokens - Maximum tokens to generate. Defaults to 256.
• :n_ctx - Context size. Defaults to 2048.
• :temp - Sampling temperature. 0.0 for greedy. Defaults to 0.8.
• :top_k - Top-K filtering. Defaults to 40.
• :top_p - Top-P (nucleus) filtering. Defaults to 0.95.
• :min_p - Min-P filtering. Defaults to 0.05.
• :seed - Random seed. Defaults to random.
• :penalty_repeat - Repetition penalty. Defaults to 1.0.
• :penalty_freq - Frequency penalty (0.0–2.0). Defaults to 0.0.
• :penalty_present - Presence penalty (0.0–2.0). Defaults to 0.0.
• :grammar - GBNF grammar string for constrained generation.
• :grammar_root - Root rule name for grammar. Defaults to "root".
• :json_schema - JSON Schema map for structured output. Automatically converted to a GBNF grammar. Cannot be used together with :grammar. Tip: set "additionalProperties" => false for tighter grammars.

@spec init() :: :ok

Initializes the llama.cpp backend. Call once at application start.

@spec load_model(
  String.t(),
  keyword()
) :: {:ok, LlamaCppEx.Model.t()} | {:error, String.t()}

Loads a GGUF model from the given file path.

See LlamaCppEx.Model.load/2 for options.

@spec load_model_from_hub(String.t(), String.t(), keyword()) ::
  {:ok, LlamaCppEx.Model.t()} | {:error, String.t()}

Downloads a GGUF model from HuggingFace Hub and loads it.

Requires the optional :req dependency.

Examples

:ok = LlamaCppEx.init()
{:ok, model} = LlamaCppEx.load_model_from_hub(
  "Qwen/Qwen3-4B-GGUF",
  "qwen3-4b-q4_k_m.gguf",
  n_gpu_layers: -1
)

Options

Accepts all options from load_model/2 plus:

• :cache_dir - Local cache directory for downloaded models.
• :token - HuggingFace API token for private repos.
• :progress - Download progress callback.
• :revision - Git revision (branch, tag, commit). Defaults to "main".

@spec stream(LlamaCppEx.Model.t(), String.t(), keyword()) :: Enumerable.t()

Returns a lazy stream of generated text chunks (tokens).

Each element is a string (the text piece for one token). The stream ends when an end-of-generation token is produced or max_tokens is reached.

Accepts the same options as generate/3, except that :timeout bounds the wait for the next chunk rather than the whole generation and therefore defaults to 30000 ms.

Errors

A generation failure is emitted as a final {:error, reason} element rather than truncating the stream silently, matching LlamaCppEx.Server.stream/3. Elements are otherwise always strings, so Enum.join/1 on a stream that may fail is unsafe — match on the element shape if you care.

Examples

model
|> LlamaCppEx.stream("Tell me a story", max_tokens: 500)
|> Enum.each(&IO.write/1)

@spec stream_chat(LlamaCppEx.Model.t(), [LlamaCppEx.Chat.message()], keyword()) ::
  Enumerable.t()

Returns a lazy stream of chat response chunks.

Applies the chat template and streams the generated response token by token. Accepts same options as chat/3.

Like stream/3, a failure is emitted as a final {:error, reason} element.

@spec stream_chat_completion(
  LlamaCppEx.Model.t() | GenServer.server(),
  [LlamaCppEx.Chat.message()],
  keyword()
) :: Enumerable.t()

Returns a lazy stream of OpenAI-compatible chat completion chunks.

Each element is a %ChatCompletionChunk{} struct. The first chunk contains delta: %{role: "assistant", content: ""}. Subsequent chunks contain delta: %{content: "token_text"}. The final chunk contains the finish_reason.

All chunks share the same id and created timestamp.

Options

Accepts same options as chat_completion/3. :timeout bounds the wait for the next chunk rather than the whole generation, so it defaults to 30000 ms for both the model- and server-routed forms. It previously flipped between 60s and 30s depending on the type of the first argument.

Errors

A failure is emitted as a final {:error, reason} element, and a chunk timeout as {:error, :timeout}, rather than truncating the stream silently. Every other element is a %ChatCompletionChunk{}.

Examples

model
|> LlamaCppEx.stream_chat_completion(messages, max_tokens: 200)
|> Enum.each(fn
  %LlamaCppEx.ChatCompletionChunk{} = chunk ->
    chunk.choices |> hd() |> get_in([:delta, :content]) |> IO.write()

  {:error, reason} ->
    IO.warn("generation failed: #{inspect(reason)}")
end)