defmodule WhisperCpp do @moduledoc """ Native Elixir bindings for [whisper.cpp](https://github.com/ggerganov/whisper.cpp). A thin wrapper around the [`whisper-rs`](https://codeberg.org/tazz4843/whisper-rs) crate, calling whisper.cpp's C API through a Rustler NIF. No `whisper-cli` subprocess, no Python, no temporary files. Structured per-segment results, `:initial_prompt` biasing, word-level timestamps, and CUDA / ROCm (hipBLAS) / Metal / CPU backends. ## Quickstart {:ok, model} = WhisperCpp.load_model("models/ggml-large-v3.bin") {:ok, %WhisperCpp.Transcription{text: text, segments: segs}} = WhisperCpp.transcribe(model, {:pcm_f32, samples}, language: "en") IO.puts(text) for s <- segs, do: IO.puts("[\#{s.start}-\#{s.end}] \#{s.text}") ## Audio contract `transcribe/3` accepts exactly one input shape: {:pcm_f32, binary()} where `binary` is little-endian IEEE-754 `f32` samples, mono, 16 kHz, normalised to `[-1.0, 1.0]`. Decode audio file formats (WAV, MP3, FLAC, M4A, Opus, ...) upstream with ffmpeg or similar: ffmpeg -i input.mp3 -f f32le -ac 1 -ar 16000 - | … Use `transcribe_slice/4` to transcribe a `[start_s, end_s)` window of an already-decoded master PCM buffer; the returned segment / word times are shifted back into the original audio timeline. """ alias WhisperCpp.AbortHandle alias WhisperCpp.Error alias WhisperCpp.Model alias WhisperCpp.Native alias WhisperCpp.Pcm alias WhisperCpp.Segment alias WhisperCpp.Transcription alias WhisperCpp.Word @typedoc "Audio input accepted by `transcribe/3`." @type audio :: {:pcm_f32, binary()} @target_sample_rate 16_000 @typedoc "Options accepted by `transcribe/3` / `transcribe_slice/4`." @type transcribe_opt :: {:language, String.t() | nil} | {:translate, boolean()} | {:initial_prompt, String.t() | nil} | {:word_timestamps, boolean()} | {:beam_size, pos_integer()} | {:best_of, pos_integer()} | {:temperature, float()} | {:n_threads, pos_integer()} | {:n_max_text_ctx, non_neg_integer()} | {:offset_ms, non_neg_integer()} | {:duration_ms, non_neg_integer()} | {:no_speech_thold, float()} | {:logprob_thold, float()} | {:suppress_blank, boolean()} | {:suppress_non_speech_tokens, boolean()} | {:single_segment, boolean()} | {:print_progress, boolean()} | {:abort_handle, AbortHandle.t() | nil} | {:progress_pid, pid() | nil} @typedoc "Options accepted by `load_model/2`." @type load_opt :: {:device, Model.device() | :auto} | {:use_gpu, boolean()} @devices [:cpu, :cuda, :hipblas, :vulkan, :metal, :coreml, :intel_sycl, :auto] @doc """ Reports the runtime backends compiled into this NIF artefact. Returns `{:ok, %{backends: [...], gpu_supported: bool}}`. The `backends` list reflects compile-time cargo features (e.g. `[:cpu, :cuda]` on a `WHISPER_CPP_VARIANT=cuda` build). Build a source artefact with GPU support via: WHISPER_CPP_BUILD=1 WHISPER_CPP_FEATURES=cuda mix compile # NVIDIA WHISPER_CPP_BUILD=1 WHISPER_CPP_FEATURES=hipblas mix compile # AMD ROCm WHISPER_CPP_BUILD=1 WHISPER_CPP_FEATURES=vulkan mix compile # cross-vendor WHISPER_CPP_BUILD=1 WHISPER_CPP_FEATURES=metal mix compile # Apple Silicon WHISPER_CPP_BUILD=1 WHISPER_CPP_FEATURES=coreml mix compile # Apple ANE WHISPER_CPP_BUILD=1 WHISPER_CPP_FEATURES=intel-sycl mix compile # Intel Arc/Xe WHISPER_CPP_BUILD=1 WHISPER_CPP_FEATURES=openblas mix compile # CPU + OpenBLAS WHISPER_CPP_BUILD=1 WHISPER_CPP_FEATURES=openmp mix compile # CPU + OpenMP Pick one accelerator per build; the backend is baked into the artefact. """ @spec available_devices() :: {:ok, %{backends: [atom()], gpu_supported: boolean()}} | {:error, Error.t()} def available_devices do case Native.available_devices() do {:ok, info} -> {:ok, %{ backends: Enum.map(info.backends, &String.to_existing_atom/1), gpu_supported: info.gpu_supported }} {:error, payload} -> {:error, Error.from_native(payload)} end end @doc """ Loads a GGUF or GGML whisper.cpp model file. Pass a path to a `.bin` (legacy GGML) or `.gguf` file. Download official weights from . ## Options - `:device` - one of `:cpu`, `:cuda`, `:hipblas`, `:vulkan`, `:metal`, `:coreml`, `:intel_sycl`, or `:auto` (default). `:auto` picks the GPU backend when the artefact was built with one; otherwise CPU. Requesting a backend that was not compiled in returns `{:error, %WhisperCpp.Error{reason: :invalid_request}}`. - `:use_gpu` - shortcut: `false` forces `device: :cpu`. Default `true`. """ @spec load_model(Path.t(), [load_opt()]) :: {:ok, Model.t()} | {:error, Error.t()} def load_model(path, opts \\ []) def load_model(path, opts) when is_binary(path) and is_list(opts) do with :ok <- validate_non_empty_string(path, :path), :ok <- validate_options(opts, load_validators()) do do_load_model(path, opts) end end def load_model(_path, _opts) do {:error, Error.new(:invalid_request, "path must be a string and opts a keyword list")} end defp do_load_model(path, opts) do with {:ok, ref} <- native_call(Native.load_model(path, build_load_opts(opts))), {:ok, info} <- native_call(Native.model_info(ref)), {:ok, device} <- decode_device(info.device) do {:ok, %Model{ ref: ref, path: path, sampling_rate: info.sampling_rate, multilingual: info.multilingual, n_vocab: info.n_vocab, device: device }} end end @doc """ Transcribes `audio` using `model`. Returns `{:ok, %WhisperCpp.Transcription{}}` whose `:segments` carry absolute start/end times, `no_speech_prob`, `avg_logprob`, the underlying text tokens, and (when `:word_timestamps` is set) per-word timing. ## Options - `:language` - ISO code (`"en"`). `nil` (default) auto-detects on multilingual models; auto-detect on monolingual models always returns `"en"`. - `:translate` - translate to English instead of transcribing. - `:initial_prompt` - free-text context prepended via `<|startofprev|>` to bias decoding (max ~224 tokens). - `:word_timestamps` - attach per-word timing. Default `false`. - `:beam_size` - beam-search width. Default `5`. - `:best_of` - greedy candidates kept when `beam_size <= 1`. - `:temperature` - sampling temperature (`0.0` = greedy/beam). - `:n_threads` - intra-op threads. Default `4`. - `:n_max_text_ctx` - cap decoder context tokens. - `:offset_ms`, `:duration_ms` - clip the audio window. - `:no_speech_thold` - silence detection threshold. Default `0.6`. - `:logprob_thold` - reject segments with `avg_logprob` below this. - `:suppress_blank`, `:suppress_non_speech_tokens` - decoder suppressions. - `:single_segment` - force a single segment for the whole audio. - `:print_progress` - whisper.cpp progress to stderr. - `:abort_handle` - `%WhisperCpp.AbortHandle{}` whose `abort/1` cancels in-flight inference. The call returns `{:ok, partial_transcription}` with whatever segments completed before the abort took effect. - `:progress_pid` - pid that receives `{:whisper_progress, percent}` messages (0..100) as work advances; duplicate percentages are coalesced. """ @spec transcribe(Model.t(), audio(), [transcribe_opt()]) :: {:ok, Transcription.t()} | {:error, Error.t()} def transcribe(model, audio, opts \\ []) def transcribe(%Model{} = model, audio, opts) when is_list(opts) do with :ok <- validate_options(opts, transcribe_validators()), {:ok, samples} <- resolve_audio(audio) do do_transcribe(model, samples, opts, 0.0) end end def transcribe(_model, _audio, _opts) do {:error, Error.new(:invalid_request, "expected a %WhisperCpp.Model{} and a keyword list")} end @doc """ Transcribes a `[start_s, end_s)` slice of `samples` and shifts the returned segment/word timestamps to absolute seconds in the original audio. Slices the f32 PCM buffer, runs whisper.cpp on the slice, and rewrites local segment times back into the absolute timeline. Returns `{:ok, %Transcription{}}` with absolute timings, or `{:error, Error.t()}`. Slices shorter than 0.3 s return an empty transcription (whisper.cpp pads short inputs and hallucinates into the padding). """ @spec transcribe_slice(Model.t(), binary(), {number(), number()}, [transcribe_opt()]) :: {:ok, Transcription.t()} | {:error, Error.t()} def transcribe_slice(model, samples, range, opts \\ []) def transcribe_slice(%Model{} = model, samples, {start_s, end_s}, opts) when is_binary(samples) and is_number(start_s) and is_number(end_s) and is_list(opts) do with :ok <- validate_slice_range(start_s, end_s), :ok <- validate_options(opts, transcribe_validators()), {:ok, slice} <- Pcm.slice(samples, sample_rate(), start_s, end_s - start_s), {:ok, transcription} <- do_transcribe(model, slice, opts, start_s * 1.0) do {:ok, transcription} else {:short, _} -> {:ok, empty_transcription(start_s, end_s)} err -> err end end def transcribe_slice(_model, _samples, _range, _opts) do {:error, Error.new(:invalid_request, "expected a %Model{}, an f32 PCM binary, and a {start_s, end_s} tuple")} end defp validate_slice_range(start_s, _end_s) when start_s < 0, do: {:error, Error.new(:invalid_request, "start_s must be >= 0", %{start_s: start_s})} defp validate_slice_range(start_s, end_s) when end_s <= start_s, do: {:error, Error.new(:invalid_request, "end_s must be greater than start_s", %{ start_s: start_s, end_s: end_s })} defp validate_slice_range(start_s, end_s) when end_s - start_s < 0.3, do: {:short, end_s - start_s} defp validate_slice_range(_start_s, _end_s), do: :ok defp empty_transcription(start_s, end_s) do %Transcription{text: "", segments: [], language: "", duration_s: (end_s - start_s) * 1.0} end defp do_transcribe(%Model{ref: ref}, samples, opts, offset_s) do abort_ref = case Keyword.get(opts, :abort_handle) do %AbortHandle{ref: ref} -> ref nil -> nil end progress_pid = Keyword.get(opts, :progress_pid) case Native.transcribe(ref, samples, build_transcribe_opts(opts), abort_ref, progress_pid) do {:ok, payload} -> {:ok, build_transcription(payload, offset_s)} {:error, payload} -> {:error, Error.from_native(payload)} end end defp resolve_audio({:pcm_f32, samples}) when is_binary(samples) do cond do byte_size(samples) == 0 -> {:error, Error.new(:invalid_request, "PCM binary is empty")} rem(byte_size(samples), 4) != 0 -> {:error, Error.new(:invalid_request, "PCM binary length must be a multiple of 4 (f32)", %{ byte_size: byte_size(samples) })} true -> {:ok, samples} end end defp resolve_audio(_) do {:error, Error.new( :invalid_request, "audio must be {:pcm_f32, binary} (little-endian f32 mono at 16 kHz); " <> "decode files upstream" )} end defp sample_rate, do: @target_sample_rate defp native_call({:ok, _} = ok), do: ok defp native_call({:error, payload}), do: {:error, Error.from_native(payload)} defp build_load_opts(opts) do device = case Keyword.get(opts, :device) do nil -> if Keyword.get(opts, :use_gpu, true), do: "auto", else: "cpu" atom when is_atom(atom) -> Atom.to_string(atom) end %{device: device} end @device_atoms Map.new(@devices, fn a -> {Atom.to_string(a), a} end) defp decode_device(label) when is_binary(label) do case Map.fetch(@device_atoms, label) do {:ok, atom} -> {:ok, atom} :error -> {:error, Error.new(:runtime_error, "NIF reported unknown device", %{device: label})} end end @doc false @spec build_transcription(map(), float()) :: Transcription.t() def build_transcription(%{language: language, duration_s: duration_s, segments: raw_segments}, offset_s) do segments = Enum.map(raw_segments, &build_segment(&1, offset_s)) text = segments |> Enum.map_join(" ", & &1.text) |> String.trim() %Transcription{ text: text, segments: segments, language: language, duration_s: duration_s } end @doc false @spec build_segment(map(), float()) :: Segment.t() def build_segment( %{ text: text, start: start, end: end_s, no_speech_prob: no_speech_prob, avg_logprob: avg_logprob, tokens: tokens, words: words }, offset_s ) do %Segment{ text: text, start: start + offset_s, end: end_s + offset_s, no_speech_prob: no_speech_prob, avg_logprob: avg_logprob, tokens: tokens, words: words && Enum.map(words, &build_word(&1, offset_s)) } end @doc false @spec build_word(map(), float()) :: Word.t() def build_word(%{text: text, start: start, end: end_s, probability: probability}, offset_s) do %Word{text: text, start: start + offset_s, end: end_s + offset_s, probability: probability} end defp build_transcribe_opts(opts) do %{ language: Keyword.get(opts, :language), translate: Keyword.get(opts, :translate), initial_prompt: Keyword.get(opts, :initial_prompt), word_timestamps: Keyword.get(opts, :word_timestamps), beam_size: Keyword.get(opts, :beam_size), best_of: Keyword.get(opts, :best_of), temperature: Keyword.get(opts, :temperature), n_threads: Keyword.get(opts, :n_threads), n_max_text_ctx: Keyword.get(opts, :n_max_text_ctx), offset_ms: Keyword.get(opts, :offset_ms), duration_ms: Keyword.get(opts, :duration_ms), no_speech_thold: Keyword.get(opts, :no_speech_thold), logprob_thold: Keyword.get(opts, :logprob_thold), suppress_blank: Keyword.get(opts, :suppress_blank), suppress_non_speech_tokens: Keyword.get(opts, :suppress_non_speech_tokens), single_segment: Keyword.get(opts, :single_segment), print_progress: Keyword.get(opts, :print_progress) } end @spec validate_non_empty_string(String.t(), atom()) :: :ok | {:error, Error.t()} defp validate_non_empty_string(value, name) do if String.trim(value) == "" do {:error, Error.new(:invalid_request, "#{name} must be a non-empty string")} else :ok end end defp load_validators do %{device: &(&1 in @devices), use_gpu: &is_boolean/1} end defp transcribe_validators do %{ language: &valid_optional_string?/1, translate: &is_boolean/1, initial_prompt: &valid_optional_string?/1, word_timestamps: &is_boolean/1, beam_size: &positive_integer?/1, best_of: &positive_integer?/1, temperature: &non_neg_number?/1, n_threads: &positive_integer?/1, n_max_text_ctx: &non_neg_integer?/1, offset_ms: &non_neg_integer?/1, duration_ms: &non_neg_integer?/1, no_speech_thold: &number?/1, logprob_thold: &number?/1, suppress_blank: &is_boolean/1, suppress_non_speech_tokens: &is_boolean/1, single_segment: &is_boolean/1, print_progress: &is_boolean/1, abort_handle: &valid_abort_handle?/1, progress_pid: &valid_optional_pid?/1 } end defp valid_abort_handle?(nil), do: true defp valid_abort_handle?(%AbortHandle{}), do: true defp valid_abort_handle?(_), do: false defp valid_optional_pid?(nil), do: true defp valid_optional_pid?(pid) when is_pid(pid), do: true defp valid_optional_pid?(_), do: false @spec validate_options(keyword(), map()) :: :ok | {:error, Error.t()} defp validate_options(opts, validators) do Enum.reduce_while(opts, :ok, fn pair, :ok -> check_option(pair, validators) end) end defp check_option({key, value}, validators) do case Map.fetch(validators, key) do :error -> {:halt, {:error, Error.new(:invalid_request, "unknown option #{inspect(key)}")}} {:ok, validator} -> if validator.(value) do {:cont, :ok} else {:halt, {:error, Error.new(:invalid_request, "invalid value for option #{inspect(key)}: #{inspect(value)}")}} end end end defp valid_optional_string?(nil), do: true defp valid_optional_string?(value) when is_binary(value), do: String.trim(value) != "" defp valid_optional_string?(_), do: false defp positive_integer?(v), do: is_integer(v) and v > 0 defp non_neg_integer?(v), do: is_integer(v) and v >= 0 defp number?(v), do: is_integer(v) or is_float(v) defp non_neg_number?(v), do: number?(v) and v >= 0 end