ReqLLM.Cache behaviour (ReqLLM v1.14.0)

Application-layer response cache hooks for generation requests.

ReqLLM does not ship a cache store. Instead, callers can pass a module that implements this behaviour to :cache and back it with Cachex, Nebulex, ETS, Redis, or any custom storage.

Summary

Types

request()

Callbacks

delete(key, opts)

generate_key(model, request, opts)

get(key, opts)

put(key, value, ttl, opts)

Functions

cache_opts(opts)

fetch(model, operation, context, opts, schema \\ nil)

request_opts(opts)

store(arg1, response, opts)

stream_response(response, model, context)

Types

request()

@type request() :: %{
  operation: :chat | :object,
  context: ReqLLM.Context.t(),
  schema: map() | nil
}

Callbacks

delete(key, opts)

@callback delete(key :: term(), opts :: keyword() | map()) :: :ok | {:error, term()}

generate_key(model, request, opts)

@callback generate_key(
  model :: LLMDB.Model.t(),
  request :: request(),
  opts :: keyword() | map()
) :: term()

get(key, opts)

@callback get(key :: term(), opts :: keyword() | map()) ::
  {:ok, ReqLLM.Response.t()} | {:error, term()}

put(key, value, ttl, opts)

@callback put(
  key :: term(),
  value :: ReqLLM.Response.t(),
  ttl :: non_neg_integer() | nil,
  opts :: keyword() | map()
) :: :ok | {:error, term()}

Functions

cache_opts(opts)

@spec cache_opts(keyword()) :: keyword() | map()

fetch(model, operation, context, opts, schema \\ nil)

@spec fetch(
  LLMDB.Model.t(),
  :chat | :object,
  ReqLLM.Context.t(),
  keyword(),
  map() | nil
) ::
  {:hit, ReqLLM.Response.t(), %{backend: module(), key: term()}}
  | {:miss, %{backend: module(), key: term()} | nil}

request_opts(opts)

@spec request_opts(keyword()) :: keyword()

store(arg1, response, opts)

@spec store(%{backend: module(), key: term()} | nil, ReqLLM.Response.t(), keyword()) ::
  ReqLLM.Response.t()

stream_response(response, model, context)

@spec stream_response(ReqLLM.Response.t(), LLMDB.Model.t(), ReqLLM.Context.t()) ::
  ReqLLM.StreamResponse.t()