defmodule ExLLM do @moduledoc """ ExLLM - Unified Elixir client library for Large Language Models. ExLLM provides a consistent interface across multiple LLM providers including OpenAI, Anthropic Claude, Ollama, and others. It features configuration injection, standardized error handling, and streaming support. ## Quick Start # Using environment variables messages = [%{role: "user", content: "Hello!"}] {:ok, response} = ExLLM.chat(:anthropic, messages) IO.puts(response.content) # Using static configuration config = %{anthropic: %{api_key: "your-key"}} {:ok, provider} = ExLLM.ConfigProvider.Static.start_link(config) {:ok, response} = ExLLM.chat(:anthropic, messages, config_provider: provider) ## Supported Providers - `:anthropic` - Anthropic Claude models - `:openai` - OpenAI GPT models - `:ollama` - Local models via Ollama - `:bedrock` - AWS Bedrock (multiple providers) - `:gemini` - Google Gemini models - `:local` - Local models via Bumblebee (Phi-2, Llama 2, Mistral, etc.) ## Features - **Unified Interface**: Same API across all providers - **Configuration Injection**: Flexible config management - **Streaming Support**: Real-time response streaming - **Error Standardization**: Consistent error handling - **No Process Dependencies**: Pure functional core - **Extensible**: Easy to add new providers ## Configuration ExLLM supports multiple configuration methods: ### Environment Variables export ANTHROPIC_API_KEY="api-..." export OPENAI_API_KEY="sk-..." export OLLAMA_API_BASE="http://localhost:11434" export GOOGLE_API_KEY="your-key" export AWS_ACCESS_KEY_ID="your-key" export AWS_SECRET_ACCESS_KEY="your-secret" ### Static Configuration config = %{ anthropic: %{api_key: "api-...", model: "claude-3-5-sonnet-20241022"}, openai: %{api_key: "sk-...", model: "gpt-4"}, ollama: %{base_url: "http://localhost:11434", model: "llama2"}, bedrock: %{access_key_id: "...", secret_access_key: "...", region: "us-east-1"}, gemini: %{api_key: "...", model: "gemini-pro"}, local: %{model: "microsoft/phi-2"} } {:ok, provider} = ExLLM.ConfigProvider.Static.start_link(config) ### Custom Configuration defmodule MyConfigProvider do @behaviour ExLLM.ConfigProvider def get([:anthropic, :api_key]), do: MyApp.get_secret("anthropic_key") def get(_), do: nil def get_all(), do: %{} end ## Examples # Simple chat {:ok, response} = ExLLM.chat(:anthropic, [ %{role: "user", content: "What is Elixir?"} ]) # With options {:ok, response} = ExLLM.chat(:anthropic, messages, model: "claude-3-haiku-20240307", temperature: 0.7, max_tokens: 1000 ) # Streaming {:ok, stream} = ExLLM.stream_chat(:anthropic, messages) for chunk <- stream do if chunk.content, do: IO.write(chunk.content) end # Check if provider is configured if ExLLM.configured?(:anthropic) do {:ok, response} = ExLLM.chat(:anthropic, messages) end # List available models {:ok, models} = ExLLM.list_models(:anthropic) Enum.each(models, fn model -> IO.puts(model.name) end) """ require Logger alias ExLLM.{Context, Cost, Session, Types} @providers %{ anthropic: ExLLM.Adapters.Anthropic, local: ExLLM.Adapters.Local, openai: ExLLM.Adapters.OpenAI, ollama: ExLLM.Adapters.Ollama, bedrock: ExLLM.Adapters.Bedrock, gemini: ExLLM.Adapters.Gemini } @type provider :: :anthropic | :openai | :ollama | :local | :bedrock | :gemini @type messages :: [Types.message()] @type options :: keyword() @doc """ Send a chat completion request to the specified LLM provider. ## Parameters - `provider` - The LLM provider (`:anthropic`, `:openai`, `:ollama`) - `messages` - List of conversation messages - `options` - Options for the request (see module docs) ## Options - `:model` - Override the default model - `:temperature` - Temperature setting (0.0 to 1.0) - `:max_tokens` - Maximum tokens in response or context - `:config_provider` - Configuration provider module or pid - `:track_cost` - Whether to track costs (default: true) - `:strategy` - Context truncation strategy (default: :sliding_window) - `:sliding_window` - Keep most recent messages - `:smart` - Preserve system messages and recent context - `:preserve_messages` - Number of recent messages to always preserve (default: 5) - `:response_model` - Ecto schema or type spec for structured output (requires instructor) - `:max_retries` - Number of retries for structured output validation ## Returns `{:ok, %ExLLM.Types.LLMResponse{}}` on success, or `{:ok, struct}` when using response_model. Returns `{:error, reason}` on failure. ## Examples # Simple usage {:ok, response} = ExLLM.chat(:anthropic, [ %{role: "user", content: "Hello!"} ]) # With custom configuration {:ok, provider} = ExLLM.ConfigProvider.Static.start_link(%{ anthropic: %{api_key: "your-key"} }) {:ok, response} = ExLLM.chat(:anthropic, messages, config_provider: provider) # With model override {:ok, response} = ExLLM.chat(:openai, messages, model: "gpt-4-turbo") # With context management {:ok, response} = ExLLM.chat(:anthropic, messages, max_tokens: 4000, strategy: :smart ) # With structured output (requires instructor) {:ok, classification} = ExLLM.chat(:anthropic, messages, response_model: EmailClassification, max_retries: 3 ) """ @spec chat(provider(), messages(), options()) :: {:ok, Types.LLMResponse.t() | struct() | map()} | {:error, term()} def chat(provider, messages, options \\ []) do # Check if structured output is requested if Keyword.has_key?(options, :response_model) do # Delegate to Instructor module if available if Code.ensure_loaded?(ExLLM.Instructor) and ExLLM.Instructor.available?() do ExLLM.Instructor.chat(provider, messages, options) else {:error, :instructor_not_available} end else # Regular chat flow case get_adapter(provider) do {:ok, adapter} -> # Apply context management if enabled prepared_messages = prepare_messages_for_provider(provider, messages, options) result = adapter.chat(prepared_messages, options) # Track costs if enabled if Keyword.get(options, :track_cost, true) and match?({:ok, _}, result) do {:ok, response} = result track_response_cost(provider, response, options) end result {:error, reason} -> {:error, reason} end end end @doc """ Send a streaming chat completion request to the specified LLM provider. ## Parameters - `provider` - The LLM provider (`:anthropic`, `:openai`, `:ollama`) - `messages` - List of conversation messages - `options` - Options for the request (see module docs) ## Options Same as `chat/3`, plus: - `:on_chunk` - Callback function for each chunk ## Returns `{:ok, stream}` on success where stream yields `%ExLLM.Types.StreamChunk{}` structs, `{:error, reason}` on failure. ## Examples {:ok, stream} = ExLLM.stream_chat(:anthropic, messages) # Process the stream for chunk <- stream do case chunk do %{content: content} when content != nil -> IO.write(content) %{finish_reason: "stop"} -> IO.puts("\\nDone!") _ -> :continue end end # With context management {:ok, stream} = ExLLM.stream_chat(:anthropic, messages, max_tokens: 4000, strategy: :smart ) """ @spec stream_chat(provider(), messages(), options()) :: {:ok, Types.stream()} | {:error, term()} def stream_chat(provider, messages, options \\ []) do case get_adapter(provider) do {:ok, adapter} -> # Apply context management if enabled prepared_messages = prepare_messages_for_provider(provider, messages, options) adapter.stream_chat(prepared_messages, options) {:error, reason} -> {:error, reason} end end @doc """ Check if the specified provider is properly configured. ## Parameters - `provider` - The LLM provider to check - `options` - Options including configuration provider ## Returns `true` if configured, `false` otherwise. ## Examples if ExLLM.configured?(:anthropic) do {:ok, response} = ExLLM.chat(:anthropic, messages) else IO.puts("Anthropic not configured") end """ @spec configured?(provider(), options()) :: boolean() def configured?(provider, options \\ []) do case get_adapter(provider) do {:ok, adapter} -> adapter.configured?(options) {:error, _} -> false end end @doc """ Get the default model for the specified provider. ## Parameters - `provider` - The LLM provider ## Returns String model identifier. ## Examples model = ExLLM.default_model(:anthropic) # => "claude-sonnet-4-20250514" """ @spec default_model(provider()) :: String.t() | {:error, term()} def default_model(provider) do case get_adapter(provider) do {:ok, adapter} -> adapter.default_model() {:error, reason} -> {:error, reason} end end @doc """ List available models for the specified provider. ## Parameters - `provider` - The LLM provider - `options` - Options including configuration provider ## Returns `{:ok, [%ExLLM.Types.Model{}]}` on success, `{:error, reason}` on failure. ## Examples {:ok, models} = ExLLM.list_models(:anthropic) Enum.each(models, fn model -> IO.puts(model.name) end) """ @spec list_models(provider(), options()) :: {:ok, [Types.Model.t()]} | {:error, term()} def list_models(provider, options \\ []) do case get_adapter(provider) do {:ok, adapter} -> adapter.list_models(options) {:error, reason} -> {:error, reason} end end @doc """ Calculate cost for token usage. ## Parameters - `provider` - LLM provider name - `model` - Model name - `token_usage` - Map with `:input_tokens` and `:output_tokens` ## Returns Cost calculation result or error map. ## Examples usage = %{input_tokens: 1000, output_tokens: 500} cost = ExLLM.calculate_cost("openai", "gpt-4", usage) # => %{total_cost: 0.06, ...} """ @spec calculate_cost(provider(), String.t(), Types.token_usage()) :: Types.cost_result() | %{error: String.t()} def calculate_cost(provider, model, token_usage) do Cost.calculate(to_string(provider), model, token_usage) end @doc """ Estimate token count for text. ## Parameters - `text` - Text to analyze (string, message map, or list) ## Returns Estimated token count. ## Examples tokens = ExLLM.estimate_tokens("Hello, world!") # => 4 """ @spec estimate_tokens(String.t() | map() | [map()]) :: non_neg_integer() def estimate_tokens(text) do Cost.estimate_tokens(text) end @doc """ Format cost for display. ## Parameters - `cost` - Cost in dollars ## Returns Formatted cost string. ## Examples ExLLM.format_cost(0.0035) # => "$0.350ยข" """ @spec format_cost(float()) :: String.t() def format_cost(cost) do Cost.format(cost) end @doc """ Get list of supported providers. ## Returns List of provider atoms. ## Examples providers = ExLLM.supported_providers() # => [:anthropic, :openai, :ollama] """ @spec supported_providers() :: [provider()] def supported_providers do Map.keys(@providers) end @doc """ Prepare messages for sending to a provider with context management. ## Parameters - `messages` - List of conversation messages - `options` - Options for context management ## Options - `:max_tokens` - Maximum tokens for context (default: model-specific) - `:strategy` - Context truncation strategy (default: :sliding_window) - `:preserve_messages` - Number of recent messages to preserve (default: 5) ## Returns Prepared messages list that fits within context window. ## Examples messages = ExLLM.prepare_messages(long_conversation, max_tokens: 4000, strategy: :smart ) """ @spec prepare_messages(messages(), options()) :: messages() def prepare_messages(messages, options \\ []) do Context.prepare_messages(messages, options) end @doc """ Validate that messages fit within a model's context window. ## Parameters - `messages` - List of conversation messages - `options` - Options including model info ## Returns `{:ok, token_count}` if valid, `{:error, reason}` if too large. ## Examples {:ok, tokens} = ExLLM.validate_context(messages, model: "claude-3-5-sonnet-20241022") # => {:ok, 3500} """ @spec validate_context(messages(), options()) :: {:ok, non_neg_integer()} | {:error, term()} def validate_context(messages, options \\ []) do Context.validate_context(messages, options) end @doc """ Get context window size for a model. ## Parameters - `provider` - LLM provider name - `model` - Model name ## Returns Context window size in tokens or nil if unknown. ## Examples tokens = ExLLM.context_window_size(:anthropic, "claude-3-5-sonnet-20241022") # => 200000 """ @spec context_window_size(provider(), String.t()) :: non_neg_integer() | nil def context_window_size(provider, model) do Context.context_window_size(to_string(provider), model) end @doc """ Get statistics about message context usage. ## Parameters - `messages` - List of conversation messages ## Returns Map with context statistics. ## Examples stats = ExLLM.context_stats(messages) # => %{total_tokens: 1500, message_count: 10, ...} """ @spec context_stats(messages()) :: map() def context_stats(messages) do Context.stats(messages) end # Session Management @doc """ Create a new conversation session. ## Parameters - `provider` - LLM provider to use for the session - `opts` - Session options (`:name` for session name) ## Returns A new session struct. ## Examples session = ExLLM.new_session(:anthropic) session = ExLLM.new_session(:openai, name: "Customer Support") """ @spec new_session(provider(), keyword()) :: Session.Types.Session.t() def new_session(provider, opts \\ []) do Session.new(to_string(provider), opts) end @doc """ Send a chat request using a session, automatically tracking messages and usage. ## Parameters - `session` - The session to use - `content` - The user message content - `options` - Chat options (same as `chat/3`) ## Returns `{:ok, {response, updated_session}}` on success, `{:error, reason}` on failure. ## Examples session = ExLLM.new_session(:anthropic) {:ok, {response, session}} = ExLLM.chat_with_session(session, "Hello!") # Session now contains the conversation history """ @spec chat_with_session(Session.Types.Session.t(), String.t(), options()) :: {:ok, {Types.LLMResponse.t(), Session.Types.Session.t()}} | {:error, term()} def chat_with_session(session, content, options \\ []) do # Add user message to session session = Session.add_message(session, "user", content) # Get provider from session provider = String.to_atom(session.llm_backend || "anthropic") # Get messages for chat messages = Session.get_messages(session) # Merge session context with options merged_options = Keyword.merge( Map.to_list(session.context || %{}), options ) # Send chat request case chat(provider, messages, merged_options) do {:ok, response} -> # Add assistant response to session session = Session.add_message(session, "assistant", response.content) # Update token usage if available session = if response.usage do Session.update_token_usage(session, response.usage) else session end {:ok, {response, session}} {:error, reason} -> {:error, reason} end end @doc """ Add a message to a session. ## Parameters - `session` - The session to update - `role` - Message role ("user", "assistant", etc.) - `content` - Message content - `opts` - Additional message metadata ## Returns Updated session. ## Examples session = ExLLM.add_session_message(session, "user", "What is Elixir?") """ @spec add_session_message(Session.Types.Session.t(), String.t(), String.t(), keyword()) :: Session.Types.Session.t() def add_session_message(session, role, content, opts \\ []) do Session.add_message(session, role, content, opts) end @doc """ Get messages from a session. ## Parameters - `session` - The session to query - `limit` - Optional message limit ## Returns List of messages. ## Examples messages = ExLLM.get_session_messages(session) last_10 = ExLLM.get_session_messages(session, 10) """ @spec get_session_messages(Session.Types.Session.t(), non_neg_integer() | nil) :: [Session.Types.message()] def get_session_messages(session, limit \\ nil) do Session.get_messages(session, limit) end @doc """ Get total token usage for a session. ## Parameters - `session` - The session to analyze ## Returns Total token count. ## Examples tokens = ExLLM.session_token_usage(session) # => 2500 """ @spec session_token_usage(Session.Types.Session.t()) :: non_neg_integer() def session_token_usage(session) do Session.total_tokens(session) end @doc """ Clear messages from a session while preserving metadata. ## Parameters - `session` - The session to clear ## Returns Updated session with no messages. ## Examples session = ExLLM.clear_session(session) """ @spec clear_session(Session.Types.Session.t()) :: Session.Types.Session.t() def clear_session(session) do Session.clear_messages(session) end @doc """ Save a session to JSON. ## Parameters - `session` - The session to save ## Returns `{:ok, json}` on success, `{:error, reason}` on failure. ## Examples {:ok, json} = ExLLM.save_session(session) File.write!("session.json", json) """ @spec save_session(Session.Types.Session.t()) :: {:ok, String.t()} | {:error, term()} def save_session(session) do Session.to_json(session) end @doc """ Load a session from JSON. ## Parameters - `json` - JSON string containing session data ## Returns `{:ok, session}` on success, `{:error, reason}` on failure. ## Examples json = File.read!("session.json") {:ok, session} = ExLLM.load_session(json) """ @spec load_session(String.t()) :: {:ok, Session.Types.Session.t()} | {:error, term()} def load_session(json) do Session.from_json(json) end # Private functions defp get_adapter(provider) do case Map.get(@providers, provider) do nil -> {:error, {:unsupported_provider, provider}} adapter -> {:ok, adapter} end end defp prepare_messages_for_provider(provider, messages, options) do # Get model from options or use default model = case Keyword.get(options, :model) do nil -> case default_model(provider) do {:error, _} -> nil model -> model end model -> model end # Add provider and model info to options for context management context_options = options |> Keyword.put(:provider, to_string(provider)) |> Keyword.put_new(:model, model) Context.prepare_messages(messages, context_options) end defp track_response_cost(provider, response, options) do # Extract usage info if available case Map.get(response, :usage) do %{input_tokens: _, output_tokens: _} = usage -> model = Keyword.get(options, :model) || default_model(provider) cost_info = calculate_cost(provider, model, usage) # Log cost info if logger is available if function_exported?(Logger, :info, 1) do Logger.info("LLM cost: #{format_cost(cost_info.total_cost)} for #{provider}/#{model}") end cost_info _ -> nil end end end