defmodule Anakin do @moduledoc """ Official Elixir SDK for the [Anakin](https://anakin.io) web-scraping API. Build a client with `Anakin.Client.new/1` and call any of the endpoint functions in this module: {:ok, client} = Anakin.Client.new(api_key: "ak-...") {:ok, doc} = Anakin.scrape(client, "https://example.com") IO.puts(doc["markdown"]) Long-running endpoints (`scrape/3`, `map/3`, `crawl/3`, `agentic_search/3`, `wire/3`) poll internally and return the final result. Synchronous `search/3` returns immediately. Every function returns `{:ok, result}` or `{:error, exception}`. """ alias Anakin.Client alias Anakin.Error @version "0.1.0" @doc "SDK version string." @spec version() :: String.t() def version, do: @version # ── public endpoints ─────────────────────────────────────────────────── @doc "Scrape a single URL. Polls internally until the job reaches a terminal status." @spec scrape(Client.t(), String.t(), keyword() | map()) :: {:ok, map()} | {:error, Exception.t()} def scrape(%Client{} = client, url, opts \\ []) do body = Map.merge(%{"url" => url}, normalise_opts(opts)) with {:ok, submit} <- request(client, :post, "/url-scraper", body), {:ok, job_id} <- require_field(submit, "job_id"), {:ok, poll} <- poll_job(client, "/url-scraper/#{job_id}") do {:ok, Map.get(poll, "result", %{})} end end @doc "Discover links for a domain. Polls until the job completes." @spec map(Client.t(), String.t(), keyword() | map()) :: {:ok, map()} | {:error, Exception.t()} def map(%Client{} = client, url, opts \\ []) do body = Map.merge(%{"url" => url}, normalise_opts(opts)) with {:ok, submit} <- request(client, :post, "/map", body), {:ok, job_id} <- require_field(submit, "job_id"), {:ok, poll} <- poll_job(client, "/map/#{job_id}") do {:ok, Map.get(poll, "result", %{})} end end @doc "Crawl a site. Polls until the job completes." @spec crawl(Client.t(), String.t(), keyword() | map()) :: {:ok, map()} | {:error, Exception.t()} def crawl(%Client{} = client, url, opts \\ []) do body = Map.merge(%{"url" => url}, normalise_opts(opts)) with {:ok, submit} <- request(client, :post, "/crawl", body), {:ok, job_id} <- require_field(submit, "job_id"), {:ok, poll} <- poll_job(client, "/crawl/#{job_id}") do {:ok, Map.get(poll, "result", %{})} end end @doc "Synchronous web search." @spec search(Client.t(), String.t(), keyword() | map()) :: {:ok, map()} | {:error, Exception.t()} def search(%Client{} = client, query, opts \\ []) do body = Map.merge(%{"prompt" => query}, normalise_opts(opts)) request(client, :post, "/search", body) end @doc "AI-synthesised agentic search. Polls until the job completes." @spec agentic_search(Client.t(), String.t(), keyword() | map()) :: {:ok, map()} | {:error, Exception.t()} def agentic_search(%Client{} = client, prompt, opts \\ []) do body = Map.merge(%{"prompt" => prompt}, normalise_opts(opts)) with {:ok, submit} <- request(client, :post, "/agentic-search", body), {:ok, job_id} <- require_field(submit, "job_id"), {:ok, poll} <- poll_job(client, "/agentic-search/#{job_id}") do {:ok, Map.get(poll, "result", %{})} end end @doc "Execute a Wire (Holocron) action by ID. Polls until the job completes." @spec wire(Client.t(), String.t(), map()) :: {:ok, map()} | {:error, Exception.t()} def wire(%Client{} = client, action_id, params \\ %{}) when is_map(params) do body = if map_size(params) == 0 do %{"action_id" => action_id} else %{"action_id" => action_id, "params" => params} end with {:ok, submit} <- request(client, :post, "/holocron/task", body), {:ok, job_id} <- require_field(submit, "job_id"), {:ok, poll} <- poll_job(client, "/holocron/task/#{job_id}") do {:ok, Map.get(poll, "result", %{})} end end # ── browser-session management ───────────────────────────────────────── @doc "List all saved sessions for the API key." @spec list_sessions(Client.t()) :: {:ok, [map()]} | {:error, Exception.t()} def list_sessions(%Client{} = client) do case request(client, :get, "/browser-sessions", nil) do {:ok, %{"sessions" => list}} when is_list(list) -> {:ok, list} {:ok, list} when is_list(list) -> {:ok, list} {:ok, _} -> {:ok, []} {:error, _} = err -> err end end @doc "Create an empty named session." @spec create_session(Client.t(), String.t(), String.t() | nil) :: {:ok, map()} | {:error, Exception.t()} def create_session(%Client{} = client, name, description \\ nil) do body = put_some(%{"name" => name}, "description", description) request(client, :post, "/browser-sessions", body) end @doc "Save the current state of a CDP session by ID." @spec save_session(Client.t(), String.t(), keyword()) :: {:ok, map()} | {:error, Exception.t()} def save_session(%Client{} = client, session_id, opts \\ []) do body = %{} |> put_some("name", Keyword.get(opts, :name)) |> put_some("description", Keyword.get(opts, :description)) request(client, :post, "/browser-sessions/#{session_id}/save", body) end @doc "Update a saved session's metadata." @spec update_session(Client.t(), String.t(), keyword()) :: {:ok, map()} | {:error, Exception.t()} def update_session(%Client{} = client, session_id, opts \\ []) do body = %{} |> put_some("name", Keyword.get(opts, :name)) |> put_some("description", Keyword.get(opts, :description)) request(client, :put, "/browser-sessions/#{session_id}", body) end @doc "Delete a saved session." @spec delete_session(Client.t(), String.t()) :: :ok | {:error, Exception.t()} def delete_session(%Client{} = client, session_id) do case request(client, :delete, "/browser-sessions/#{session_id}", nil) do {:ok, _} -> :ok {:error, _} = err -> err end end # ── HTTP plumbing ────────────────────────────────────────────────────── @doc false @spec request(Client.t(), atom(), String.t(), map() | nil) :: {:ok, map() | list()} | {:error, Exception.t()} def request(%Client{} = client, method, path, body \\ nil) do do_request(client, method, path, body, 0, nil) end defp do_request(%Client{} = client, method, path, body, attempt, prev_resp) do if attempt > 0 do Process.sleep(backoff_ms(attempt, prev_resp)) end base_opts = [ method: method, url: client.base_url <> path, headers: [ {"x-api-key", client.api_key}, {"accept", "application/json"}, {"user-agent", "anakin-elixir/#{@version}"} ], receive_timeout: client.request_timeout_ms, retry: false ] |> maybe_put(:json, body) |> Keyword.merge(client.req_options || []) case Req.request(base_opts) do {:ok, %Req.Response{status: status} = resp} when status in [429] or status >= 500 -> if attempt < client.max_retries do do_request(client, method, path, body, attempt + 1, resp) else {:error, map_error(resp)} end {:ok, %Req.Response{status: status, body: rbody}} when status in 200..299 -> {:ok, normalise_body(rbody)} {:ok, %Req.Response{} = resp} -> {:error, map_error(resp)} {:error, error} -> if attempt < client.max_retries do do_request(client, method, path, body, attempt + 1, nil) else {:error, %Error.Network{ message: "http request after #{client.max_retries} retries", reason: error }} end end end defp normalise_body(body) when is_binary(body) do case Jason.decode(body) do {:ok, decoded} -> decoded {:error, _} -> %{} end end defp normalise_body(body) when is_map(body) or is_list(body), do: body defp normalise_body(_), do: %{} defp map_error(%Req.Response{status: status, body: body, headers: headers}) do parsed = if is_map(body), do: body, else: normalise_body(body) message = parsed["error"] || "" code = parsed["code"] message = if message == "", do: "HTTP #{status}", else: message case status do 400 -> %Error.InvalidRequest{message: message, status: status, code: code} 401 -> %Error.Authentication{message: message, status: status, code: code} 402 -> %Error.InsufficientCredits{ message: message, status: status, code: code, balance: parsed["balance"] || 0, required: parsed["required"] || 0 } 429 -> %Error.RateLimit{ message: message, status: status, code: code, retry_after: parse_retry_after(headers) } s when s >= 500 -> %Error.Server{message: message, status: status, code: code} _ -> %Error{message: message, status: status, code: code} end end defp parse_retry_after(headers) when is_map(headers) do case Map.get(headers, "retry-after") do [val | _] -> parse_retry_after_val(val) val when is_binary(val) -> parse_retry_after_val(val) _ -> 0 end end defp parse_retry_after(headers) when is_list(headers) do case List.keyfind(headers, "retry-after", 0) do {_, [val | _]} -> parse_retry_after_val(val) {_, val} when is_binary(val) -> parse_retry_after_val(val) _ -> 0 end end defp parse_retry_after(_), do: 0 defp parse_retry_after_val(val) when is_binary(val) do case Integer.parse(String.trim(val)) do {n, _} when n >= 0 -> n _ -> 0 end end defp parse_retry_after_val(_), do: 0 defp backoff_ms(attempt, %Req.Response{headers: headers}) do case parse_retry_after(headers) do n when n > 0 -> n * 1_000 _ -> default_backoff(attempt) end end defp backoff_ms(attempt, _), do: default_backoff(attempt) defp default_backoff(attempt) do ms = trunc(:math.pow(2, attempt - 1) * 500) min(ms, 30_000) end # ── polling ──────────────────────────────────────────────────────────── defp poll_job(%Client{} = client, path) do deadline = System.monotonic_time(:millisecond) + client.poll_timeout_ms poll_loop(client, path, deadline, client.poll_interval_ms) end defp poll_loop(client, path, deadline, delay_ms) do case request(client, :get, path, nil) do {:ok, node} -> status = node["status"] error = node["error"] || "" job_id = node["job_id"] cond do status in ["completed", "succeeded"] -> {:ok, node} status == "failed" -> {:error, %Error.JobFailed{ message: "job failed: #{error}", job_id: job_id, reason: error }} System.monotonic_time(:millisecond) > deadline -> {:error, %Error.JobTimeout{ message: "polling timed out before job reached terminal status", job_id: job_id, elapsed_ms: client.poll_timeout_ms }} true -> Process.sleep(delay_ms) next = min(trunc(delay_ms * 1.5), client.poll_max_interval_ms) poll_loop(client, path, deadline, next) end {:error, _} = err -> err end end # ── helpers ──────────────────────────────────────────────────────────── defp maybe_put(opts, _key, nil), do: opts defp maybe_put(opts, key, value), do: Keyword.put(opts, key, value) defp put_some(map, _key, nil), do: map defp put_some(map, key, value), do: Map.put(map, key, value) defp normalise_opts(opts) when is_list(opts) do Map.new(opts, fn {k, v} -> {to_string(k), v} end) end defp normalise_opts(opts) when is_map(opts) do Map.new(opts, fn {k, v} when is_atom(k) -> {Atom.to_string(k), v} {k, v} -> {k, v} end) end defp require_field(map, field) do case Map.get(map, field) do v when is_binary(v) and v != "" -> {:ok, v} _ -> {:error, %Error{message: "API response missing required field: #{field}"}} end end end