defmodule OpenResponsesWeb.ComplianceLive do @moduledoc """ Interactive compliance tester matching https://www.openresponses.org/compliance Point it at any running Open Responses server, enter an API key and model, and run the official test suite against the live endpoint. """ use OpenResponsesWeb, :live_view @test_cases [ %{id: "basic-text-response", name: "Basic Text Response", description: "Non-streaming response with a simple user message."}, %{id: "streaming-response", name: "Streaming Response", description: "SSE stream delivers events and ends with response.completed."}, %{id: "system-prompt", name: "System Prompt", description: "System prompt is included in the input and respected."}, %{id: "tool-calling", name: "Tool Calling", description: "Model emits a function_call output item when given a matching tool."}, %{id: "multi-turn", name: "Multi-turn Conversation", description: "Model recalls context supplied earlier in the same input."}, %{id: "previous-response-id", name: "previous_response_id", description: "Second request recalls context from the first via previous_response_id."} ] @impl Phoenix.LiveView def mount(_params, _session, socket) do base_url = OpenResponsesWeb.Endpoint.url() <> "/v1" {:ok, assign(socket, base_url: base_url, api_key: "", model: "gpt-4o-mini", results: %{}, running: false, running_id: nil, test_cases: @test_cases )} end @impl Phoenix.LiveView def handle_event("update_field", %{"field" => field, "value" => value}, socket) do {:noreply, assign(socket, String.to_existing_atom(field), value)} end def handle_event("run_all", _params, socket) do send(self(), :run_next) {:noreply, assign(socket, running: true, results: %{}, running_id: nil)} end def handle_event("run_one", %{"id" => id}, socket) do send(self(), {:run_one, id}) {:noreply, assign(socket, running: true, running_id: id)} end @impl Phoenix.LiveView def handle_info(:run_next, socket) do pending = @test_cases |> Enum.map(& &1.id) |> Enum.reject(&Map.has_key?(socket.assigns.results, &1)) case pending do [] -> {:noreply, assign(socket, running: false, running_id: nil)} [next | _] -> result = execute_test(next, socket.assigns) results = Map.put(socket.assigns.results, next, result) send(self(), :run_next) {:noreply, assign(socket, results: results, running_id: next)} end end def handle_info({:run_one, id}, socket) do result = execute_test(id, socket.assigns) results = Map.put(socket.assigns.results, id, result) {:noreply, assign(socket, results: results, running: false, running_id: nil)} end defp execute_test(id, assigns) do config = %{base_url: assigns.base_url, api_key: assigns.api_key, model: assigns.model} try do response = if id == "previous-response-id" do run_multi_step(config) else post_request(config, build_request(id, assigns.model)) end case response do {:ok, body} -> case validate_response(id, body) do {:pass, _} -> %{status: :pass, response: body} {:fail, reason} -> %{status: :fail, reason: reason, response: body} end {:error, reason} -> %{status: :error, reason: inspect(reason)} end rescue e -> %{status: :error, reason: Exception.message(e)} end end defp build_request("basic-text-response", model) do %{ model: model, input: [%{type: "message", role: "user", content: "Say hello in exactly 3 words."}], stream: false } end defp build_request("streaming-response", model) do %{ model: model, input: [%{type: "message", role: "user", content: "Count from 1 to 5."}], stream: true } end defp build_request("system-prompt", model) do %{ model: model, input: [ %{type: "message", role: "system", content: "You are a pirate. Always respond in pirate speak."}, %{type: "message", role: "user", content: "Say hello."} ], stream: false } end defp build_request("tool-calling", model) do %{ model: model, input: [%{type: "message", role: "user", content: "What's the weather like in San Francisco?"}], tools: [ %{ type: "function", name: "get_weather", description: "Get the current weather for a location", parameters: %{ type: "object", properties: %{location: %{type: "string", description: "City and state"}}, required: ["location"] } } ], stream: false } end defp build_request("multi-turn", model) do %{ model: model, input: [ %{type: "message", role: "user", content: "My name is Alice."}, %{type: "message", role: "assistant", content: "Hello Alice! Nice to meet you."}, %{type: "message", role: "user", content: "What is my name?"} ], stream: false } end defp validate_response("streaming-response", result) do events = result["events"] || [] last = List.last(events) || %{} cond do events == [] -> {:fail, "No SSE events received"} last["type"] != "response.completed" -> {:fail, "Last event was #{inspect(last["type"])}, expected response.completed"} true -> {:pass, nil} end end defp validate_response("tool-calling", response) do function_calls = Enum.filter(response["output"] || [], &(&1["type"] == "function_call")) cond do response["status"] != "completed" -> {:fail, "Expected status \"completed\", got #{inspect(response["status"])}"} function_calls == [] -> {:fail, "Expected at least one function_call output item"} true -> {:pass, nil} end end defp validate_response("multi-turn", response) do text = extract_text(response) cond do response["status"] != "completed" -> {:fail, "Expected status \"completed\", got #{inspect(response["status"])}"} not String.contains?(text, "alice") -> {:fail, "Model did not recall the name Alice from conversation history"} true -> {:pass, nil} end end defp validate_response("previous-response-id", response) do text = extract_text(response) cond do response["status"] != "completed" -> {:fail, "Expected status \"completed\", got #{inspect(response["status"])}"} not String.contains?(text, "zephyr") -> {:fail, "Model did not recall the secret word from the previous response"} true -> {:pass, nil} end end defp validate_response(_id, response) do cond do response["status"] != "completed" -> {:fail, "Expected status \"completed\", got #{inspect(response["status"])}"} not is_list(response["output"]) or response["output"] == [] -> {:fail, "Expected at least one output item"} true -> {:pass, nil} end end defp extract_text(response) do (response["output"] || []) |> Enum.filter(&(&1["type"] == "message")) |> Enum.flat_map(fn item -> content = item["content"] || [] if is_list(content), do: Enum.map(content, & &1["text"]), else: [content] end) |> Enum.join("") |> String.downcase() end defp run_multi_step(config) do with {:ok, first} <- post_request(config, %{ model: config.model, input: [%{type: "message", role: "user", content: "My secret word is Zephyr."}], stream: false }), id when is_binary(id) <- first["id"] do post_request(config, %{ model: config.model, previous_response_id: id, input: [%{type: "message", role: "user", content: "What was my secret word?"}], stream: false }) else _ -> {:error, "First request did not return a valid id"} end end defp post_request(%{base_url: base_url, api_key: api_key}, body) do headers = if api_key != "", do: [{"authorization", "Bearer #{api_key}"}], else: [] url = String.trim_trailing(base_url, "/") <> "/responses" case Req.post(url, json: body, headers: headers, receive_timeout: 60_000) do {:ok, %{status: 200, body: body}} when is_map(body) -> {:ok, body} {:ok, %{status: 200, body: body}} -> case Jason.decode(body) do {:ok, decoded} -> {:ok, decoded} {:error, _} -> {:ok, parse_sse_response(body)} end {:ok, %{status: status, body: body}} -> {:error, "HTTP #{status}: #{inspect(body)}"} {:error, reason} -> {:error, reason} end end defp parse_sse_response(body) when is_binary(body) do events = body |> String.split("\n\n") |> Enum.flat_map(fn chunk -> lines = String.split(chunk, "\n") event_type = Enum.find_value(lines, fn line -> case String.split(line, "event: ", parts: 2) do [_, type] -> String.trim(type) _ -> nil end end) data_json = Enum.find_value(lines, fn line -> case String.split(line, "data: ", parts: 2) do [_, "[DONE]"] -> nil [_, json] -> json _ -> nil end end) case data_json && Jason.decode(data_json) do {:ok, event} -> event = if event_type && !Map.has_key?(event, "type"), do: Map.put(event, "type", event_type), else: event [event] _ -> [] end end) %{"events" => events} end @impl Phoenix.LiveView def render(assigns) do ~H"""

compliance tester

Test any Open Responses server against the official specification. Mirrors openresponses.org/compliance.

<%= if map_size(@results) > 0 do %> <%= Enum.count(@results, fn {_, r} -> r.status == :pass end) %> / <%= map_size(@results) %> passed <% end %>
<%= for test <- @test_cases do %> <% result = Map.get(@results, test.id) %> <% is_running = @running_id == test.id %>
<%= result_icon(result, is_running) %>
<%= test.name %>
<%= test.description %>
<%= if result && result.status in [:fail, :error] do %>
<%= result.reason %>
<% end %>
<% end %>
""" end defp row_state_class(nil, true), do: "state-running" defp row_state_class(nil, false), do: "state-idle" defp row_state_class(%{status: :pass}, _), do: "state-pass" defp row_state_class(%{status: :fail}, _), do: "state-fail" defp row_state_class(%{status: :error}, _), do: "state-error" defp result_icon(nil, true), do: "›" defp result_icon(nil, false), do: "·" defp result_icon(%{status: :pass}, _), do: "✓" defp result_icon(%{status: :fail}, _), do: "✗" defp result_icon(%{status: :error}, _), do: "!" end