ExUnit integration for LLM evaluations.

Usage

defmodule MyApp.RAGEvalTest do
  use ExUnit.Case
  use Tribunal.ExUnit

  @moduletag :eval

  test "response is faithful" do
    response = MyApp.RAG.query("What's the return policy?")

    assert response =~ "30 days"
    assert_faithful response, context: @docs, threshold: 0.8
  end
end

Dataset-Driven Tests

defmodule MyApp.RAGEvalTest do
  use ExUnit.Case
  use Tribunal.ExUnit

  @moduletag :eval

  tribunal_dataset "test/evals/datasets/questions.json",
    provider: {MyApp.RAG, :query}
end

Run with: mix test --only eval