Import, Eval, And Trace

Section

This notebook shows a data-authored agent imported from YAML, explicit registries for executable references, deterministic evals, and trace recording.

Mix.install(
  [
    {:kino, "~> 0.14"},
    {:jidoka, path: Path.expand("..", __DIR__)}
  ],
  consolidate_protocols: false
)

Jidoka.Kino.setup_notebook(model: "test:notebook-model", check_provider?: false)

Register Executable References

defmodule NotebookEcho do
  use Jidoka.Action,
    name: "echo_value",
    description: "Echoes a deterministic value.",
    schema:
      Zoi.object(%{
        value: Zoi.string()
      })

  @impl true
  def run(params, _context) do
    value = Map.get(params, :value) || Map.get(params, "value")
    {:ok, %{value: value, source: "notebook"}}
  end
end

Import A YAML Agent

yaml = """
agent:
  id: imported_eval_agent
  model:
    provider: test
    id: notebook-model
  instructions: Use echo_value before answering.
  runtime_defaults:
    max_model_turns: 4
tools:
  actions:
    - echo
controls:
  max_turns: 4
"""

{:ok, spec} =
  Jidoka.import(yaml,
    actions: %{"echo" => NotebookEcho}
  )

{:ok, _inspection} = Jidoka.Kino.debug_agent(spec)
{:ok, _diagram} = Jidoka.Kino.agent_diagram(spec)

Run And Trace The Turn

fake_llm = fn _intent, journal ->
  llm_calls =
    journal.results
    |> Map.values()
    |> Enum.count(&(&1.kind == :llm))

  case llm_calls do
    0 -> {:ok, %{type: :operation, name: "echo_value", arguments: %{"value" => "Ada"}}}
    1 -> {:ok, %{type: :final, content: "Echo returned Ada from the notebook."}}
  end
end

{:ok, result} =
  Jidoka.turn(spec, "Echo Ada.",
    llm: fake_llm,
    operations: Jidoka.Runtime.JidoActions.operations([NotebookEcho])
  )

{:ok, trace_pid} = Jidoka.Trace.Sink.InMemory.start_link()

:ok =
  Jidoka.Trace.record(result.events, {Jidoka.Trace.Sink.InMemory, pid: trace_pid},
    policy: [redact_keys: [], omit_keys: []]
  )

Jidoka.Trace.Sink.InMemory.list(trace_pid)
|> Enum.map(& &1.event)

{:ok, _timeline} = Jidoka.Kino.timeline(result)
{:ok, _graph} = Jidoka.Kino.call_graph(result)

Turn The Flow Into An Eval

{:ok, eval_run} =
  Jidoka.Eval.run_case(
    [
      id: "imported_echo_eval",
      agent: spec,
      input: "Echo Ada.",
      assertions: %{
        contains: "Ada",
        operation_called: "echo_value"
      }
    ],
    llm: fake_llm,
    operations: Jidoka.Runtime.JidoActions.operations([NotebookEcho])
  )

Jidoka.inspect(eval_run)