ReqLLM.Compatibility.ScenarioCatalog (ReqLLM v1.19.0)

View Source

Compiled catalog of compatibility scenarios used by ReqLLM's test tooling.

The catalog centralizes capability membership, operation metadata, and provider-specific test routing while preserving the established scenario and command-line selection order.

Summary

Types

capability()
operation()
route()
scenario()

Functions

baseline_scenarios(operation)

Returns fixture-replay baseline scenarios required for an operation support tier.

capabilities()

Returns the catalog's capability definitions in declaration order.

fetch_scenario(id)

Returns metadata for a known scenario.

fetch_scenario!(id)

Returns metadata for a known scenario or raises for an unknown ID.

fixture!(id, index \\ 0)

Returns one fixture name from a scenario's ordered fixture contract.

fixtures(id)

Returns the ordered fixture contract for a known scenario.

model_compat_scenarios_for_capability(capability, provider)

Returns ordered per-model compatibility scenarios for a capability and provider.

operation_defaults(operation, selected_capability)

Returns the legacy default scenarios for a selected operation capability.

operation_test_file(provider, operation)

Returns the established coverage test file for a provider operation.

operations()

Returns operation routing metadata in canonical operation order.

routes()

Returns all provider-specific test routes.

scenario_test_file(provider, scenario)

Returns the focused test file for a provider and scenario, if configured.

scenarios()

Returns the catalog's scenario definitions in selection order.

scenarios_for_capability(capability)

Returns ordered scenario IDs for a capability.

test_file(provider, operation, scenario \\ nil)

Returns the selected test file for a provider, operation, and optional scenario.

Types

capability()

@type capability() :: %{id: binary(), operation: atom()}

operation()

@type operation() :: %{id: atom(), test_file: binary() | :provider_directory}

route()

@type route() :: %{provider: atom(), scenario: binary(), test_file: binary()}

scenario()

@type scenario() :: %{
  id: binary(),
  capability: binary(),
  operation: atom(),
  input_modalities: [atom()],
  output_modalities: [atom()],
  requirements: [atom()],
  transports: [atom()],
  fixtures: [binary()],
  proof: atom(),
  applicability: atom(),
  providers: :all | [atom()]
}

Functions

baseline_scenarios(operation)

@spec baseline_scenarios(atom()) :: [binary()]

Returns fixture-replay baseline scenarios required for an operation support tier.

capabilities()

@spec capabilities() :: [capability()]

Returns the catalog's capability definitions in declaration order.

fetch_scenario(id)

@spec fetch_scenario(atom() | binary()) :: {:ok, scenario()} | :error

Returns metadata for a known scenario.

fetch_scenario!(id)

@spec fetch_scenario!(atom() | binary()) :: scenario()

Returns metadata for a known scenario or raises for an unknown ID.

fixture!(id, index \\ 0)

@spec fixture!(atom() | binary(), non_neg_integer()) :: binary()

Returns one fixture name from a scenario's ordered fixture contract.

fixtures(id)

@spec fixtures(atom() | binary()) :: [binary()]

Returns the ordered fixture contract for a known scenario.

model_compat_scenarios_for_capability(capability, provider)

@spec model_compat_scenarios_for_capability(binary(), atom()) ::
  {:ok, [binary()]} | :error

Returns ordered per-model compatibility scenarios for a capability and provider.

operation_defaults(operation, selected_capability)

@spec operation_defaults(atom(), binary() | nil) :: [binary()]

Returns the legacy default scenarios for a selected operation capability.

operation_test_file(provider, operation)

@spec operation_test_file(atom(), atom()) :: binary()

Returns the established coverage test file for a provider operation.

operations()

@spec operations() :: [operation()]

Returns operation routing metadata in canonical operation order.

routes()

@spec routes() :: [route()]

Returns all provider-specific test routes.

scenario_test_file(provider, scenario)

@spec scenario_test_file(atom(), atom() | binary() | nil) :: binary() | nil

Returns the focused test file for a provider and scenario, if configured.

scenarios()

@spec scenarios() :: [scenario()]

Returns the catalog's scenario definitions in selection order.

scenarios_for_capability(capability)

@spec scenarios_for_capability(binary()) :: {:ok, [binary()]} | :error

Returns ordered scenario IDs for a capability.

test_file(provider, operation, scenario \\ nil)

@spec test_file(atom(), atom(), atom() | binary() | nil) :: binary()

Returns the selected test file for a provider, operation, and optional scenario.