defmodule Mix.Tasks.Livebook.Test do @moduledoc """ Runs Livebook notebooks as tests. This task discovers `.livemd` files, converts them to Elixir scripts, executes them, and reports results, similar to `mix test` but for Livebook notebooks. ## Usage mix livebook.test mix livebook.test --path livebooks/**/*.livemd mix livebook.test --exclude '**/broken/**/*.livemd' mix livebook.test --mode local mix livebook.test --mode remote mix livebook.test --timeout 120 mix livebook.test --verbose ## Options - `--path` - glob pattern for notebook discovery (can be repeated) - `--exclude` - glob pattern to exclude from discovery (can be repeated) - `--mode` - dependency mode: `local` or `remote` (default: from config) - `--timeout` - per-notebook timeout in seconds (default: from config) - `--verbose` - enable verbose output with per-notebook details ## Exit codes - `0` - all notebooks passed - `1` - one or more notebooks failed (or preflight check failed) - `2` - no notebooks discovered ## Preflight checks Before discovery, the task verifies Elixir, OTP, and Livebook compatibility. Unsupported environments fail fast with troubleshooting guidance. ## CI/CD integration Add to your CI workflow: - name: Test Livebooks run: mix livebook.test For local dependency testing: - name: Test Livebooks (local deps) run: mix livebook.test --mode local ## Configuration See `LivebookTest.Config` for application environment configuration. """ use Mix.Task @shortdoc "Runs Livebook notebooks as tests" @switches [ path: :keep, exclude: :keep, mode: :string, timeout: :string, verbose: :boolean ] @impl Mix.Task def run(args) do {opts, _args, errors} = OptionParser.parse(args, switches: @switches) if errors != [] do errors |> Enum.each(fn {key, value} -> Mix.shell().error("Unknown option: #{key} #{inspect(value)}") end) Mix.raise("Invalid options provided. Run \"mix help livebook.test\" for usage information") end Mix.Task.run("app.start", []) LivebookTest.Preflight.check!() config = build_config_from_opts(opts) report = LivebookTest.run_with_config(config) output = if config.verbose do LivebookTest.Report.format_verbose(report) else LivebookTest.Report.format(report) end Mix.shell().info(output) exit_code = LivebookTest.Report.exit_code(report) if exit_code != 0 do Mix.raise("Livebook tests failed") end end @doc false def build_config_from_opts(opts) do paths = parse_repeated_opt(opts, :path) excludes = parse_repeated_opt(opts, :exclude) overrides = [] |> put_paths(paths) |> put_excludes(paths, excludes) |> put_mode(opts) |> put_timeout(opts) |> put_verbose(opts) LivebookTest.Config.resolve(overrides) end defp parse_repeated_opt(opts, key) do case Keyword.get_values(opts, key) do [] -> nil values -> values end end defp put_paths(overrides, nil), do: overrides defp put_paths(overrides, paths), do: Keyword.put(overrides, :paths, paths) defp put_excludes(overrides, _paths, nil), do: overrides defp put_excludes(overrides, _paths, excludes), do: Keyword.put(overrides, :exclude, excludes) defp put_mode(overrides, opts) do case Keyword.get(opts, :mode) do nil -> overrides "local" -> Keyword.put(overrides, :dependency_mode, :local) "remote" -> Keyword.put(overrides, :dependency_mode, :remote) mode when is_atom(mode) -> Keyword.put(overrides, :dependency_mode, mode) other -> Mix.raise("Invalid --mode value: #{inspect(other)}. Expected: local or remote") end end defp put_timeout(overrides, opts) do case Keyword.get(opts, :timeout) do nil -> overrides timeout when is_binary(timeout) -> case Integer.parse(timeout) do {value, ""} -> Keyword.put(overrides, :timeout, value * 1000) _ -> Mix.raise( "Invalid --timeout value: #{inspect(timeout)}. Expected a number in seconds" ) end timeout when is_integer(timeout) -> Keyword.put(overrides, :timeout, timeout) end end defp put_verbose(overrides, opts) do case Keyword.get(opts, :verbose) do nil -> overrides true -> Keyword.put(overrides, :verbose, true) false -> overrides end end end