defmodule LivebookTest do @moduledoc """ Test your Livebook notebooks - "mix test for Livebooks". LivebookTest discovers `.livemd` notebooks, converts them to executable Elixir scripts, runs them, and reports failures. It supports local dependency overrides so standalone notebooks can be tested against the current repository checkout. ## Quick start Add to your Mix project: def deps do [{:livebook_test, "~> 0.1", only: [:dev, :test], runtime: false}] end Run: mix livebook.test ## Pipeline The library processes notebooks through a pipeline: 1. **Discovery** - find `.livemd` files via glob patterns 2. **Export** - convert notebooks to `.exs` scripts 3. **Patch** - optionally rewrite Mix.install deps to local paths 4. **Run** - execute each script as a subprocess 5. **Report** - summarize results and return CI exit codes ## Local dependency testing The key feature: notebooks that use `Mix.install` can be automatically patched to use your local checkout instead of published Hex packages. # In config/config.exs config :livebook_test, dependency_mode: :local, local_deps: [ my_lib: "." ] ## Programmatic API LivebookTest.run() LivebookTest.run(paths: ["examples/**/*.livemd"], mode: :local, timeout: 120_000) ## Configuration See `LivebookTest.Config` for full configuration options. """ @typedoc "Options accepted by `run/1`" @type run_option :: {:paths, [String.t()]} | {:exclude, [String.t()]} | {:mode, LivebookTest.Config.dependency_mode()} | {:timeout, non_neg_integer()} | {:local_deps, LivebookTest.Config.local_deps()} | {:verbose, boolean()} | {:runner, module()} | {:preflight, boolean()} @typedoc "Result of a complete test run" @type run_result :: {LivebookTest.Config.t(), LivebookTest.Report.t()} @doc """ Discovers, exports, patches, runs, and reports on Livebook notebooks. This is the primary entry point. It orchestrates the full pipeline and returns a tuple with the resolved config and the report. ## Options - `:paths` - list of glob patterns (default from config) - `:exclude` - list of glob patterns to exclude from discovery - `:mode` - `:remote` or `:local` dependency mode - `:timeout` - per-notebook timeout in milliseconds - `:local_deps` - dependency name → path mapping - `:verbose` - enable verbose output - `:runner` - runner module (default `LivebookTest.Runner`; for test injection) - `:preflight` - run environment checks before discovery (default `true`) ## Examples iex> {config, report} = LivebookTest.run(paths: ["examples/**/*.livemd"]) iex> is_struct(config, LivebookTest.Config) and is_struct(report, LivebookTest.Report) true """ @spec run([run_option()]) :: run_result() def run(opts \\ []) do {run_opts, config_opts} = split_run_opts(opts) overrides = build_overrides(config_opts) config = LivebookTest.Config.resolve(overrides) {config, run_with_config(config, run_opts)} end @doc """ Runs the full pipeline using a pre-resolved config. Useful when you need fine-grained control over configuration before running. ## Options - `:runner` - runner module (default `LivebookTest.Runner`) - `:preflight` - run environment checks (default `true`; set `false` in tests) ## Examples iex> config = LivebookTest.Config.resolve(paths: ["examples/**/*.livemd"]) iex> report = LivebookTest.run_with_config(config) iex> is_struct(report, LivebookTest.Report) true """ @spec run_with_config(LivebookTest.Config.t(), keyword()) :: LivebookTest.Report.t() def run_with_config(%LivebookTest.Config{} = config, opts \\ []) do case Keyword.get(opts, :preflight, true) do true -> LivebookTest.Preflight.check!() false -> :ok end runner_mod = Keyword.get(opts, :runner, LivebookTest.Runner) notebooks = LivebookTest.Discovery.find(config.paths, exclude: config.exclude) log_verbose(config, "Discovered #{length(notebooks)} notebook(s)") script_pairs = export_notebooks(notebooks, config) try do results = runner_mod.run_all(script_pairs, timeout: config.timeout) LivebookTest.Report.build(results) after cleanup_scripts(script_pairs) end end defp log_verbose(config, message) do if config.verbose, do: IO.puts("[LivebookTest] #{message}") end defp export_notebooks(notebooks, config) do notebooks |> Enum.map(fn notebook_path -> case exporter_module().to_temp_file(notebook_path) do {:ok, script_path} -> {notebook_path, script_path} {:error, reason} -> log_verbose(config, "Failed to export #{notebook_path}: #{inspect(reason)}") nil end end) |> Enum.reject(&is_nil/1) |> maybe_patch_deps(config) end defp maybe_patch_deps(script_pairs, config) do if config.dependency_mode == :local do Enum.map(script_pairs, &patch_local_deps(&1, config)) else script_pairs end end defp patch_local_deps({notebook_path, script_path}, config) do log_verbose(config, "Patching #{notebook_path} with local deps") case File.read(script_path) do {:ok, script_content} -> patched = LivebookTest.DependencyPatcher.patch(script_content, :local, config.local_deps) case write_patched_script(script_path, patched) do :ok -> {notebook_path, script_path} {:error, reason} -> log_verbose( config, "Failed to write patched script #{script_path}: #{inspect(reason)}" ) {notebook_path, script_path} end {:error, reason} -> log_verbose(config, "Failed to read script #{script_path}: #{inspect(reason)}") {notebook_path, script_path} end end defp cleanup_scripts(script_pairs) do Enum.each(script_pairs, fn {_notebook_path, script_path} -> File.rm(script_path) end) end defp exporter_module do Application.get_env(:livebook_test, :exporter_module, LivebookTest.Exporter) end defp write_patched_script(script_path, patched) do writer = Application.get_env(:livebook_test, :patch_writer, &File.write/2) writer.(script_path, patched) end @doc """ Convenience function that runs the pipeline and prints the report. Returns the exit code: `0` when all notebooks pass, `1` when any fail, and `2` when no notebooks are discovered. ## Examples iex> LivebookTest.run_and_report(paths: ["examples/**/*.livemd"]) in [0, 1, 2] true """ @spec run_and_report([run_option()]) :: 0 | 1 | 2 def run_and_report(opts \\ []) do {config, report} = run(opts) output = if config.verbose do LivebookTest.Report.format_verbose(report) else LivebookTest.Report.format(report) end IO.puts(output) LivebookTest.Report.exit_code(report) end defp build_overrides(opts) do [] |> maybe_put(opts, :paths, :paths) |> maybe_put_mode(opts) |> maybe_put(opts, :timeout, :timeout) |> maybe_put(opts, :local_deps, :local_deps) |> maybe_put(opts, :exclude, :exclude) |> maybe_put(opts, :verbose, :verbose) end defp split_run_opts(opts) do run_keys = [:runner, :preflight] {Keyword.take(opts, run_keys), Keyword.drop(opts, run_keys)} end defp maybe_put(overrides, opts, source_key, dest_key) do case Keyword.get(opts, source_key) do nil -> overrides value -> Keyword.put(overrides, dest_key, value) end end defp maybe_put_mode(overrides, opts) do case Keyword.get(opts, :mode) do nil -> overrides mode -> Keyword.put(overrides, :dependency_mode, mode) end end end