defmodule Mix.Tasks.Exsql.Slt do @shortdoc "Runs sqllogictest corpus files against ExSQL" @moduledoc """ Runs SQLite's sqllogictest conformance corpus against ExSQL and reports a score. mix exsql.slt path/to/select1.test path/to/random/expr mix exsql.slt --sample 25 /Users/you/src/sqllogictest/test mix exsql.slt --limit-files 10 --limit-records 500 --timings test Directories are searched recursively for `*.test` files. `--sample N` takes every Nth file of each directory, for a quick representative pass over the full 600-file corpus. `--limit-files N` caps the expanded file list, `--limit-records N` caps each file's parsed records, `--timeout-ms N` aborts a slow file, `--slow-ms N` labels slow files, `--timings` prints per-file elapsed time, and `--verbose` prints failure details. """ use Mix.Task @impl true def run(argv) do {opts, paths} = OptionParser.parse!(argv, strict: [ sample: :integer, verbose: :boolean, limit_files: :integer, limit_records: :integer, timeout_ms: :integer, slow_ms: :integer, timings: :boolean ] ) opts = validate_opts(opts) if paths == [] do Mix.raise( "usage: mix exsql.slt [--sample N] [--limit-files N] [--limit-records N] " <> "[--timeout-ms N] [--slow-ms N] [--timings] [--verbose] ..." ) end Mix.Task.run("compile") files = paths |> Enum.flat_map(&expand/1) |> sample(opts[:sample]) |> limit_files(opts[:limit_files]) started = System.monotonic_time(:millisecond) totals = Enum.reduce(files, empty_totals(), fn file, totals -> case timed_run_file(file, opts) do {:ok, stats, elapsed} -> print_stats(file, stats, elapsed, opts) %{ ok: totals.ok + stats.ok, fail: totals.fail + stats.fail, skip: totals.skip + stats.skip, aborted: totals.aborted + if(stats.aborted, do: 1, else: 0), timed_out: totals.timed_out, files: totals.files + 1 } {:timeout, elapsed} -> Mix.shell().info( "#{Path.relative_to_cwd(file)}: TIMEOUT after #{format_elapsed(elapsed)}" ) %{totals | timed_out: totals.timed_out + 1, files: totals.files + 1} end end) elapsed = System.monotonic_time(:millisecond) - started ran = totals.ok + totals.fail score = if ran > 0, do: Float.round(totals.ok * 100 / ran, 2), else: 0.0 Mix.shell().info(""" #{totals.files} files, #{ran} records run in #{format_elapsed(elapsed)} \ (#{totals.skip} skipped, #{totals.aborted} files aborted, #{totals.timed_out} files timed out) ok: #{totals.ok} fail: #{totals.fail} score: #{score}% """) end defp empty_totals, do: %{ok: 0, fail: 0, skip: 0, aborted: 0, timed_out: 0, files: 0} defp validate_opts(opts) do validate_positive(opts, :sample) Enum.each([:limit_files, :limit_records], &validate_non_negative(opts, &1)) Enum.each([:timeout_ms, :slow_ms], &validate_positive(opts, &1)) opts end defp validate_non_negative(opts, key) do value = opts[key] if is_integer(value) and value < 0 do Mix.raise("--#{switch_name(key)} must be greater than or equal to 0") end end defp validate_positive(opts, key) do value = opts[key] if is_integer(value) and value <= 0 do Mix.raise("--#{switch_name(key)} must be greater than 0") end end defp switch_name(key), do: key |> Atom.to_string() |> String.replace("_", "-") defp timed_run_file(file, opts) do started = System.monotonic_time(:millisecond) fun = fn -> ExSQL.SqlLogicTest.run_file(file, max_records: opts[:limit_records]) end result = case opts[:timeout_ms] do nil -> {:ok, fun.()} timeout -> task = Task.async(fun) case Task.yield(task, timeout) || Task.shutdown(task, :brutal_kill) do {:ok, stats} -> {:ok, stats} nil -> :timeout end end elapsed = System.monotonic_time(:millisecond) - started case result do {:ok, stats} -> {:ok, stats, elapsed} :timeout -> {:timeout, elapsed} end end defp print_stats(file, stats, elapsed, opts) do status = if stats.aborted, do: " ABORTED", else: "" timing = timing_suffix(elapsed, opts) Mix.shell().info( "#{Path.relative_to_cwd(file)}: #{stats.ok} ok, #{stats.fail} fail, " <> "#{stats.skip} skip#{status}#{timing}" ) if opts[:verbose] do Enum.each(stats.failures, fn failure -> Mix.shell().info(" line #{failure.line}: #{failure.detail}") Mix.shell().info(" #{String.replace(failure.sql, "\n", " ")}") end) end end defp timing_suffix(elapsed, opts) do slow? = opts[:slow_ms] != nil and elapsed >= opts[:slow_ms] timings? = opts[:timings] == true cond do timings? and slow? -> " (#{format_elapsed(elapsed)}, SLOW)" timings? -> " (#{format_elapsed(elapsed)})" slow? -> " SLOW #{format_elapsed(elapsed)}" true -> "" end end defp expand(path) do cond do File.dir?(path) -> path |> Path.join("**/*.test") |> Path.wildcard() |> Enum.sort() File.exists?(path) -> [path] true -> Mix.raise("no such file or directory: #{path}") end end defp sample(files, nil), do: files defp sample(files, n), do: files |> Enum.take_every(n) defp limit_files(files, nil), do: files defp limit_files(files, n), do: Enum.take(files, n) defp format_elapsed(ms) when ms < 1000, do: "#{ms}ms" defp format_elapsed(ms), do: "#{Float.round(ms / 1000, 2)}s" end