defmodule Mix.Tasks.Firebird.Target.Bench do @moduledoc """ Run benchmarks comparing compiled WASM against native BEAM execution. ## Usage # Run the full benchmark suite (optimized by default) mix firebird.target.bench # Run with custom iterations mix firebird.target.bench --iterations 500 # Compare unoptimized vs optimized vs BEAM side-by-side mix firebird.target.bench --compare # Run without optimizations to see raw compiler output mix firebird.target.bench --no-optimize --no-tco --no-inline # Filter benchmarks by name mix firebird.target.bench --filter fibonacci # Output as JSON / Markdown / CSV mix firebird.target.bench --json mix firebird.target.bench --markdown mix firebird.target.bench --csv ## What's Benchmarked The suite tests compiled WASM performance across categories: - **Arithmetic** - add, multiply operations - **Pattern matching** - abs, max with branching - **Light recursion** - fibonacci, factorial with small inputs - **Tail-recursive** - accumulator variants that benefit from TCO - **Deep recursion** - sum_to_acc, collatz_acc with large inputs (TCO showcase) - **Math operations** - gcd, power Each benchmark runs the function through both the WASM runtime (via Wasmex) and the native BEAM VM, then compares timing. ## Comparison Mode (--compare) Compiles each benchmark twice — once without optimizations and once with all optimizations (TCO, inlining, constant folding) — then shows a three-way comparison table demonstrating the concrete impact of each compiler optimization pass. """ use Mix.Task @shortdoc "Benchmark WASM target vs native BEAM execution" @switches [ iterations: :integer, warmup: :integer, filter: :string, json: :boolean, markdown: :boolean, csv: :boolean, verbose: :boolean, compare: :boolean, optimize: :boolean, tco: :boolean, inline: :boolean, licm: :boolean ] @aliases [ n: :iterations, f: :filter, v: :verbose, c: :compare ] @impl Mix.Task def run(args) do {opts, _rest, _invalid} = OptionParser.parse(args, switches: @switches, aliases: @aliases) iterations = Keyword.get(opts, :iterations, 100) warmup = Keyword.get(opts, :warmup, 10) filter = Keyword.get(opts, :filter) compare_mode = Keyword.get(opts, :compare, false) Mix.shell().info("🔥 Firebird WASM Target — Benchmark Suite") Mix.shell().info(" Iterations: #{iterations}, Warmup: #{warmup}") if filter do Mix.shell().info(" Filter: #{filter}") end if compare_mode do Mix.shell().info(" Mode: Comparison (unoptimized vs optimized vs BEAM)") else opt_flags = [] opt_flags = if Keyword.get(opts, :optimize, true), do: ["opt" | opt_flags], else: opt_flags opt_flags = if Keyword.get(opts, :tco, true), do: ["tco" | opt_flags], else: opt_flags opt_flags = if Keyword.get(opts, :inline, true), do: ["inline" | opt_flags], else: opt_flags opt_flags = if Keyword.get(opts, :licm, true), do: ["licm" | opt_flags], else: opt_flags Mix.shell().info(" Optimizations: #{Enum.join(Enum.reverse(opt_flags), ", ")}") end Mix.shell().info("") if compare_mode do run_comparison_mode(opts, iterations, warmup, filter) else run_standard_mode(opts, iterations, warmup, filter) end :ok end defp run_standard_mode(opts, iterations, warmup, filter) do results = Firebird.Target.Benchmark.run_suite( iterations: iterations, warmup: warmup, filter: filter, optimize: Keyword.get(opts, :optimize, true), tco: Keyword.get(opts, :tco, true), inline: Keyword.get(opts, :inline, true), licm: Keyword.get(opts, :licm, true) ) cond do Keyword.get(opts, :json, false) -> Mix.shell().info(Firebird.Target.Benchmark.format_json(results)) Keyword.get(opts, :markdown, false) -> Mix.shell().info(Firebird.Target.Benchmark.format_markdown(results)) Keyword.get(opts, :csv, false) -> Mix.shell().info(Firebird.Target.Benchmark.format_csv(results)) true -> Mix.shell().info(Firebird.Target.Benchmark.format_results(results)) end if Keyword.get(opts, :verbose, false) do print_verbose(results) end end defp run_comparison_mode(opts, iterations, warmup, filter) do results = Firebird.Target.Benchmark.run_comparison_suite( iterations: iterations, warmup: warmup, filter: filter ) cond do Keyword.get(opts, :markdown, false) -> Mix.shell().info(Firebird.Target.Benchmark.format_comparison_markdown(results)) true -> Mix.shell().info(Firebird.Target.Benchmark.format_comparison(results)) end if Keyword.get(opts, :verbose, false) do print_comparison_verbose(results) end end defp print_verbose(results) do Mix.shell().info("") Mix.shell().info("Detailed Results:") Enum.each(results, fn r -> Mix.shell().info("") Mix.shell().info(" #{r.name}:") Mix.shell().info( " WASM: avg=#{Float.round(r.wasm.avg, 1)}μs min=#{r.wasm.min}μs max=#{r.wasm.max}μs p95=#{Float.round(r.wasm.p95, 1)}μs" ) Mix.shell().info( " BEAM: avg=#{Float.round(r.beam.avg, 1)}μs min=#{r.beam.min}μs max=#{r.beam.max}μs p95=#{Float.round(r.beam.p95, 1)}μs" ) Mix.shell().info(" Speedup: #{r.speedup}x, Correct: #{r.correct}") Mix.shell().info(" WASM binary: #{Map.get(r, :wasm_size, 0)} bytes") Mix.shell().info(" Optimizations: #{Map.get(r, :optimizations, []) |> Enum.join(", ")}") end) end defp print_comparison_verbose(results) do Mix.shell().info("") Mix.shell().info("Detailed Comparison:") Enum.each(results, fn r -> Mix.shell().info("") Mix.shell().info(" #{r.name}:") Mix.shell().info( " Raw WASM: avg=#{Float.round(r.unoptimized.avg, 1)}μs p95=#{Float.round(r.unoptimized.p95, 1)}μs" ) Mix.shell().info( " Opt WASM: avg=#{Float.round(r.optimized.avg, 1)}μs p95=#{Float.round(r.optimized.p95, 1)}μs" ) Mix.shell().info( " BEAM: avg=#{Float.round(r.beam.avg, 1)}μs p95=#{Float.round(r.beam.p95, 1)}μs" ) Mix.shell().info(" Opt gain: #{r.opt_vs_raw}x faster than raw WASM") Mix.shell().info(" vs BEAM: #{r.opt_vs_beam}x") Mix.shell().info( " Size: #{r.unopt_wasm_size}B → #{r.opt_wasm_size}B (#{r.size_reduction}B saved)" ) end) end end