defmodule Mix.Tasks.Benchmark do @moduledoc """ Runs performance benchmarks for Raxol. ## Usage mix benchmark [options] [suite_names] ## Options * `--all` - Run all benchmark suites * `--suite` - Run specific suite(s) (can be used multiple times) * `--compare` - Compare with baseline * `--save-baseline` - Save results as new baseline * `--profile` - Enable profiling mode * `--format` - Output format: console, html, json, markdown (default: console) * `--output` - Output directory (default: bench/output) * `--time` - Duration to run each benchmark in seconds (default: 5) * `--warmup` - Warmup time in seconds (default: 2) * `--memory` - Include memory measurements * `--only` - Run only benchmarks matching pattern * `--except` - Exclude benchmarks matching pattern ## Available Suites * `terminal` - Terminal emulation benchmarks * `rendering` - UI rendering pipeline benchmarks * `buffer` - Buffer operations benchmarks * `plugin` - Plugin system benchmarks * `component` - UI component benchmarks * `security` - Security operations benchmarks ## Examples # Run all benchmarks mix benchmark --all # Run specific suites mix benchmark --suite terminal --suite buffer # Run with comparison to baseline mix benchmark --all --compare # Save new baseline mix benchmark --all --save-baseline # Run in profile mode mix benchmark --suite terminal --profile # Generate HTML report mix benchmark --all --format html --output bench/reports # Run only specific benchmarks mix benchmark --suite terminal --only "ANSI.*" # Quick benchmark (reduced time) mix benchmark --suite terminal --time 1 --warmup 0.5 """ use Mix.Task alias Raxol.Benchmark.{Reporter, Storage, Analyzer} @all_suites ~w(terminal rendering buffer plugin component security) @shortdoc "Run performance benchmarks" @impl Mix.Task def run(args) do {opts, _, _} = parse_args(args) if opts[:help] do Mix.shell().info(@moduledoc) System.halt(0) end execute_benchmarks(opts) end defp parse_args(args) do OptionParser.parse(args, switches: [ all: :boolean, suite: :keep, compare: :boolean, save_baseline: :boolean, profile: :boolean, format: :string, output: :string, time: :float, warmup: :float, memory: :boolean, only: :string, except: :string, quick: :boolean, help: :boolean ], aliases: [ a: :all, s: :suite, c: :compare, p: :profile, f: :format, o: :output, t: :time, w: :warmup, m: :memory, h: :help ] ) end defp execute_benchmarks(opts) do Mix.Task.run("app.start") benchmark_opts = configure_options(opts) suites = determine_suites(opts) validate_suites(suites) log_execution_info(suites, benchmark_opts) results = run_suites(suites, benchmark_opts, opts) process_results(results, opts) Mix.shell().info("\nBenchmark completed successfully!") end defp validate_suites(suites) do if Enum.empty?(suites) do Mix.shell().error( "No benchmark suites specified. Use --all or --suite " ) Mix.shell().info("Available suites: #{Enum.join(@all_suites, ", ")}") System.halt(1) end end defp log_execution_info(suites, benchmark_opts) do Mix.shell().info("Running benchmark suites: #{Enum.join(suites, ", ")}") Mix.shell().info("Configuration: #{inspect(benchmark_opts)}") end defp run_suites(suites, benchmark_opts, opts) do if opts[:profile] do run_profiling(suites, benchmark_opts) else run_benchmarks(suites, benchmark_opts) end end defp configure_options(opts) do base_opts = %{ time: opts[:time] || 5, warmup: opts[:warmup] || 2, memory_time: if(opts[:memory], do: 2, else: 0), parallel: 1 } # Quick mode for faster iteration base_opts = if opts[:quick] do %{base_opts | time: 1, warmup: 0.5} else base_opts end # Add filters base_opts = if opts[:only] do Map.put(base_opts, :only, compile_filter(opts[:only])) else base_opts end base_opts = if opts[:except] do Map.put(base_opts, :except, compile_filter(opts[:except])) else base_opts end base_opts end defp determine_suites(opts) do cond do opts[:all] -> @all_suites opts[:suite] -> # Handle multiple --suite options opts |> Keyword.get_values(:suite) |> Enum.flat_map(&String.split(&1, ",")) |> Enum.map(&String.trim/1) |> Enum.filter(&(&1 in @all_suites)) |> Enum.uniq() true -> [] end end @dialyzer {:no_return, run_benchmarks: 2} defp run_benchmarks(suites, opts) do Enum.map(suites, fn suite -> Mix.shell().info("\n▶ Running #{suite} benchmarks...") suite_module = get_suite_module(suite) suite_data = get_suite_data(suite_module, suite) results = suite_module.run_suite(suite_data, opts) %{ suite_name: suite, results: results, duration: results.duration, timestamp: DateTime.utc_now() } end) end @dialyzer {:no_return, get_suite_data: 2} defp get_suite_data(runner_module, suite) do case suite do "terminal" -> runner_module.terminal_benchmarks() "rendering" -> runner_module.rendering_benchmarks() "buffer" -> runner_module.buffer_benchmarks() "plugin" -> runner_module.plugin_benchmarks() "component" -> runner_module.component_benchmarks() "security" -> runner_module.security_benchmarks() _ -> raise "Unknown suite: #{suite}" end end @dialyzer {:no_return, run_profiling: 2} defp run_profiling(suites, opts) do Mix.shell().info("\n🔍 Running in profiling mode...") Enum.map(suites, fn suite -> Mix.shell().info("\n▶ Profiling #{suite} suite...") suite_module = get_suite_module(suite) suite_data = get_suite_data(suite_module, suite) profile_opts = Map.put(opts, :profile_after, true) results = suite_module.run_suite(suite_data, profile_opts) # Analyze profile data profile_report = Analyzer.analyze_profile(suite, results.results) print_profile_report(profile_report) %{ suite_name: suite, results: results.results, profile: profile_report, duration: results.duration, timestamp: DateTime.utc_now() } end) end @dialyzer {:no_return, process_results: 2} defp process_results(results, opts) do # Compare with baseline if requested if opts[:compare] do Mix.shell().info("\n📊 Comparing with baseline...") regressions = Analyzer.check_regressions(results) if Enum.any?(regressions) do Mix.shell().warning("\n⚠️ Performance regressions detected!") Analyzer.report_regressions(regressions) else Mix.shell().info("✅ No regressions detected") end end # Save baseline if requested if opts[:save_baseline] do Mix.shell().info("\n💾 Saving new baseline...") Enum.each(results, fn result -> Storage.save_baseline(result.suite_name, result.results) end) Mix.shell().info("✅ Baseline saved") end # Generate reports format = opts[:format] || "console" output_path = opts[:output] || "bench/output" Mix.shell().info("\n📄 Generating #{format} report...") Reporter.generate_comprehensive_report(results, format: String.to_atom(format), output_path: output_path ) # Always save results for historical tracking Enum.each(results, fn result -> Storage.save_results(result.suite_name, result.results, result.duration) end) end @dialyzer {:no_return, get_suite_module: 1} defp get_suite_module(_suite) do Raxol.Benchmark.Runner end @dialyzer {:no_return, compile_filter: 1} defp compile_filter(pattern) do case Regex.compile(pattern) do {:ok, regex} -> regex {:error, _} -> Mix.shell().error("Invalid regex pattern: #{pattern}") System.halt(1) end end @dialyzer {:no_return, print_profile_report: 1} defp print_profile_report(report) do IO.puts(""" Profile Analysis: #{report.name} ===================================== Total Time: #{format_time(report.total_time)} Memory Usage: #{format_memory(report.memory_usage)} Hot Spots: #{format_hot_spots(report.hot_spots)} Optimization Opportunities: #{format_optimizations(report.optimization_opportunities)} """) end @dialyzer {:no_return, format_time: 1} defp format_time(microseconds) when is_number(microseconds) do cond do microseconds < 1_000 -> "#{microseconds}μs" microseconds < 1_000_000 -> "#{Float.round(microseconds / 1_000, 2)}ms" true -> "#{Float.round(microseconds / 1_000_000, 2)}s" end end defp format_time(_), do: "N/A" @dialyzer {:no_return, format_memory: 1} defp format_memory(bytes) when is_number(bytes) do cond do bytes < 1_024 -> "#{bytes}B" bytes < 1_048_576 -> "#{Float.round(bytes / 1_024, 2)}KB" bytes < 1_073_741_824 -> "#{Float.round(bytes / 1_048_576, 2)}MB" true -> "#{Float.round(bytes / 1_073_741_824, 2)}GB" end end defp format_memory(_), do: "N/A" defp format_hot_spots([]), do: " None identified" @dialyzer {:no_return, format_hot_spots: 1} defp format_hot_spots(spots) do Enum.map_join(spots, "\n", fn spot -> " • #{spot}" end) end @dialyzer {:no_return, format_optimizations: 1} defp format_optimizations([]), do: " None identified" defp format_optimizations(opts) do Enum.map_join(opts, "\n", fn opt -> " • #{opt}" end) end end