defmodule Mix.Tasks.Raxol.Bench.Advanced do @moduledoc """ Advanced benchmarking commands leveraging the new world-class infrastructure. ## Commands mix raxol.bench.advanced suite # Run registered benchmark suites mix raxol.bench.advanced compare # Compare with competitors mix raxol.bench.advanced regression # Detailed regression analysis mix raxol.bench.advanced profile # Deep profiling with flame graphs mix raxol.bench.advanced continuous # Continuous monitoring mode mix raxol.bench.advanced report # Generate comprehensive reports mix raxol.bench.advanced analyze # Statistical analysis of results ## Options --suite NAME Run specific benchmark suite --tags TAG1,TAG2 Filter scenarios by tags --baseline VERSION Compare against specific baseline --threshold PERCENT Regression threshold (default: 5%) --format FORMAT Output format (text|json|html|markdown) --output FILE Save results to file --iterations N Number of iterations for statistical significance --confidence LEVEL Confidence level for tests (default: 0.95) --competitor NAME Compare against specific competitor --realtime Show results in realtime --verbose Verbose output """ use Mix.Task alias Raxol.Benchmark.{ SuiteRegistry, StatisticalAnalyzer, RegressionDetector, CompetitorSuite } @shortdoc "Advanced benchmarking with statistical analysis" @switches [ suite: :string, tags: :string, baseline: :string, threshold: :float, format: :string, output: :string, iterations: :integer, confidence: :float, competitor: :string, realtime: :boolean, verbose: :boolean, help: :boolean ] def run(args) do {opts, commands, _} = OptionParser.parse(args, switches: @switches) if opts[:help] do print_help() System.halt(0) end Mix.Task.run("app.start") # Start the suite registry {:ok, _pid} = SuiteRegistry.start_link() case commands do ["suite"] -> run_suite_benchmarks(opts) ["compare"] -> run_competitor_comparison(opts) ["regression"] -> run_regression_analysis(opts) ["profile"] -> run_profiling_session(opts) ["continuous"] -> run_continuous_monitoring(opts) ["report"] -> generate_comprehensive_report(opts) ["analyze"] -> run_statistical_analysis(opts) ["discover"] -> discover_and_run_suites(opts) _ -> Mix.shell().error("Unknown command: #{Enum.join(commands, " ")}") print_help() end end # Command implementations defp run_suite_benchmarks(opts) do Mix.shell().info("šŸš€ Running Benchmark Suites...") # Discover available suites {:ok, count} = SuiteRegistry.discover_suites() Mix.shell().info("Discovered #{count} benchmark suites") # Filter suites based on options filter = build_suite_filter(opts) # Run the suites {:ok, results} = SuiteRegistry.run_suites(filter, build_benchmark_config(opts)) # Analyze results analysis = analyze_suite_results(results) # Generate output output_results(analysis, opts) # Check for regressions if baseline specified if opts[:baseline] do check_regressions(results, opts[:baseline], opts) end end defp run_competitor_comparison(opts) do Mix.shell().info("⚔ Running Competitor Comparison...") competitor = opts[:competitor] || "all" Mix.shell().info("Comparing against: #{competitor}") # Run competitor benchmarks results = CompetitorSuite.run_benchmarks() # Generate comparison report comparison = CompetitorSuite.generate_comparison_report(results) # Calculate performance scores scores = CompetitorSuite.calculate_performance_score(results) # Output results output_competitor_comparison(comparison, scores, opts) end defp run_regression_analysis(opts) do Mix.shell().info("šŸ“Š Running Regression Analysis...") # Load current and baseline results current = load_latest_results() baseline = load_baseline_results(opts[:baseline] || "latest") if current && baseline do # Perform regression detection threshold = opts[:threshold] || 0.05 detection = RegressionDetector.detect(current, baseline, threshold: threshold) # Generate detailed report report = RegressionDetector.generate_report(detection, format: opts[:format] || :text ) # Output report output_regression_report(report, detection, opts) # Exit with error code if regressions found if length(detection.regressions) > 0 do Mix.shell().error("āŒ Performance regressions detected!") System.halt(1) else Mix.shell().info("āœ… No performance regressions detected") end else Mix.shell().error("Unable to load benchmark results for comparison") System.halt(1) end end defp run_profiling_session(opts) do Mix.shell().info("šŸ” Running Profiling Session...") suite = opts[:suite] || "all" # Start profiling :fprof.start() :fprof.trace([:start]) # Run benchmarks with profiling enabled results = run_with_profiling(suite, opts) # Stop profiling :fprof.trace([:stop]) :fprof.profile() :fprof.analyse(dest: ~c"profile_output.txt") # Generate flame graph if available if System.find_executable("flamegraph.pl") do generate_flame_graph() end # Analyze profiling results profile_analysis = analyze_profile_output() # Output results output_profiling_results(profile_analysis, results, opts) end defp run_continuous_monitoring(opts) do Mix.shell().info("šŸ“ˆ Starting Continuous Monitoring...") # Default: 1 minute interval = opts[:interval] || 60_000 # Start monitoring loop spawn(fn -> loop_continuous_monitoring(interval, opts) end) Mix.shell().info("Monitoring started. Press Ctrl+C to stop.") # Keep the process running :timer.sleep(:infinity) end defp generate_comprehensive_report(opts) do Mix.shell().info("šŸ“ Generating Comprehensive Report...") # Load all available results results = load_all_results() # Perform comprehensive analysis analysis = %{ statistical: perform_statistical_analysis(results), trends: analyze_performance_trends(results), comparison: compare_with_competitors(results), regression: detect_historical_regressions(results), recommendations: generate_recommendations(results) } # Generate report in requested format format = String.to_atom(opts[:format] || "html") report = generate_report(analysis, format) # Save report output_file = opts[:output] || "benchmark_report_#{timestamp()}.#{format}" File.write!(output_file, report) Mix.shell().info("Report generated: #{output_file}") end defp run_statistical_analysis(opts) do Mix.shell().info("šŸ“ Running Statistical Analysis...") # Load results results = load_latest_results() if results do # Extract time series data time_data = extract_time_series(results) # Perform statistical analysis analysis = StatisticalAnalyzer.analyze(time_data) # Calculate advanced metrics advanced = %{ percentiles: analysis.percentiles, outliers: analysis.outliers, distribution: analysis.distribution, confidence_intervals: analysis.confidence_intervals, statistical_tests: perform_statistical_tests(time_data, opts) } # Output analysis output_statistical_analysis(advanced, opts) else Mix.shell().error("No results available for analysis") end end defp discover_and_run_suites(opts) do Mix.shell().info("šŸ”Ž Discovering and Running Benchmark Suites...") # Discover all available suites {:ok, count} = SuiteRegistry.discover_suites() Mix.shell().info("Found #{count} benchmark suites") # List discovered suites suites = SuiteRegistry.list_suites() Enum.each(suites, fn suite -> Mix.shell().info(" • #{suite.name} (#{suite.scenario_count} scenarios)") end) # Run all discovered suites if Mix.shell().yes?("Run all discovered suites?") do {:ok, results} = SuiteRegistry.run_all(build_benchmark_config(opts)) output_results(results, opts) end end # Helper functions defp build_suite_filter(opts) do filter = [] filter = if opts[:suite] do Keyword.put(filter, :names, [opts[:suite]]) else filter end filter = if opts[:tags] do tags = String.split(opts[:tags], ",") |> Enum.map(&String.to_atom/1) Keyword.put(filter, :tags, tags) else filter end filter end defp build_benchmark_config(opts) do config = [ time: opts[:time] || 5, memory_time: opts[:memory_time] || 2, warmup: opts[:warmup] || 1, parallel: opts[:parallel] || 1 ] config = if opts[:iterations] do Keyword.put(config, :iterations, opts[:iterations]) else config end config end defp analyze_suite_results(results) do results |> Enum.map(fn {suite, data} -> {suite, %{ statistics: calculate_statistics(data), performance: analyze_performance(data), memory: analyze_memory(data) }} end) |> Enum.into(%{}) end defp calculate_statistics(data) do times = extract_execution_times(data) StatisticalAnalyzer.analyze(times) end defp analyze_performance(data) do # Performance analysis logic %{ average_time: calculate_average(data), throughput: calculate_throughput(data), latency: calculate_latency(data) } end defp analyze_memory(data) do # Memory analysis logic %{ peak_memory: calculate_peak_memory(data), average_memory: calculate_average_memory(data), allocations: count_allocations(data) } end defp output_results(results, opts) do format = String.to_atom(opts[:format] || "text") output = case format do :json -> Jason.encode!(results, pretty: true) :html -> generate_html_output(results) :markdown -> generate_markdown_output(results) _ -> generate_text_output(results) end if opts[:output] do File.write!(opts[:output], output) Mix.shell().info("Results saved to: #{opts[:output]}") else IO.puts(output) end end defp output_competitor_comparison(comparison, scores, _opts) do Mix.shell().info("\n=== Competitor Comparison Results ===\n") # Performance ranking Mix.shell().info("Performance Ranking:") Enum.each(comparison.performance_ranking, fn %{ rank: rank, test: test, time: time } -> Mix.shell().info(" #{rank}. #{test}: #{Float.round(time, 2)}μs") end) # Raxol score Mix.shell().info( "\nRaxol Performance Score: #{Float.round(scores.raxol_score, 1)}/100" ) # Relative performance Mix.shell().info("\nRelative Performance vs Competitors:") Enum.each(scores.comparison, fn {terminal, data} -> percentage = Float.round(data.relative_performance, 1) symbol = cond do percentage > 110 -> "šŸš€" percentage > 100 -> "āœ…" percentage > 90 -> "āš ļø" true -> "āŒ" end Mix.shell().info(" #{symbol} vs #{terminal}: #{percentage}%") end) end defp load_latest_results do # Load the most recent benchmark results case File.ls("bench/results") do {:ok, files} -> latest_file = files |> Enum.filter(&String.ends_with?(&1, ".benchee")) |> Enum.sort() |> List.last() if latest_file do Benchee.load(Path.join("bench/results", latest_file)) end _ -> nil end end defp load_baseline_results(version) do # Load baseline results for comparison baseline_file = "bench/baselines/#{version}.benchee" if File.exists?(baseline_file) do Benchee.load(baseline_file) else nil end end defp check_regressions(results, baseline_version, opts) do baseline = load_baseline_results(baseline_version) if baseline do threshold = opts[:threshold] || 0.05 detection = RegressionDetector.detect(results, baseline, threshold: threshold) if length(detection.regressions) > 0 do Mix.shell().error("\nāš ļø Performance Regressions Detected:") Enum.each(detection.regressions, fn reg -> Mix.shell().error( " • #{reg.scenario}: +#{Float.round(reg.percent_change, 1)}% (#{reg.severity})" ) end) end end end defp loop_continuous_monitoring(interval, opts) do # Run benchmarks {:ok, results} = SuiteRegistry.run_all(build_benchmark_config(opts)) # Analyze results analysis = analyze_suite_results(results) # Display summary display_monitoring_summary(analysis) # Check for anomalies check_for_anomalies(analysis) # Sleep and repeat :timer.sleep(interval) loop_continuous_monitoring(interval, opts) end defp display_monitoring_summary(analysis) do timestamp = DateTime.utc_now() |> DateTime.to_string() Mix.shell().info("\n[#{timestamp}] Monitoring Update:") Enum.each(analysis, fn {suite, data} -> avg_time = get_in(data, [:statistics, :mean]) || 0 p99 = get_in(data, [:statistics, :percentiles, :p99]) || 0 Mix.shell().info( " #{suite}: avg=#{Float.round(avg_time, 2)}μs, p99=#{Float.round(p99, 2)}μs" ) end) end defp check_for_anomalies(analysis) do Enum.each(analysis, fn {suite, data} -> outliers = get_in(data, [:statistics, :outliers, :outliers]) || [] if length(outliers) > 0 do Mix.shell().warn(" āš ļø #{suite}: #{length(outliers)} outliers detected") end end) end defp generate_text_output(results) do Enum.map(results, fn {suite, data} -> """ Suite: #{suite} ================ Mean: #{Float.round(get_in(data, [:statistics, :mean]) || 0, 2)}μs Median: #{Float.round(get_in(data, [:statistics, :median]) || 0, 2)}μs P95: #{Float.round(get_in(data, [:statistics, :percentiles, :p95]) || 0, 2)}μs P99: #{Float.round(get_in(data, [:statistics, :percentiles, :p99]) || 0, 2)}μs """ end) |> Enum.join("\n") end defp generate_html_output(_results) do # Generate HTML report "

Benchmark Results

" end defp generate_markdown_output(results) do # Generate Markdown report Enum.map(results, fn {suite, _data} -> "## #{suite}\n\nResults pending...\n" end) |> Enum.join("\n") end defp timestamp do DateTime.utc_now() |> DateTime.to_iso8601() |> String.replace(~r/[:\.]/, "_") end defp print_help do Mix.shell().info(@moduledoc) end # Stub implementations for complex functions defp extract_execution_times(_data), do: [1.0, 2.0, 3.0] defp calculate_average(_data), do: 2.0 defp calculate_throughput(_data), do: 1000.0 defp calculate_latency(_data), do: 1.0 defp calculate_peak_memory(_data), do: 100_000 defp calculate_average_memory(_data), do: 50_000 defp count_allocations(_data), do: 1000 defp load_all_results, do: %{} defp perform_statistical_analysis(_results), do: %{} defp analyze_performance_trends(_results), do: %{} defp compare_with_competitors(_results), do: %{} defp detect_historical_regressions(_results), do: %{} defp generate_recommendations(_results), do: [] defp generate_report(_analysis, _format), do: "" defp extract_time_series(_results), do: [] defp perform_statistical_tests(_data, _opts), do: %{} defp output_statistical_analysis(_analysis, _opts), do: :ok defp output_regression_report(_report, _detection, _opts), do: :ok defp run_with_profiling(_suite, _opts), do: %{} defp generate_flame_graph, do: :ok defp analyze_profile_output, do: %{} defp output_profiling_results(_analysis, _results, _opts), do: :ok end