defmodule Mix.Tasks.Raxol.Bench.Advanced do @moduledoc """ Advanced benchmarking commands leveraging the new world-class infrastructure. ## Commands mix raxol.bench.advanced suite # Run registered benchmark suites mix raxol.bench.advanced compare # Compare with competitors mix raxol.bench.advanced regression # Detailed regression analysis mix raxol.bench.advanced profile # Deep profiling with flame graphs mix raxol.bench.advanced continuous # Continuous monitoring mode mix raxol.bench.advanced report # Generate comprehensive reports mix raxol.bench.advanced analyze # Statistical analysis of results ## Options --suite NAME Run specific benchmark suite --tags TAG1,TAG2 Filter scenarios by tags --baseline VERSION Compare against specific baseline --threshold PERCENT Regression threshold (default: 5%) --format FORMAT Output format (text|json|html|markdown) --output FILE Save results to file --iterations N Number of iterations for statistical significance --confidence LEVEL Confidence level for tests (default: 0.95) --competitor NAME Compare against specific competitor --realtime Show results in realtime --verbose Verbose output """ use Mix.Task alias Raxol.Benchmark.{ CompetitorSuite, RegressionDetector, StatisticalAnalyzer, SuiteRegistry } alias Raxol.Core.Runtime.Log @shortdoc "Advanced benchmarking with statistical analysis" @switches [ suite: :string, tags: :string, baseline: :string, threshold: :float, format: :string, output: :string, iterations: :integer, confidence: :float, competitor: :string, realtime: :boolean, verbose: :boolean, help: :boolean ] @spec run(list()) :: no_return() def run(args) do {opts, commands, _} = OptionParser.parse(args, switches: @switches) handle_help_option(opts) _ = setup_benchmark_environment() execute_command(commands, opts) end defp handle_help_option(opts) do if opts[:help] do print_help() System.halt(0) end end defp setup_benchmark_environment do Mix.Task.run("app.start") # Start the suite registry {:ok, _pid} = SuiteRegistry.start_link() end defp execute_command(commands, opts) do command_handler = get_command_handler(commands) case command_handler do nil -> handle_unknown_command(commands) handler -> handler.(opts) end end defp get_command_handler(commands) do command_map = %{ ["suite"] => &run_suite_benchmarks/1, ["compare"] => &run_competitor_comparison/1, ["regression"] => &run_regression_analysis/1, ["profile"] => &run_profiling_session/1, ["continuous"] => &run_continuous_monitoring/1, ["report"] => &generate_comprehensive_report/1, ["analyze"] => &run_statistical_analysis/1, ["discover"] => &discover_and_run_suites/1 } Map.get(command_map, commands) end defp handle_unknown_command(commands) do Mix.shell().error("Unknown command: #{Enum.join(commands, " ")}") print_help() end # Command implementations defp run_suite_benchmarks(opts) do Mix.shell().info("[FAST] Running Benchmark Suites...") # Discover available suites {:ok, count} = SuiteRegistry.discover_suites() Mix.shell().info("Discovered #{count} benchmark suites") # Filter suites based on options filter = build_suite_filter(opts) # Run the suites {:ok, results} = SuiteRegistry.run_suites(filter, build_benchmark_config(opts)) # Analyze results analysis = analyze_suite_results(results) # Generate output output_results(analysis, opts) # Check for regressions if baseline specified if opts[:baseline] do check_regressions(results, opts[:baseline], opts) end end defp run_competitor_comparison(opts) do Mix.shell().info("[POWER] Running Competitor Comparison...") competitor = opts[:competitor] || "all" Mix.shell().info("Comparing against: #{competitor}") # Run competitor benchmarks results = CompetitorSuite.run_benchmarks() # Generate comparison report comparison = CompetitorSuite.generate_comparison_report(results) # Calculate performance scores scores = CompetitorSuite.calculate_performance_score(results) # Output results output_competitor_comparison(comparison, scores, opts) end defp run_regression_analysis(opts) do Mix.shell().info("[STATS] Running Regression Analysis...") # Load current and baseline results current = load_latest_results() baseline = load_baseline_results(opts[:baseline] || "latest") if current && baseline do # Perform regression detection threshold = opts[:threshold] || 0.05 detection = RegressionDetector.detect(current, baseline, threshold: threshold) # Generate detailed report report = RegressionDetector.generate_report(detection, format: opts[:format] || :text ) # Output report output_regression_report(report, detection, opts) # Exit with error code if regressions found if detection.regressions != [] do Mix.shell().error("[FAIL] Performance regressions detected!") System.halt(1) else Mix.shell().info("[OK] No performance regressions detected") end else Mix.shell().error("Unable to load benchmark results for comparison") System.halt(1) end end defp run_profiling_session(opts) do Mix.shell().info("[SEARCH] Running Profiling Session...") suite = opts[:suite] || "all" alias Raxol.Performance.FprofWrapper _ = Application.ensure_all_started(:tools) _ = FprofWrapper.start() _ = FprofWrapper.trace([:start]) results = run_with_profiling(suite, opts) _ = FprofWrapper.trace([:stop]) _ = FprofWrapper.profile() _ = FprofWrapper.analyse(dest: ~c"profile_output.txt") # Generate flame graph if available if System.find_executable("flamegraph.pl") do generate_flame_graph() end # Analyze profiling results profile_analysis = analyze_profile_output() # Output results output_profiling_results(profile_analysis, results, opts) end defp run_continuous_monitoring(opts) do Mix.shell().info("[TREND] Starting Continuous Monitoring...") # Default: 1 minute interval = opts[:interval] || 60_000 # Start monitoring loop spawn(fn -> loop_continuous_monitoring(interval, opts) end) Mix.shell().info("Monitoring started. Press Ctrl+C to stop.") # Keep the process running :timer.sleep(:infinity) end defp generate_comprehensive_report(opts) do Mix.shell().info("[DOC] Generating Comprehensive Report...") # Load all available results results = load_all_results() # Perform comprehensive analysis analysis = %{ statistical: perform_statistical_analysis(results), trends: analyze_performance_trends(results), comparison: compare_with_competitors(results), regression: detect_historical_regressions(results), recommendations: generate_recommendations(results) } # Generate report in requested format format = String.to_atom(opts[:format] || "html") report = generate_report(analysis, format) # Save report output_file = opts[:output] || "benchmark_report_#{timestamp()}.#{format}" File.write!(output_file, report) Mix.shell().info("Report generated: #{output_file}") end defp run_statistical_analysis(opts) do Mix.shell().info("[STAT] Running statistical analysis") # Load results results = load_latest_results() if results do # Extract time series data time_data = extract_time_series(results) # Perform statistical analysis analysis = StatisticalAnalyzer.analyze(time_data) # Calculate advanced metrics advanced = %{ percentiles: analysis.percentiles, outliers: analysis.outliers, distribution: analysis.distribution, confidence_intervals: analysis.confidence_intervals, statistical_tests: perform_statistical_tests(time_data, opts) } # Output analysis output_statistical_analysis(advanced, opts) else Mix.shell().error("No results available for analysis") end end defp discover_and_run_suites(opts) do Mix.shell().info("[DISC] Discovering and running benchmark suites") # Discover all available suites {:ok, count} = SuiteRegistry.discover_suites() Mix.shell().info("Found #{count} benchmark suites") # List discovered suites suites = SuiteRegistry.list_suites() Enum.each(suites, fn suite -> Mix.shell().info(" • #{suite.name} (#{suite.scenario_count} scenarios)") end) # Run all discovered suites if Mix.shell().yes?("Run all discovered suites?") do {:ok, results} = SuiteRegistry.run_all(build_benchmark_config(opts)) output_results(results, opts) end end # Helper functions defp build_suite_filter(opts) do filter = [] filter = if opts[:suite] do Keyword.put(filter, :names, [opts[:suite]]) else filter end filter = if opts[:tags] do tags = String.split(opts[:tags], ",") |> Enum.map(&String.to_atom/1) Keyword.put(filter, :tags, tags) else filter end filter end defp build_benchmark_config(opts) do config = [ time: opts[:time] || 5, memory_time: opts[:memory_time] || 2, warmup: opts[:warmup] || 1, parallel: opts[:parallel] || 1 ] config = if opts[:iterations] do Keyword.put(config, :iterations, opts[:iterations]) else config end config end defp analyze_suite_results(results) do results |> Enum.map(fn {suite, data} -> {suite, %{ statistics: calculate_statistics(data), performance: analyze_performance(data), memory: analyze_memory(data) }} end) |> Enum.into(%{}) end defp calculate_statistics(data) do times = extract_execution_times(data) StatisticalAnalyzer.analyze(times) end defp analyze_performance(data) do # Performance analysis logic %{ average_time: calculate_average(data), throughput: calculate_throughput(data), latency: calculate_latency(data) } end defp analyze_memory(data) do # Memory analysis logic %{ peak_memory: calculate_peak_memory(data), average_memory: calculate_average_memory(data), allocations: count_allocations(data) } end defp output_results(results, opts) do format = String.to_atom(opts[:format] || "text") output = case format do :json -> Jason.encode!(results, pretty: true) :html -> generate_html_output(results) :markdown -> generate_markdown_output(results) _ -> generate_text_output(results) end if opts[:output] do File.write!(opts[:output], output) Mix.shell().info("Results saved to: #{opts[:output]}") else Log.info(output) end end defp output_competitor_comparison(comparison, scores, _opts) do Mix.shell().info("\n=== Competitor Comparison Results ===\n") Mix.shell().info(comparison.summary) Mix.shell().info("\nPerformance Scores (higher is better):") scores |> Enum.sort_by(fn {_name, score} -> score end, :desc) |> Enum.each(fn {name, score} -> Mix.shell().info(" #{name}: #{Float.round(score, 2)}/100") end) end defp load_latest_results do # Load the most recent benchmark results case File.ls("bench/results") do {:ok, files} -> latest_file = files |> Enum.filter(&String.ends_with?(&1, ".benchee")) |> Enum.sort() |> List.last() if latest_file do # Returns path to latest benchee result file # Note: Benchee result deserialization would require custom implementation {:ok, Path.join("bench/results", latest_file)} end _ -> nil end end defp load_baseline_results(version) do # Load baseline results for comparison baseline_file = "bench/baselines/#{version}.benchee" if File.exists?(baseline_file) do # Returns path to baseline benchee result file # Note: Benchee result deserialization would require custom implementation {:ok, baseline_file} else nil end end defp check_regressions(results, baseline_version, opts) do baseline = load_baseline_results(baseline_version) if baseline do threshold = opts[:threshold] || 0.05 detection = RegressionDetector.detect(results, baseline, threshold: threshold) if detection.regressions != [] do print_regressions(detection.regressions) end end end defp print_regressions(regressions) do Mix.shell().error("\n[WARN] Performance Regressions Detected:") Enum.each(regressions, fn reg -> Mix.shell().error( " • #{reg.scenario}: +#{Float.round(reg.percent_change, 1)}% (#{reg.severity})" ) end) end defp loop_continuous_monitoring(interval, opts) do # Run benchmarks {:ok, results} = SuiteRegistry.run_all(build_benchmark_config(opts)) # Analyze results analysis = analyze_suite_results(results) # Display summary display_monitoring_summary(analysis) # Check for anomalies check_for_anomalies(analysis) # Sleep and repeat :timer.sleep(interval) loop_continuous_monitoring(interval, opts) end defp display_monitoring_summary(analysis) do timestamp = DateTime.utc_now() |> DateTime.to_string() Mix.shell().info("\n[#{timestamp}] Monitoring Update:") Enum.each(analysis, fn {suite, data} -> avg_time = get_in(data, [:statistics, :mean]) || 0 p99 = get_in(data, [:statistics, :percentiles, :p99]) || 0 Mix.shell().info( " #{suite}: avg=#{Float.round(avg_time, 2)}μs, p99=#{Float.round(p99, 2)}μs" ) end) end defp check_for_anomalies(analysis) do Enum.each(analysis, fn {suite, data} -> outliers = get_in(data, [:statistics, :outliers, :outliers]) || [] if outliers != [] do Mix.shell().warn( " [WARN] #{suite}: #{length(outliers)} outliers detected" ) end end) end defp generate_text_output(results) do Enum.map_join(results, "\n", fn {suite, data} -> """ Suite: #{suite} ================ Mean: #{Float.round(get_in(data, [:statistics, :mean]) || 0, 2)}μs Median: #{Float.round(get_in(data, [:statistics, :median]) || 0, 2)}μs P95: #{Float.round(get_in(data, [:statistics, :percentiles, :p95]) || 0, 2)}μs P99: #{Float.round(get_in(data, [:statistics, :percentiles, :p99]) || 0, 2)}μs """ end) end defp generate_html_output(_results) do # Generate HTML report "

Benchmark Results

" end defp generate_markdown_output(results) do # Generate Markdown report Enum.map_join(results, "\n", fn {suite, _data} -> "## #{suite}\n\nResults pending...\n" end) end defp timestamp do DateTime.utc_now() |> DateTime.to_iso8601() |> String.replace(~r/[:\.]/, "_") end defp print_help do Mix.shell().info(@moduledoc) end # Stub implementations for complex functions defp extract_execution_times(_data), do: [1.0, 2.0, 3.0] defp calculate_average(_data), do: 2.0 defp calculate_throughput(_data), do: 1000.0 defp calculate_latency(_data), do: 1.0 defp calculate_peak_memory(_data), do: 100_000 defp calculate_average_memory(_data), do: 50_000 defp count_allocations(_data), do: 1000 defp load_all_results, do: %{} defp perform_statistical_analysis(_results), do: %{} defp analyze_performance_trends(_results), do: %{} defp compare_with_competitors(_results), do: %{} defp detect_historical_regressions(_results), do: %{} defp generate_recommendations(_results), do: [] defp generate_report(_analysis, _format), do: "" defp extract_time_series(_results), do: [] defp perform_statistical_tests(_data, _opts), do: %{} defp output_statistical_analysis(_analysis, _opts), do: :ok defp output_regression_report(_report, _detection, _opts), do: :ok defp run_with_profiling(_suite, _opts), do: %{} defp generate_flame_graph, do: :ok defp analyze_profile_output, do: %{} defp output_profiling_results(_analysis, _results, _opts), do: :ok end