defmodule Services.ModelPerformanceTracker do @moduledoc """ A GenServer that tracks AI model performance metrics during sessions. This service tracks request-level timing and token usage to help evaluate different model configurations and their effectiveness. """ use GenServer defstruct [ :session_id, :active_requests, :completed_requests ] @type tracking_id :: String.t() @type model :: AI.Model.t() @type usage_data :: map() @type request_data :: %{ id: tracking_id(), model: model(), start_time: integer(), end_time: integer() | nil, usage: usage_data() | nil } @type t :: %__MODULE__{ session_id: String.t(), active_requests: %{tracking_id() => request_data()}, completed_requests: [request_data()] } # Client API @spec start_link() :: {:ok, pid()} def start_link do GenServer.start_link(__MODULE__, %{}, name: __MODULE__) end @spec start_session() :: String.t() def start_session do GenServer.call(__MODULE__, :start_session) end @spec begin_tracking(model()) :: tracking_id() def begin_tracking(model) do GenServer.call(__MODULE__, {:begin_tracking, model}) end @spec end_tracking(tracking_id(), usage_data()) :: :ok def end_tracking(tracking_id, usage_data) do GenServer.call(__MODULE__, {:end_tracking, tracking_id, usage_data}) end @spec generate_report() :: String.t() def generate_report do GenServer.call(__MODULE__, :generate_report) end @spec reset_session() :: :ok def reset_session do GenServer.call(__MODULE__, :reset_session) end # Server Callbacks @impl GenServer def init(_args) do {:ok, %__MODULE__{ session_id: generate_session_id(), active_requests: %{}, completed_requests: [] }} end @impl GenServer def handle_call(:start_session, _from, state) do new_session_id = generate_session_id() new_state = %{state | session_id: new_session_id, completed_requests: []} {:reply, new_session_id, new_state} end @impl GenServer def handle_call({:begin_tracking, model}, _from, state) do tracking_id = generate_tracking_id() request_data = %{ id: tracking_id, model: model, start_time: System.monotonic_time(:millisecond), end_time: nil, usage: nil } new_active_requests = Map.put(state.active_requests, tracking_id, request_data) new_state = %{state | active_requests: new_active_requests} {:reply, tracking_id, new_state} end @impl GenServer def handle_call({:end_tracking, tracking_id, usage_data}, _from, state) do case Map.get(state.active_requests, tracking_id) do nil -> {:reply, :ok, state} request_data -> completed_request = %{ request_data | end_time: System.monotonic_time(:millisecond), usage: usage_data } new_active_requests = Map.delete(state.active_requests, tracking_id) new_completed_requests = [completed_request | state.completed_requests] new_state = %{ state | active_requests: new_active_requests, completed_requests: new_completed_requests } {:reply, :ok, new_state} end end @impl GenServer def handle_call(:generate_report, _from, state) do report = build_performance_report(state.completed_requests) {:reply, report, state} end @impl GenServer def handle_call(:reset_session, _from, state) do new_state = %{ state | session_id: generate_session_id(), active_requests: %{}, completed_requests: [] } {:reply, :ok, new_state} end # Private Functions defp generate_session_id do :crypto.strong_rand_bytes(8) |> Base.encode16(case: :lower) end defp generate_tracking_id do :crypto.strong_rand_bytes(4) |> Base.encode16(case: :lower) end defp build_performance_report([]), do: "" defp build_performance_report(requests) do total_requests = length(requests) if total_requests == 0 do "" else model_stats = calculate_model_statistics(requests) overall_stats = calculate_overall_statistics(requests) """ ### Model Performance Report **Session Summary:** - Total API Requests: #{total_requests} - Total Time: #{overall_stats.total_time_ms}ms - Total Tokens: #{overall_stats.total_tokens} #{format_model_breakdown(model_stats)} #{format_detailed_metrics(model_stats)} """ end end defp calculate_overall_statistics(requests) do total_time_ms = requests |> Enum.map(fn req -> req.end_time - req.start_time end) |> Enum.sum() total_tokens = requests |> Enum.map(fn req -> get_total_tokens(req.usage) end) |> Enum.sum() %{ total_time_ms: total_time_ms, total_tokens: total_tokens } end defp calculate_model_statistics(requests) do requests |> Enum.group_by(fn req -> %{ model: req.model.model, reasoning: req.model.reasoning } end) |> Enum.map(fn {model_config, model_requests} -> total_time_ms = model_requests |> Enum.map(fn req -> req.end_time - req.start_time end) |> Enum.sum() request_count = length(model_requests) avg_time_ms = if request_count > 0, do: total_time_ms / request_count, else: 0 total_input_tokens = model_requests |> Enum.map(fn req -> get_input_tokens(req.usage) end) |> Enum.sum() total_output_tokens = model_requests |> Enum.map(fn req -> get_output_tokens(req.usage) end) |> Enum.sum() total_reasoning_tokens = model_requests |> Enum.map(fn req -> get_reasoning_tokens(req.usage) end) |> Enum.sum() total_tokens = total_input_tokens + total_output_tokens + total_reasoning_tokens # Calculate tokens per minute total_time_minutes = total_time_ms / 1000 / 60 tokens_per_minute = if total_time_minutes > 0, do: total_tokens / total_time_minutes, else: 0 output_tokens_per_minute = if total_time_minutes > 0, do: total_output_tokens / total_time_minutes, else: 0 # Calculate input token analysis input_analysis = calculate_input_analysis(model_requests) %{ model_config: model_config, request_count: request_count, total_time_ms: total_time_ms, avg_time_ms: avg_time_ms, total_input_tokens: total_input_tokens, total_output_tokens: total_output_tokens, total_reasoning_tokens: total_reasoning_tokens, total_tokens: total_tokens, tokens_per_minute: tokens_per_minute, output_tokens_per_minute: output_tokens_per_minute, input_analysis: input_analysis } end) |> Enum.sort_by(fn stat -> {stat.model_config.model, reasoning_level_to_int(stat.model_config.reasoning)} end) end defp calculate_input_analysis(requests) do if length(requests) == 0 do %{ avg_input_size: 0, input_processing_speed_ms_per_token: 0.0, scaling_analysis: %{}, input_correlation: 0.0 } else # Basic input metrics input_sizes = Enum.map(requests, fn req -> get_input_tokens(req.usage) end) processing_times = Enum.map(requests, fn req -> req.end_time - req.start_time end) avg_input_size = Enum.sum(input_sizes) / length(input_sizes) # Calculate input processing speed (ms per input token) total_input_tokens = Enum.sum(input_sizes) total_processing_time = Enum.sum(processing_times) input_processing_speed = if total_input_tokens > 0 do total_processing_time / total_input_tokens else 0.0 end # Input size bucketing analysis scaling_analysis = calculate_scaling_analysis(requests) # Calculate correlation between input size and processing time input_correlation = calculate_correlation(input_sizes, processing_times) %{ avg_input_size: avg_input_size, input_processing_speed_ms_per_token: input_processing_speed, scaling_analysis: scaling_analysis, input_correlation: input_correlation } end end defp calculate_scaling_analysis(requests) do # Group requests by input size buckets buckets = %{ # < 2000 tokens small: [], # 2000-10000 tokens medium: [], # > 10000 tokens large: [] } bucketed_requests = Enum.reduce(requests, buckets, fn req, acc -> input_tokens = get_input_tokens(req.usage) processing_time = req.end_time - req.start_time request_data = %{input_tokens: input_tokens, processing_time: processing_time} cond do input_tokens < 2000 -> %{acc | small: [request_data | acc.small]} input_tokens <= 10000 -> %{acc | medium: [request_data | acc.medium]} true -> %{acc | large: [request_data | acc.large]} end end) # Calculate metrics for each bucket bucket_stats = %{ small: calculate_bucket_stats(bucketed_requests.small), medium: calculate_bucket_stats(bucketed_requests.medium), large: calculate_bucket_stats(bucketed_requests.large) } # Calculate scaling factors small_speed = bucket_stats.small.avg_processing_speed_ms_per_token medium_speed = bucket_stats.medium.avg_processing_speed_ms_per_token large_speed = bucket_stats.large.avg_processing_speed_ms_per_token scaling_factors = %{ medium_vs_small: if(small_speed > 0, do: medium_speed / small_speed, else: 0.0), large_vs_small: if(small_speed > 0, do: large_speed / small_speed, else: 0.0), large_vs_medium: if(medium_speed > 0, do: large_speed / medium_speed, else: 0.0) } %{ buckets: bucket_stats, scaling_factors: scaling_factors } end defp calculate_bucket_stats([]), do: %{ count: 0, avg_input_size: 0, avg_processing_time: 0, avg_processing_speed_ms_per_token: 0.0 } defp calculate_bucket_stats(bucket_data) do count = length(bucket_data) total_input = Enum.sum(Enum.map(bucket_data, & &1.input_tokens)) total_time = Enum.sum(Enum.map(bucket_data, & &1.processing_time)) avg_input_size = if count > 0, do: total_input / count, else: 0 avg_processing_time = if count > 0, do: total_time / count, else: 0 avg_processing_speed = if total_input > 0, do: total_time / total_input, else: 0.0 %{ count: count, avg_input_size: avg_input_size, avg_processing_time: avg_processing_time, avg_processing_speed_ms_per_token: avg_processing_speed } end defp calculate_correlation([], []), do: 0.0 # Need at least 2 points defp calculate_correlation([_], [_]), do: 0.0 defp calculate_correlation(x_values, y_values) when length(x_values) != length(y_values), do: 0.0 defp calculate_correlation(x_values, y_values) do n = length(x_values) if n < 2 do 0.0 else # Calculate means x_mean = Enum.sum(x_values) / n y_mean = Enum.sum(y_values) / n # Calculate correlation coefficient numerator = Enum.zip(x_values, y_values) |> Enum.map(fn {x, y} -> (x - x_mean) * (y - y_mean) end) |> Enum.sum() x_variance = x_values |> Enum.map(fn x -> (x - x_mean) * (x - x_mean) end) |> Enum.sum() y_variance = y_values |> Enum.map(fn y -> (y - y_mean) * (y - y_mean) end) |> Enum.sum() denominator = :math.sqrt(x_variance * y_variance) if denominator > 0 do numerator / denominator else 0.0 end end end defp format_model_breakdown(model_stats) do if length(model_stats) <= 1 do "" else breakdown = model_stats |> Enum.map(fn stat -> "- #{format_model_name(stat.model_config)}: #{stat.request_count} requests, #{stat.total_time_ms}ms" end) |> Enum.join("\n") """ **By Model:** #{breakdown} """ end end defp format_detailed_metrics(model_stats) do model_stats |> Enum.map(fn stat -> input_analysis_text = format_input_analysis(stat.input_analysis) """ **#{format_model_name(stat.model_config)}:** - Requests: #{stat.request_count}, Avg Input: #{format_number(stat.input_analysis.avg_input_size)} tokens - Avg Response Time: #{Float.round(stat.avg_time_ms, 1)}ms (#{Float.round(stat.input_analysis.input_processing_speed_ms_per_token, 2)}ms/token input) - Total Tokens: #{stat.total_tokens} (Input: #{stat.total_input_tokens}, Output: #{stat.total_output_tokens}#{format_reasoning_tokens(stat.total_reasoning_tokens)}) - Throughput: #{Float.round(stat.tokens_per_minute, 1)} tokens/min (#{Float.round(stat.output_tokens_per_minute, 1)} output/min)#{input_analysis_text} """ end) |> Enum.join("\n") end defp format_model_name(%{model: model, reasoning: reasoning}) do case reasoning do :none -> model reasoning_level -> "#{model} (reasoning: #{reasoning_level})" end end defp format_reasoning_tokens(0), do: "" defp format_reasoning_tokens(count), do: ", Reasoning: #{count}" defp format_input_analysis(%{ scaling_analysis: scaling_analysis, input_correlation: correlation }) do scaling_text = format_scaling_analysis(scaling_analysis) correlation_text = format_correlation(correlation) case {scaling_text, correlation_text} do {"", ""} -> "" {scaling, ""} -> "\n#{scaling}" {"", corr} -> "\n#{corr}" {scaling, corr} -> "\n#{scaling}\n#{corr}" end end defp format_scaling_analysis(%{buckets: buckets, scaling_factors: factors}) do # Only show scaling analysis if we have meaningful data in multiple buckets bucket_counts = [ buckets.small.count, buckets.medium.count, buckets.large.count ] active_buckets = Enum.count(bucket_counts, fn count -> count > 0 end) if active_buckets < 2 do "" else parts = [] # Show bucket breakdown bucket_info = [ if buckets.small.count > 0 do "Small (<2K): #{buckets.small.count} requests, #{Float.round(buckets.small.avg_processing_time, 0)}ms avg" end, if buckets.medium.count > 0 do "Medium (2-10K): #{buckets.medium.count} requests, #{Float.round(buckets.medium.avg_processing_time, 0)}ms avg" end, if buckets.large.count > 0 do "Large (>10K): #{buckets.large.count} requests, #{Float.round(buckets.large.avg_processing_time, 0)}ms avg" end ] |> Enum.filter(& &1) |> Enum.join(", ") parts = if bucket_info != "", do: ["- Input Size Analysis: #{bucket_info}" | parts], else: parts # Show most significant scaling factor {significant_factor, factor_value} = [ {"Large vs Small", factors.large_vs_small}, {"Large vs Medium", factors.large_vs_medium}, {"Medium vs Small", factors.medium_vs_small} ] # Only show significant differences |> Enum.filter(fn {_name, value} -> value > 1.2 end) |> Enum.max_by(fn {_name, value} -> value end, fn -> {nil, 0.0} end) scaling_info = if significant_factor do "- Scaling Impact: #{significant_factor} inputs are #{Float.round(factor_value, 1)}x slower" else nil end parts = if scaling_info, do: [scaling_info | parts], else: parts if length(parts) > 0 do Enum.reverse(parts) |> Enum.join("\n") else "" end end end defp format_correlation(correlation) when correlation > 0.7 do "- Input Size Impact: Strong correlation (#{Float.round(correlation, 2)}) - larger inputs significantly slower" end defp format_correlation(correlation) when correlation > 0.4 do "- Input Size Impact: Moderate correlation (#{Float.round(correlation, 2)}) - some scaling effect observed" end defp format_correlation(_), do: "" defp format_number(num) when is_float(num), do: format_number(round(num)) defp format_number(num) when num >= 1_000_000 do "#{Float.round(num / 1_000_000, 1)}M" end defp format_number(num) when num >= 1_000 do # Format with commas for readability num |> Integer.to_string() |> String.graphemes() |> Enum.reverse() |> Enum.chunk_every(3) |> Enum.map(&Enum.reverse/1) |> Enum.reverse() |> Enum.join(",") end defp format_number(num), do: Integer.to_string(num) # Convert reasoning levels to integers for sorting (least to most effort) defp reasoning_level_to_int(:none), do: 0 defp reasoning_level_to_int(:minimal), do: 1 defp reasoning_level_to_int(:low), do: 2 defp reasoning_level_to_int(:medium), do: 3 defp reasoning_level_to_int(:high), do: 4 defp get_total_tokens(%{"total_tokens" => total}), do: total defp get_total_tokens(%{total_tokens: total}), do: total defp get_total_tokens(_), do: 0 defp get_input_tokens(%{"prompt_tokens" => prompt}), do: prompt defp get_input_tokens(%{prompt_tokens: prompt}), do: prompt defp get_input_tokens(_), do: 0 defp get_output_tokens(%{"completion_tokens" => completion}), do: completion defp get_output_tokens(%{completion_tokens: completion}), do: completion defp get_output_tokens(_), do: 0 defp get_reasoning_tokens(%{"reasoning_tokens" => reasoning}), do: reasoning defp get_reasoning_tokens(%{reasoning_tokens: reasoning}), do: reasoning defp get_reasoning_tokens(_), do: 0 end