defmodule Sentry.Telemetry.Scheduler do @moduledoc """ GenServer implementing a weighted round-robin scheduler for telemetry buffers. The scheduler cycles through category buffers based on priority weights, ensuring critical telemetry gets priority over high-volume data under load. ## Weights * `:critical` - weight 5 (errors) * `:high` - weight 4 (check-ins) * `:medium` - weight 3 (transactions) * `:low` - weight 2 (logs) ## Signal-Based Wake The scheduler sleeps until signaled via `signal/1`. When signaled, it wakes and attempts to process items from the current buffer in the cycle. If the buffer is not ready or the transport queue is full, it advances to the next position. ## Transport Queue The scheduler includes a bounded FIFO queue for transport concurrency control, processing one envelope at a time (single-worker model). The queue is capped at a configurable capacity (default 1000 items) to prevent unbounded memory growth. For log envelopes, each log event counts as one item toward capacity. """ @moduledoc since: "12.0.0" use GenServer require Logger alias __MODULE__ alias Sentry.Telemetry.{Buffer, Category} alias Sentry.{ CheckIn, ClientReport, Config, Envelope, Event, LogEvent, Metric, Transaction, Transport } alias Sentry.Transport.RateLimiter @default_capacity 1000 @type buffers :: %{ error: GenServer.server(), check_in: GenServer.server(), transaction: GenServer.server(), log: GenServer.server(), metric: GenServer.server() } defstruct [ :buffers, :priority_cycle, :cycle_position, :on_envelope, :capacity, :active_ref, :active_item_count, queue: :queue.new(), size: 0 ] @type t :: %Scheduler{ buffers: buffers(), priority_cycle: [Category.t()], cycle_position: non_neg_integer(), on_envelope: (Envelope.t() -> any()) | nil, capacity: pos_integer(), queue: :queue.queue(), size: non_neg_integer(), active_ref: reference() | nil, active_item_count: non_neg_integer() } ## Public API @doc """ Builds a priority cycle based on category weights. Returns a list of categories where each category appears a number of times equal to its priority weight. ## Examples iex> Sentry.Telemetry.Scheduler.build_priority_cycle() [:error, :error, :error, :error, :error, :check_in, :check_in, :check_in, :check_in, :transaction, :transaction, :transaction, :log, :log, :metric, :metric] """ @spec build_priority_cycle(map() | nil) :: [Category.t()] def build_priority_cycle(weights \\ nil) def build_priority_cycle(nil), do: build_priority_cycle(default_weights()) def build_priority_cycle(weights) when weights == %{}, do: build_priority_cycle(default_weights()) def build_priority_cycle(weights) when is_map(weights) do merged_weights = Map.merge(default_weights(), weights) for {category, priority} <- category_priority_mapping(), _i <- 1..Map.fetch!(merged_weights, priority) do category end end @doc """ Starts the scheduler process. ## Options * `:buffers` - Map of category to buffer pid (required) * `:name` - The name to register the GenServer under (optional) * `:weights` - Custom priority weights (optional) * `:on_envelope` - Callback function invoked with built envelopes (optional) * `:capacity` - Maximum items in the transport queue (default: #{@default_capacity}) """ @spec start_link(keyword()) :: GenServer.on_start() def start_link(opts) when is_list(opts) do {name, opts} = Keyword.pop(opts, :name) if name do GenServer.start_link(__MODULE__, opts, name: name) else GenServer.start_link(__MODULE__, opts) end end @doc """ Signals the scheduler to wake and process items. This is a non-blocking call that triggers the scheduler to check buffers and send any ready items. """ @spec signal(GenServer.server()) :: :ok def signal(server) do GenServer.cast(server, :signal) end @doc """ Flushes all buffers by draining their contents and sending all items. This is a blocking call that returns when all items have been processed, including any envelopes queued for transport. """ @spec flush(GenServer.server(), timeout()) :: :ok def flush(server, timeout \\ 5000) do GenServer.call(server, :flush, timeout) end ## GenServer Callbacks @impl true def init(opts) do buffers = Keyword.fetch!(opts, :buffers) weights = Keyword.get(opts, :weights) on_envelope = Keyword.get(opts, :on_envelope) capacity = Keyword.get(opts, :capacity, @default_capacity) priority_cycle = build_priority_cycle(weights) |> Enum.filter(fn category -> Map.has_key?(buffers, category) end) state = %Scheduler{ buffers: buffers, priority_cycle: priority_cycle, cycle_position: 0, on_envelope: on_envelope, capacity: capacity } {:ok, state} end @impl true def handle_cast(:signal, %Scheduler{} = state) do state = process_cycle(state) {:noreply, state} end @impl true def handle_call(:flush, _from, %Scheduler{} = state) do state = flush_all_buffers(state) state = wait_for_active(state) state = flush_queue(state) {:reply, :ok, state} end @impl true def handle_info({:DOWN, ref, :process, _pid, reason}, %{active_ref: ref} = state) do if reason != :normal do Logger.warning("Sentry transport send process exited abnormally: #{inspect(reason)}") end state = %{ state | active_ref: nil, size: state.size - state.active_item_count, active_item_count: 0 } state = maybe_process_next(state) {:noreply, state} end def handle_info({:DOWN, _ref, :process, _pid, _reason}, state) do {:noreply, state} end defp process_cycle(%Scheduler{} = state) do cycle_length = length(state.priority_cycle) max_attempts = cycle_length process_cycle(state, 0, max_attempts) end defp process_cycle(state, attempts, max_attempts) when attempts >= max_attempts do state end defp process_cycle(%Scheduler{} = state, attempts, max_attempts) do if not transport_has_space?(state) do # Transport queue is full, stop processing. Items stay in buffers. state else category = Enum.at(state.priority_cycle, state.cycle_position) if category_rate_limited?(state, category) do state = drain_rate_limited(state, category) state = advance_cycle(state) process_cycle(state, attempts + 1, max_attempts) else buffer = Map.fetch!(state.buffers, category) case Buffer.poll_if_ready(buffer) do {:ok, items} when items != [] -> state = send_items(state, category, items) state = advance_cycle(state) process_cycle(state, attempts + 1, max_attempts) _ -> state = advance_cycle(state) process_cycle(state, attempts + 1, max_attempts) end end end end defp send_items(state, :error, [%Event{} = event]) do process_and_send_event(state, event, &send_envelope/2) end defp send_items(state, :check_in, [%CheckIn{} = check_in]) do process_and_send_check_in(state, check_in, &send_envelope/2) end defp send_items(state, :transaction, [%Transaction{} = transaction]) do process_and_send_transaction(state, transaction, &send_envelope/2) end defp send_items(state, :log, log_events) do process_and_send_logs(state, log_events, &send_envelope/2) end defp send_items(state, :metric, metrics) do process_and_send_metrics(state, metrics, &send_envelope/2) end defp flush_all_buffers(%Scheduler{} = state) do for {category, buffer} <- state.buffers do items = Buffer.drain(buffer) if items != [] do case category do :error -> Enum.each(items, fn event -> process_and_send_event(state, event, &send_envelope_direct/2) end) :check_in -> Enum.each(items, fn check_in -> process_and_send_check_in(state, check_in, &send_envelope_direct/2) end) :transaction -> Enum.each(items, fn transaction -> process_and_send_transaction(state, transaction, &send_envelope_direct/2) end) :log -> process_and_send_logs(state, items, &send_envelope_direct/2) :metric -> process_and_send_metrics(state, items, &send_envelope_direct/2) end end end state end defp process_and_send_event(state, %Event{} = event, send_fn) do envelope = Envelope.from_event(event) send_fn.(state, envelope) end defp process_and_send_check_in(state, %CheckIn{} = check_in, send_fn) do envelope = Envelope.from_check_in(check_in) send_fn.(state, envelope) end defp process_and_send_transaction(state, %Transaction{} = transaction, send_fn) do envelope = Envelope.from_transaction(transaction) send_fn.(state, envelope) end defp process_and_send_logs(state, log_events, send_fn) do processed_logs = apply_before_send_log_callbacks(log_events) if processed_logs != [] do envelope = Envelope.from_log_events(processed_logs) send_fn.(state, envelope) else state end end defp process_and_send_metrics(state, metrics, send_fn) do processed_metrics = apply_before_send_metric_callbacks(metrics) if processed_metrics != [] do envelope = Envelope.from_metric_events(processed_metrics) send_fn.(state, envelope) else state end end defp apply_before_send_log_callbacks(log_events) do callback = Config.before_send_log() if callback do for log_event <- log_events, %LogEvent{} = modified_event <- [call_before_send_log(log_event, callback)] do modified_event end else log_events end end defp call_before_send_log(log_event, function) when is_function(function, 1) do function.(log_event) rescue error -> Logger.warning("before_send_log callback failed: #{inspect(error)}") log_event end defp call_before_send_log(log_event, {mod, fun}) do apply(mod, fun, [log_event]) rescue error -> Logger.warning("before_send_log callback failed: #{inspect(error)}") log_event end defp apply_before_send_metric_callbacks(metrics) do callback = Config.before_send_metric() if callback do for metric <- metrics, %Metric{} = modified_metric <- [Metric.call_before_send_callback(metric, callback)] do modified_metric end else metrics end end defp advance_cycle(%Scheduler{} = state) do cycle_length = length(state.priority_cycle) new_position = rem(state.cycle_position + 1, cycle_length) %{state | cycle_position: new_position} end # Used during normal processing - enqueues to internal transport queue defp send_envelope(%Scheduler{on_envelope: callback} = state, envelope) when is_function(callback, 1) do callback.(envelope) state end defp send_envelope(%Scheduler{on_envelope: nil} = state, envelope) do item_count = Envelope.item_count(envelope) if state.size + item_count > state.capacity do Logger.warning("Sentry: transport queue full, dropping #{item_count} item(s)") ClientReport.Sender.record_discarded_events(:queue_overflow, envelope.items) state else queue = :queue.in({envelope, item_count}, state.queue) state = %{state | queue: queue, size: state.size + item_count} maybe_process_next(state) end end # Used during flush - bypasses transport queue, sends directly or via callback defp send_envelope_direct(%Scheduler{on_envelope: callback}, envelope) when is_function(callback, 1) do callback.(envelope) end defp send_envelope_direct(%Scheduler{}, envelope) do send_direct(envelope) end defp send_direct(envelope) do if Config.dsn() do client = Config.client() request_retries = Application.get_env(:sentry, :request_retries, Transport.default_retries()) case Transport.encode_and_post_envelope(envelope, client, request_retries) do {:ok, _id} -> :ok {:error, error} -> Logger.warning("Sentry: failed to send envelope: #{Exception.message(error)}") {:error, error} end else :ok end end defp transport_has_space?(%{on_envelope: cb}) when is_function(cb, 1), do: true defp transport_has_space?(%{size: size, capacity: capacity}), do: size < capacity # Transport queue management (merged from QueueWorker) defp maybe_process_next(%{active_ref: ref} = state) when not is_nil(ref), do: state defp maybe_process_next(%Scheduler{} = state) do case :queue.out(state.queue) do {{:value, {envelope, item_count}}, queue} -> {_pid, ref} = spawn_monitor(fn -> send(envelope) end) %{state | queue: queue, active_ref: ref, active_item_count: item_count} {:empty, _queue} -> state end end defp send(envelope) do if Config.dsn() do client = Config.client() request_retries = Application.get_env(:sentry, :request_retries, Transport.default_retries()) Transport.encode_and_post_envelope(envelope, client, request_retries) else :ok end end defp wait_for_active(%{active_ref: nil} = state), do: state defp wait_for_active(%{active_ref: ref} = state) do receive do {:DOWN, ^ref, :process, _pid, _reason} -> %{ state | active_ref: nil, size: state.size - state.active_item_count, active_item_count: 0 } after 5000 -> Process.demonitor(ref, [:flush]) %{ state | active_ref: nil, size: state.size - state.active_item_count, active_item_count: 0 } end end defp flush_queue(%Scheduler{} = state) do {entries, queue} = drain_queue(state.queue) Enum.each(entries, fn {envelope, _item_count} -> send(envelope) end) %{state | queue: queue, size: 0} end defp drain_queue(queue), do: drain_queue(queue, []) defp drain_queue(queue, acc) do case :queue.out(queue) do {{:value, entry}, queue} -> drain_queue(queue, [entry | acc]) {:empty, queue} -> {Enum.reverse(acc), queue} end end # Drains all items from a rate-limited buffer and records client reports # so Sentry knows data was dropped due to rate limiting. defp drain_rate_limited(state, category) do buffer = Map.fetch!(state.buffers, category) items = Buffer.drain(buffer) if items != [] do data_category = Category.data_category(category) Enum.each(items, fn _item -> ClientReport.Sender.record_discarded_events(:ratelimit_backoff, data_category) end) end state end # Skip rate limit checks when on_envelope callback is set (unit test mode) defp category_rate_limited?(%{on_envelope: cb}, _category) when is_function(cb, 1), do: false defp category_rate_limited?(_state, category) do data_category = Category.data_category(category) RateLimiter.rate_limited?(data_category) end defp default_weights do %{ critical: Category.weight(:critical), high: Category.weight(:high), medium: Category.weight(:medium), low: Category.weight(:low) } end defp category_priority_mapping do [ {:error, :critical}, {:check_in, :high}, {:transaction, :medium}, {:log, :low}, {:metric, :low} ] end end