defmodule Object.Supervisor do @moduledoc """ Enhanced supervision tree for the AAOS Object system with comprehensive fault tolerance. Implements advanced supervision strategies including circuit breaker integration, progressive restart policies, health monitoring integration, and intelligent failure isolation to ensure maximum system resilience. ## Features - **Layered Supervision**: Multi-tier supervision with different restart strategies - **Intelligent Restart Policies**: Context-aware restart decisions - **Circuit Breaker Integration**: Automatic service isolation during failures - **Health Monitoring**: Continuous supervision health assessment - **Progressive Backoff**: Increasing restart delays for persistent failures - **Failure Isolation**: Prevent cascading failures across components - **Resource Protection**: Monitor and limit resource consumption ## Supervision Hierarchy ``` Object.Supervisor (rest_for_one) ├── Infrastructure Supervisor (one_for_one) │ ├── Registry │ ├── Error Handling │ ├── Health Monitor │ └── Resource Monitor ├── Core Services Supervisor (one_for_one) │ ├── Schema Registry │ ├── Dead Letter Queue │ ├── Graceful Degradation │ └── Coordination Service ├── Object Management Supervisor (rest_for_one) │ ├── Dynamic Supervisor │ ├── Performance Monitor │ └── Schema Evolution Manager └── Integration Services Supervisor (one_for_one) ├── Message Router └── AI Reasoning (if available) ``` ## Restart Strategies - **Infrastructure**: Critical components that must be restarted immediately - **Core Services**: Essential services with progressive restart delays - **Object Management**: Components that may require dependent restarts - **Integration**: Optional services that can fail without affecting core functionality """ use Supervisor require Logger # Supervision configuration @max_restarts 5 @max_seconds 60 @restart_intensity_period 300 # 5 minutes # Progressive backoff configuration @initial_restart_delay 1000 # 1 second @max_restart_delay 300_000 # 5 minutes @backoff_multiplier 2.0 @doc """ Starts the enhanced supervision tree with comprehensive error handling. ## Options - `:restart_strategy` - Overall restart strategy (default: :rest_for_one) - `:max_restarts` - Maximum restarts in time period (default: 5) - `:max_seconds` - Time period for restart counting (default: 60) - `:enable_circuit_breakers` - Enable circuit breaker integration (default: true) - `:enable_health_monitoring` - Enable health monitoring (default: true) - `:enable_progressive_backoff` - Enable progressive restart delays (default: true) """ def start_link(init_arg \\ []) do Supervisor.start_link(__MODULE__, init_arg, name: __MODULE__) end @doc """ Gets the current supervision tree health status. """ def get_supervision_health do children = Supervisor.which_children(__MODULE__) health_info = children |> Enum.map(fn {id, pid, type, modules} -> status = if is_pid(pid) and Process.alive?(pid) do :running else :stopped end {id, %{pid: pid, type: type, modules: modules, status: status}} end) |> Map.new() %{ supervisor_status: :running, children: health_info, total_children: length(children), running_children: Enum.count(children, fn {_, pid, _, _} -> is_pid(pid) and Process.alive?(pid) end), timestamp: DateTime.utc_now() } end @doc """ Gracefully restarts a specific child process with backoff. """ def restart_child_with_backoff(child_id) do GenServer.call(__MODULE__, {:restart_child_with_backoff, child_id}) end @doc """ Triggers emergency shutdown with graceful degradation. """ def emergency_shutdown(reason) do Logger.error("Emergency shutdown triggered: #{inspect(reason)}") # Trigger graceful degradation first try do # GracefulDegradation.trigger_degradation(:shutdown, reason) :timer.sleep(5000) # Allow 5 seconds for graceful degradation rescue _ -> :ok end # Then perform shutdown Supervisor.stop(__MODULE__, reason) end @impl true def init(opts) do # Extract configuration options restart_strategy = Keyword.get(opts, :restart_strategy, :rest_for_one) max_restarts = Keyword.get(opts, :max_restarts, @max_restarts) max_seconds = Keyword.get(opts, :max_seconds, @max_seconds) _enable_circuit_breakers = Keyword.get(opts, :enable_circuit_breakers, true) enable_health_monitoring = Keyword.get(opts, :enable_health_monitoring, true) # Build supervision tree with layered architecture children = build_supervision_tree(opts) # Start supervision monitoring if enabled if enable_health_monitoring do schedule_supervision_health_check() end Logger.info("Starting enhanced supervision tree with strategy: #{restart_strategy}") Logger.info("Supervision limits: #{max_restarts} restarts in #{max_seconds} seconds") Supervisor.init(children, strategy: restart_strategy, max_restarts: max_restarts, max_seconds: max_seconds ) end # NOTE: Supervisor behavior does not support handle_call/handle_info callbacks # These functions should be implemented in a separate GenServer if needed # Public API functions that can be called directly def trigger_health_check do perform_supervision_health_check() end # Private functions defp build_supervision_tree(opts) do [ # Infrastructure Supervisor - Critical system components infrastructure_supervisor_spec(opts), # Core Services Supervisor - Essential application services core_services_supervisor_spec(opts), # Object Management Supervisor - Object lifecycle management object_management_supervisor_spec(opts), # Integration Services Supervisor - Optional external integrations integration_services_supervisor_spec(opts) ] end defp infrastructure_supervisor_spec(_opts) do children = [ # Registry must be first {Registry, keys: :unique, name: Object.Registry} ] %{ id: :infrastructure_supervisor, start: {Supervisor, :start_link, [children, [strategy: :one_for_one, name: Object.InfrastructureSupervisor]]}, restart: :permanent, type: :supervisor } end defp core_services_supervisor_spec(opts) do children = [ # Schema registry for object tracking {Object.SchemaRegistry, Keyword.get(opts, :schema_registry_opts, [])} ] %{ id: :core_services_supervisor, start: {Supervisor, :start_link, [children, [strategy: :one_for_one, name: Object.CoreServicesSupervisor]]}, restart: :permanent, type: :supervisor } end defp object_management_supervisor_spec(opts) do children = [ # Dynamic supervisor for object processes enhanced_dynamic_supervisor_spec(opts), # Schema evolution manager {Object.SchemaEvolutionManager, Keyword.get(opts, :schema_evolution_opts, [])} ] %{ id: :object_management_supervisor, start: {Supervisor, :start_link, [children, [strategy: :rest_for_one, name: Object.ObjectManagementSupervisor]]}, restart: :permanent, type: :supervisor } end defp integration_services_supervisor_spec(opts) do children = [ # Message router for object communication message_router_spec(opts), # AI reasoning integration (optional) ai_reasoning_spec(opts) ] |> Enum.filter(& &1 != nil) if Enum.empty?(children) do # Return a placeholder supervisor if no integration services %{ id: :integration_services_supervisor, start: {Supervisor, :start_link, [[], [strategy: :one_for_one, name: Object.IntegrationServicesSupervisor]]}, restart: :temporary, type: :supervisor } else %{ id: :integration_services_supervisor, start: {Supervisor, :start_link, [children, [strategy: :one_for_one, name: Object.IntegrationServicesSupervisor]]}, restart: :temporary, type: :supervisor } end end defp enhanced_dynamic_supervisor_spec(opts) do # Enhanced dynamic supervisor with circuit breaker integration strategy = Keyword.get(opts, :dynamic_supervisor_strategy, :one_for_one) max_children = Keyword.get(opts, :max_dynamic_children, 1000) %{ id: :dynamic_supervisor, start: {DynamicSupervisor, :start_link, [[name: Object.DynamicSupervisor, strategy: strategy, max_children: max_children]]}, restart: :permanent, type: :supervisor } end # Intentionally unused - placeholder for future resource monitoring defp resource_monitor_spec(opts) do # Resource monitoring disabled - Object.ResourceMonitor not available if Keyword.get(opts, :enable_resource_monitoring, false) do # Return a minimal placeholder %{ id: :resource_monitor_placeholder, start: {Agent, :start_link, [fn -> %{} end]}, restart: :temporary, type: :worker } else nil end end # Intentionally unused - placeholder for future health monitoring defp health_monitor_spec(opts) do # Health monitoring disabled - Object.HealthMonitor not available if Keyword.get(opts, :enable_health_monitoring, false) do # Return a minimal placeholder %{ id: :health_monitor_placeholder, start: {Agent, :start_link, [fn -> %{} end]}, restart: :temporary, type: :worker } else nil end end defp message_router_spec(opts) do if Keyword.get(opts, :enable_message_router, true) do {Object.MessageRouter, Keyword.get(opts, :message_router_opts, [])} else nil end end defp ai_reasoning_spec(opts) do if Keyword.get(opts, :enable_ai_reasoning, false) do {Object.AIReasoning, Keyword.get(opts, :ai_reasoning_opts, [])} else nil end end # Intentionally unused - would require separate GenServer for implementation defp restart_child_with_progressive_backoff(child_id) do # Calculate backoff delay based on recent restart history restart_count = get_recent_restart_count(child_id) delay = calculate_backoff_delay(restart_count) Logger.info("Restarting #{child_id} with #{delay}ms backoff (attempt #{restart_count + 1})") # Since Supervisor doesn't support delayed restarts, we'll restart immediately # In a production system, you'd want a separate GenServer to handle delayed restarts case Supervisor.restart_child(__MODULE__, child_id) do {:ok, _} -> :ok error -> error end end # Helper for restart_child_with_progressive_backoff - currently unused defp get_recent_restart_count(child_id) do # This would track restart history per child # For now, return a simple count based on process dictionary restart_key = {:restart_count, child_id} current_count = Process.get(restart_key, 0) # Reset count if it's been more than the intensity period last_restart_key = {:last_restart, child_id} last_restart = Process.get(last_restart_key, 0) current_time = System.monotonic_time(:millisecond) if current_time - last_restart > @restart_intensity_period * 1000 do Process.put(restart_key, 0) Process.put(last_restart_key, current_time) 0 else Process.put(restart_key, current_count + 1) Process.put(last_restart_key, current_time) current_count end end # Helper for restart_child_with_progressive_backoff - currently unused defp calculate_backoff_delay(restart_count) do delay = @initial_restart_delay * :math.pow(@backoff_multiplier, restart_count) min(trunc(delay), @max_restart_delay) end defp schedule_supervision_health_check do # NOTE: This would need to be implemented in a separate GenServer # that monitors the supervisor, as Supervisors don't handle messages :ok end defp perform_supervision_health_check do health_info = get_supervision_health() # Check for unhealthy children stopped_children = health_info.children |> Enum.filter(fn {_id, info} -> info.status == :stopped end) |> Enum.map(fn {id, _info} -> id end) if not Enum.empty?(stopped_children) do Logger.warning("Supervision health check found stopped children: #{inspect(stopped_children)}") # Report to health monitor if available try do # HealthMonitor.report_health_event(:supervision_tree, :degraded, # %{stopped_children: stopped_children}) rescue _ -> :ok end end # Check restart frequency running_ratio = health_info.running_children / health_info.total_children if running_ratio < 0.8 do Logger.error("Supervision tree health degraded: only #{trunc(running_ratio * 100)}% of children running") # Consider triggering emergency procedures if running_ratio < 0.5 do try do # GracefulDegradation.trigger_degradation(:critical, :supervision_failure) rescue _ -> :ok end end end end end