-module(lightspeed@ops@failover_harness). -compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch, inline]). -define(FILEPATH, "src/lightspeed/ops/failover_harness.gleam"). -export([run_scenario/1, run_matrix/0, scenario_label/1, pass_fail_label/1, signature/1, scenario/1, deterministic/1, outcomes/1, failed_scenarios/1, nondeterministic_failures/1, report_signature/1, snapshot_signature/0, snapshot_report_markdown/0]). -export_type([scenario/0, scenario_outcome/0, report/0]). -if(?OTP_RELEASE >= 27). -define(MODULEDOC(Str), -moduledoc(Str)). -define(DOC(Str), -doc(Str)). -else. -define(MODULEDOC(Str), -compile([])). -define(DOC(Str), -compile([])). -endif. ?MODULEDOC(" Deterministic failover certification harness for M27.\n"). -type scenario() :: supported_backends | ownership_fencing | crash_restart_failover | resume_policy_matrix | reconnect_continuity_slo. -type scenario_outcome() :: {scenario_outcome, scenario(), boolean(), boolean(), binary()}. -type report() :: {report, list(scenario_outcome()), integer(), integer()}. -file("src/lightspeed/ops/failover_harness.gleam", 470). -spec count_nondeterministic(list(scenario_outcome())) -> integer(). count_nondeterministic(Outcomes) -> case Outcomes of [] -> 0; [Outcome | Rest] -> case erlang:element(4, Outcome) of true -> count_nondeterministic(Rest); false -> 1 + count_nondeterministic(Rest) end end. -file("src/lightspeed/ops/failover_harness.gleam", 459). -spec count_failed(list(scenario_outcome())) -> integer(). count_failed(Outcomes) -> case Outcomes of [] -> 0; [Outcome | Rest] -> case erlang:element(3, Outcome) of true -> count_failed(Rest); false -> 1 + count_failed(Rest) end end. -file("src/lightspeed/ops/failover_harness.gleam", 488). -spec join_with(binary(), list(binary())) -> binary(). join_with(Separator, Values) -> case Values of [] -> <<""/utf8>>; [Value] -> Value; [Value@1 | Rest] -> <<<>/binary, (join_with(Separator, Rest))/binary>> end. -file("src/lightspeed/ops/failover_harness.gleam", 449). -spec entries_all_met(list({boolean(), binary()})) -> boolean(). entries_all_met(Entries) -> case Entries of [] -> true; [Entry | Rest] -> {Met, _} = Entry, Met andalso entries_all_met(Rest) end. -file("src/lightspeed/ops/failover_harness.gleam", 360). -spec evaluate_reconnect_continuity_slo() -> {boolean(), binary()}. evaluate_reconnect_continuity_slo() -> Entries = gleam@list:map( lightspeed@cluster@durable_session:supported_backends(), fun(Backend) -> Base = begin _pipe = lightspeed@cluster@durable_session:start( <<"s-4"/utf8>>, <<"node-a"/utf8>>, <<"/chat"/utf8>>, Backend, rehydrate, 0, 25, 1 ), _pipe@1 = lightspeed@cluster@durable_session:append_counter_delta( _pipe, 1, 2 ), _pipe@2 = lightspeed@cluster@durable_session:append_counter_delta( _pipe@1, 1, 3 ), _pipe@3 = lightspeed@cluster@durable_session:append_counter_delta( _pipe@2, 1, 4 ), _pipe@4 = lightspeed@cluster@durable_session:crash( _pipe@3, <<"boom"/utf8>> ), lightspeed@cluster@durable_session:restart(_pipe@4, 200) end, Recovered = case lightspeed@cluster@durable_session:request_takeover( Base, <<"node-b"/utf8>>, 2, lightspeed@cluster@durable_session:expected_token( <<"s-4"/utf8>>, <<"node-b"/utf8>>, 2 ), 210 ) of {ok, Updated} -> Updated; {error, _} -> Base end, Reconnected = lightspeed@cluster@durable_session:reconnect( Recovered, rehydrate, <<"/chat"/utf8>>, 220 ), Continuity = lightspeed@cluster@durable_session:continuity_report( Reconnected, 200, 220 ), {erlang:element(4, Continuity), <<<<(lightspeed@cluster@durable_session:backend_label(Backend))/binary, ":"/utf8>>/binary, (lightspeed@cluster@durable_session:continuity_signature( Continuity ))/binary>>} end ), Passed = entries_all_met(Entries), Signatures = gleam@list:map( Entries, fun(Entry) -> {_, Signature} = Entry, Signature end ), {Passed, join_with(<<";"/utf8>>, Signatures)}. -file("src/lightspeed/ops/failover_harness.gleam", 322). -spec evaluate_resume_policy_matrix() -> {boolean(), binary()}. evaluate_resume_policy_matrix() -> Baseline = begin _pipe = lightspeed@cluster@durable_session:start( <<"s-3"/utf8>>, <<"node-a"/utf8>>, <<"/todos"/utf8>>, {journal_only, <<"journal"/utf8>>}, resume, 0, 25, 0 ), lightspeed@cluster@durable_session:append_counter_delta(_pipe, 2, 6) end, Resumed = lightspeed@cluster@durable_session:reconnect( Baseline, resume, <<"/todos"/utf8>>, 40 ), Rehydrated = lightspeed@cluster@durable_session:reconnect( Baseline, rehydrate, <<"/todos"/utf8>>, 40 ), Remounted = lightspeed@cluster@durable_session:reconnect( Baseline, remount, <<"/todos"/utf8>>, 40 ), Resumed_counter = erlang:element( 4, lightspeed@cluster@durable_session:snapshot(Resumed) ), Rehydrated_counter = erlang:element( 4, lightspeed@cluster@durable_session:snapshot(Rehydrated) ), Remounted_counter = erlang:element( 4, lightspeed@cluster@durable_session:snapshot(Remounted) ), Passed = ((Resumed_counter =:= 2) andalso (Rehydrated_counter =:= 2)) andalso (Remounted_counter =:= 0), {Passed, <<<<<<<<<<"resume="/utf8, (erlang:integer_to_binary(Resumed_counter))/binary>>/binary, "|rehydrate="/utf8>>/binary, (erlang:integer_to_binary(Rehydrated_counter))/binary>>/binary, "|remount="/utf8>>/binary, (erlang:integer_to_binary(Remounted_counter))/binary>>}. -file("src/lightspeed/ops/failover_harness.gleam", 267). -spec evaluate_crash_restart_failover() -> {boolean(), binary()}. evaluate_crash_restart_failover() -> Durable = begin _pipe = lightspeed@cluster@durable_session:start( <<"s-2"/utf8>>, <<"node-a"/utf8>>, <<"/counter"/utf8>>, {snapshot_journal, <<"snap"/utf8>>, <<"journal"/utf8>>, 2}, rehydrate, 0, 25, 1 ), _pipe@1 = lightspeed@cluster@durable_session:append_counter_delta( _pipe, 3, 5 ), _pipe@2 = lightspeed@cluster@durable_session:crash( _pipe@1, <<"boom"/utf8>> ), lightspeed@cluster@durable_session:restart(_pipe@2, 100) end, Recovered = case lightspeed@cluster@durable_session:request_takeover( Durable, <<"node-b"/utf8>>, 2, lightspeed@cluster@durable_session:expected_token( <<"s-2"/utf8>>, <<"node-b"/utf8>>, 2 ), 110 ) of {ok, Updated} -> Updated; {error, _} -> Durable end, Reconnected = lightspeed@cluster@durable_session:reconnect( Recovered, rehydrate, <<"/counter"/utf8>>, 118 ), Continuity = lightspeed@cluster@durable_session:continuity_report( Reconnected, 100, 118 ), Snapshot = lightspeed@cluster@durable_session:snapshot(Reconnected), Passed = ((erlang:element(4, Snapshot) =:= 3) andalso (lightspeed@cluster@durable_session:lifecycle_label( erlang:element(5, Snapshot) ) =:= <<"live"/utf8>>)) andalso erlang:element(4, Continuity), {Passed, <<<<(lightspeed@cluster@durable_session:signature(Reconnected))/binary, "|continuity="/utf8>>/binary, (lightspeed@cluster@durable_session:continuity_signature(Continuity))/binary>>}. -file("src/lightspeed/ops/failover_harness.gleam", 440). -spec ownership_result_label( {ok, lightspeed@cluster@durable_session:durable_session()} | {error, lightspeed@cluster@durable_session:ownership_error()} ) -> binary(). ownership_result_label(Value) -> case Value of {ok, _} -> <<"ok"/utf8>>; {error, Error} -> lightspeed@cluster@durable_session:ownership_error_label(Error) end. -file("src/lightspeed/ops/failover_harness.gleam", 194). -spec evaluate_ownership_fencing() -> {boolean(), binary()}. evaluate_ownership_fencing() -> Baseline = lightspeed@cluster@durable_session:start( <<"s-1"/utf8>>, <<"node-a"/utf8>>, <<"/counter"/utf8>>, {snapshot_journal, <<"snap"/utf8>>, <<"journal"/utf8>>, 2}, rehydrate, 0, 25, 1 ), Takeover_token = lightspeed@cluster@durable_session:expected_token( <<"s-1"/utf8>>, <<"node-b"/utf8>>, 2 ), Takeover = lightspeed@cluster@durable_session:request_takeover( Baseline, <<"node-b"/utf8>>, 2, Takeover_token, 10 ), {Takeover_label, Takeover_signature} = case Takeover of {ok, Updated} -> {<<"ok"/utf8>>, lightspeed@cluster@durable_session:fence_signature( lightspeed@cluster@durable_session:fence(Updated) )}; {error, Error} -> {lightspeed@cluster@durable_session:ownership_error_label(Error), <<"none"/utf8>>} end, Stale_label = case Takeover of {ok, Updated@1} -> _pipe = lightspeed@cluster@durable_session:request_takeover( Updated@1, <<"node-a"/utf8>>, 1, lightspeed@cluster@durable_session:expected_token( <<"s-1"/utf8>>, <<"node-a"/utf8>>, 1 ), 11 ), ownership_result_label(_pipe); {error, Error@1} -> lightspeed@cluster@durable_session:ownership_error_label(Error@1) end, Split_label = case Takeover of {ok, Updated@2} -> _pipe@1 = lightspeed@cluster@durable_session:request_takeover( Updated@2, <<"node-c"/utf8>>, 2, lightspeed@cluster@durable_session:expected_token( <<"s-1"/utf8>>, <<"node-c"/utf8>>, 2 ), 12 ), ownership_result_label(_pipe@1); {error, Error@2} -> lightspeed@cluster@durable_session:ownership_error_label(Error@2) end, Passed = ((Takeover_label =:= <<"ok"/utf8>>) andalso (Stale_label =:= <<"stale_owner:2:1"/utf8>>)) andalso (Split_label =:= <<"split_brain:node-b:node-c:2"/utf8>>), {Passed, <<<<<<<<<<<<<<"takeover="/utf8, Takeover_label/binary>>/binary, "|fence="/utf8>>/binary, Takeover_signature/binary>>/binary, "|stale="/utf8>>/binary, Stale_label/binary>>/binary, "|split="/utf8>>/binary, Split_label/binary>>}. -file("src/lightspeed/ops/failover_harness.gleam", 420). -spec backends_valid( list(lightspeed@cluster@durable_session:persistence_backend()) ) -> boolean(). backends_valid(Backends) -> case Backends of [] -> true; [Backend | Rest] -> Contract = lightspeed@cluster@durable_session:start( <<"s-check"/utf8>>, <<"node-a"/utf8>>, <<"/counter"/utf8>>, Backend, rehydrate, 0, 25, 1 ), lightspeed@cluster@durable_session:valid(Contract) andalso backends_valid( Rest ) end. -file("src/lightspeed/ops/failover_harness.gleam", 186). -spec evaluate_supported_backends() -> {boolean(), binary()}. evaluate_supported_backends() -> Backends = lightspeed@cluster@durable_session:supported_backends(), Labels = gleam@list:map( Backends, fun lightspeed@cluster@durable_session:backend_label/1 ), Passed = (erlang:length(Backends) =:= 3) andalso backends_valid(Backends), {Passed, join_with(<<","/utf8>>, Labels)}. -file("src/lightspeed/ops/failover_harness.gleam", 176). -spec evaluate(scenario()) -> {boolean(), binary()}. evaluate(Scenario) -> case Scenario of supported_backends -> evaluate_supported_backends(); ownership_fencing -> evaluate_ownership_fencing(); crash_restart_failover -> evaluate_crash_restart_failover(); resume_policy_matrix -> evaluate_resume_policy_matrix(); reconnect_continuity_slo -> evaluate_reconnect_continuity_slo() end. -file("src/lightspeed/ops/failover_harness.gleam", 57). ?DOC(" Run one scenario twice and require deterministic parity.\n"). -spec run_scenario(scenario()) -> scenario_outcome(). run_scenario(Scenario) -> {First_passed, First_signature} = evaluate(Scenario), {Second_passed, Second_signature} = evaluate(Scenario), Deterministic = (First_passed =:= Second_passed) andalso (First_signature =:= Second_signature), Passed = (First_passed andalso Second_passed) andalso Deterministic, {scenario_outcome, Scenario, Passed, Deterministic, First_signature}. -file("src/lightspeed/ops/failover_harness.gleam", 38). ?DOC(" Run all M27 scenarios.\n"). -spec run_matrix() -> report(). run_matrix() -> Outcomes = begin _pipe = [supported_backends, ownership_fencing, crash_restart_failover, resume_policy_matrix, reconnect_continuity_slo], gleam@list:map(_pipe, fun run_scenario/1) end, {report, Outcomes, count_failed(Outcomes), count_nondeterministic(Outcomes)}. -file("src/lightspeed/ops/failover_harness.gleam", 73). ?DOC(" Scenario label.\n"). -spec scenario_label(scenario()) -> binary(). scenario_label(Scenario) -> case Scenario of supported_backends -> <<"supported_backends"/utf8>>; ownership_fencing -> <<"ownership_fencing"/utf8>>; crash_restart_failover -> <<"crash_restart_failover"/utf8>>; resume_policy_matrix -> <<"resume_policy_matrix"/utf8>>; reconnect_continuity_slo -> <<"reconnect_continuity_slo"/utf8>> end. -file("src/lightspeed/ops/failover_harness.gleam", 84). ?DOC(" Stable pass/fail label.\n"). -spec pass_fail_label(scenario_outcome()) -> binary(). pass_fail_label(Outcome) -> case erlang:element(3, Outcome) of true -> <<"pass"/utf8>>; false -> <<"fail"/utf8>> end. -file("src/lightspeed/ops/failover_harness.gleam", 92). ?DOC(" Scenario signature.\n"). -spec signature(scenario_outcome()) -> binary(). signature(Outcome) -> erlang:element(5, Outcome). -file("src/lightspeed/ops/failover_harness.gleam", 97). ?DOC(" Scenario accessor.\n"). -spec scenario(scenario_outcome()) -> scenario(). scenario(Outcome) -> erlang:element(2, Outcome). -file("src/lightspeed/ops/failover_harness.gleam", 102). ?DOC(" Scenario determinism accessor.\n"). -spec deterministic(scenario_outcome()) -> boolean(). deterministic(Outcome) -> erlang:element(4, Outcome). -file("src/lightspeed/ops/failover_harness.gleam", 107). ?DOC(" Report outcomes.\n"). -spec outcomes(report()) -> list(scenario_outcome()). outcomes(Report) -> erlang:element(2, Report). -file("src/lightspeed/ops/failover_harness.gleam", 112). ?DOC(" Failed scenario count.\n"). -spec failed_scenarios(report()) -> integer(). failed_scenarios(Report) -> erlang:element(3, Report). -file("src/lightspeed/ops/failover_harness.gleam", 117). ?DOC(" Nondeterministic scenario count.\n"). -spec nondeterministic_failures(report()) -> integer(). nondeterministic_failures(Report) -> erlang:element(4, Report). -file("src/lightspeed/ops/failover_harness.gleam", 481). -spec bool_label(boolean()) -> binary(). bool_label(Value) -> case Value of true -> <<"true"/utf8>>; false -> <<"false"/utf8>> end. -file("src/lightspeed/ops/failover_harness.gleam", 122). ?DOC(" Stable report signature.\n"). -spec report_signature(report()) -> binary(). report_signature(Report) -> Entries = gleam@list:map( erlang:element(2, Report), fun(Outcome) -> <<<<<<<<<<<<(scenario_label(erlang:element(2, Outcome)))/binary, "="/utf8>>/binary, (pass_fail_label(Outcome))/binary>>/binary, ":deterministic="/utf8>>/binary, (bool_label(erlang:element(4, Outcome)))/binary>>/binary, ":"/utf8>>/binary, (erlang:element(5, Outcome))/binary>> end ), join_with(<<";"/utf8>>, Entries). -file("src/lightspeed/ops/failover_harness.gleam", 138). ?DOC(" Deterministic snapshot signature for fixture drift gates.\n"). -spec snapshot_signature() -> binary(). snapshot_signature() -> <<<<<<"m27.snapshot.v"/utf8, (erlang:integer_to_binary(1))/binary>>/binary, "|"/utf8>>/binary, (report_signature(run_matrix()))/binary>>. -file("src/lightspeed/ops/failover_harness.gleam", 146). ?DOC(" Deterministic markdown report for fixture scripts.\n"). -spec snapshot_report_markdown() -> binary(). snapshot_report_markdown() -> Report = run_matrix(), Failed = failed_scenarios(Report), Nondeterministic = nondeterministic_failures(Report), Status = case (Failed =:= 0) andalso (Nondeterministic =:= 0) of true -> <<"OK"/utf8>>; false -> <<"FAIL"/utf8>> end, <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<"# Failover Fixture Report\n\n"/utf8, "snapshot_version: "/utf8>>/binary, (erlang:integer_to_binary( 1 ))/binary>>/binary, "\n"/utf8>>/binary, "status: "/utf8>>/binary, Status/binary>>/binary, "\n"/utf8>>/binary, "failed_scenarios: "/utf8>>/binary, (erlang:integer_to_binary( Failed ))/binary>>/binary, "\n"/utf8>>/binary, "nondeterministic_failures: "/utf8>>/binary, (erlang:integer_to_binary(Nondeterministic))/binary>>/binary, "\n\n"/utf8>>/binary, "snapshot_signature: "/utf8>>/binary, (snapshot_signature())/binary>>/binary, "\n\n"/utf8>>/binary, "report_signature: "/utf8>>/binary, (report_signature(Report))/binary>>/binary, "\n"/utf8>>.