%% Copyright 2021-2024 snabbkaffe contributors %% Copyright 2019-2020 Klarna Bank AB %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. %% You may obtain a copy of the License at %% %% http://www.apache.org/licenses/LICENSE-2.0 %% %% Unless required by applicable law or agreed to in writing, software %% distributed under the License is distributed on an "AS IS" BASIS, %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. %% @doc This module implements "nemesis" process that injects faults %% into system under test in order to test its fault-tolerance. %% %% == Usage == %% %% === Somewhere in the tested code === %% %% ``` %% ?maybe_crash(kind1, #{data1 => Foo, field2 => Bar}) %% ''' %% %% === Somewhere in the run stage === %% %% ``` %% ?inject_crash( #{?snk_kind := kind1, data1 := 42} %% , snabbkaffe_nemesis:always_crash() %% ) %% ''' %% @end -module(snabbkaffe_nemesis). -include("snabbkaffe_internal.hrl"). -behaviour(gen_server). %% API -export([ start_link/0 , inject_crash/2 , inject_crash/3 , fix_crash/1 , cleanup/0 , maybe_crash/2 %% Delay , force_ordering/3 , maybe_delay/1 %% Failure scenarios , always_crash/0 , recover_after/1 , random_crash/1 , periodic_crash/3 ]). -export_type([fault_scenario/0]). %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2]). -define(SERVER, ?MODULE). -define(ERROR_TAB, snabbkaffe_injected_errors). -define(STATE_TAB, snabbkaffe_fault_states). -define(DELAY_TAB, snabbkaffe_injected_delays). -define(SINGLETON_KEY, 0). %%%=================================================================== %%% Types %%%=================================================================== %% Type of fault patterns, such as "always fail", "fail randomly" or %% "recover after N attempts" %% %% This type is pretty magical. For performance reasons, state of the %% failure scenario is encoded as an integer counter, that is %% incremented every time the scenario is run. (BEAM VM can do this %% atomically and fast). Therefore "failure scenario" should map %% integer to boolean. -opaque fault_scenario() :: fun((integer()) -> boolean()). -type fault_key() :: term(). %% State of failure point (it's a simple counter, see above comment): %-type fault_state() :: {fault_key(), integer()}. %% Injected error: -record(fault, { reference :: reference() , predicate :: snabbkaffe:predicate() , scenario :: fault_scenario() , reason :: term() }). %% Injected delay: -record(delay, { reference :: reference() , delay_predicate :: snabbkaffe:predicate() % Event that is being delayed , continue_predicate :: snabbkaffe:predicate2() % Event that unlocks execution of the delayed process , n_events :: non_neg_integer() }). %% Currently this gen_server just holds the ets tables and %% synchronizes writes to the fault table, but in the future it may be %% used to mess up the system in more interesting ways -record(s, { injected_errors :: ets:tid() , fault_states :: ets:tid() , injected_delays :: ets:tid() }). %%%=================================================================== %%% API %%%=================================================================== %% @private Start the server start_link() -> gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). %% @equiv inject_crash(Predicate, Scenario, notmyday) -spec inject_crash(snabbkaffe:predicate(), fault_scenario()) -> reference(). inject_crash(Predicate, Scenario) -> inject_crash(Predicate, Scenario, notmyday). %% @doc Inject crash into the system -spec inject_crash(snabbkaffe:predicate(), fault_scenario(), term()) -> reference(). inject_crash(Predicate, Scenario, Reason) -> Ref = make_ref(), Crash = #fault{ reference = Ref , predicate = Predicate , scenario = Scenario , reason = Reason }, ok = gen_server:call(?SERVER, {inject_crash, Crash}, infinity), Ref. %% @doc Remove injected fault -spec fix_crash(reference()) -> ok. fix_crash(Ref) -> gen_server:call(?SERVER, {fix_crash, Ref}, infinity). %% @doc Remove all injected crashes -spec cleanup() -> ok. cleanup() -> gen_server:call(?SERVER, cleanup, infinity). %% @private Check if there are any injected crashes that match this %% data, and respond with the crash reason if so. -spec maybe_crash(fault_key(), map()) -> ok. maybe_crash(Key, Data) -> [begin NewVal = ets:update_counter(?STATE_TAB, Key, {2, 1}, {Key, 0}), %% Run fault_scenario function to see if we need to crash this %% time: case S(NewVal) of true -> snabbkaffe_collector:tp( debug , Data#{ crash_kind => Key , ?snk_kind => snabbkaffe_crash } , #{} ), exit(R); false -> ok end end || {_, Faults} <- lookup_singleton(?ERROR_TAB), #fault{ scenario = S , reason = R , predicate = Pred } <- Faults, Pred(Data)], ok. %% @doc Inject scheduling into the system -spec force_ordering(snabbkaffe:predicate(), non_neg_integer(), snabbkaffe:predicate2()) -> reference(). force_ordering(DelayPredicate, NEvents, ContinuePredicate) when NEvents > 0 -> Ref = make_ref(), Delay = #delay{ reference = Ref , delay_predicate = DelayPredicate , continue_predicate = ContinuePredicate , n_events = NEvents }, ok = gen_server:call(?SERVER, {force_ordering, Delay}, infinity), Ref. %% @private Check if the trace point should be delayed. -spec maybe_delay(map()) -> ok. maybe_delay(Event) -> [snabbkaffe_collector:block_until( {fun(WU) -> ContP(Event, WU) end, NEvents} , infinity , infinity ) || {_, Delays} <- lookup_singleton(?DELAY_TAB), #delay{ continue_predicate = ContP , delay_predicate = DelayP , n_events = NEvents } <- Delays, DelayP(Event)], ok. %%%=================================================================== %%% Fault scenarios %%%=================================================================== -spec always_crash() -> fault_scenario(). always_crash() -> fun(_) -> true end. %% @doc Crash N times consequently, then recover -spec recover_after(non_neg_integer()) -> fault_scenario(). recover_after(Times) -> fun(X) -> X =< Times end. %% @doc Crash with probability P -spec random_crash(float()) -> fault_scenario(). random_crash(CrashProbability) -> fun(X) -> Range = 2 bsl 16, %% Turn a sequential number into a sufficiently plausible %% pseudorandom one: Val = erlang:phash2(X, Range), Val < CrashProbability * Range end. %% @doc A type of fault that occurs and fixes periodically. -spec periodic_crash(integer(), float(), float()) -> fault_scenario(). periodic_crash(Period, DutyCycle, Phase) -> DC = DutyCycle * Period, P = round(Phase/(math:pi()*2)*Period), fun(X) -> (X + P) rem Period >= DC end. %%%=================================================================== %%% gen_server callbacks %%%=================================================================== %% @private init([]) -> ST = ets:new(?STATE_TAB, [ named_table , {write_concurrency, true} , {read_concurrency, true} , public ]), FT = ets:new(?ERROR_TAB, [ named_table , {write_concurrency, false} , {read_concurrency, true} , protected ]), DT = ets:new(?DELAY_TAB, [ named_table , {write_concurrency, false} , {read_concurrency, true} , public ]), init_data(), {ok, #s{ injected_errors = FT , fault_states = ST , injected_delays = DT }}. %% @private handle_call({inject_crash, Crash}, _From, State) -> [{_, Faults}] = ets:lookup(?ERROR_TAB, ?SINGLETON_KEY), ets:insert(?ERROR_TAB, {?SINGLETON_KEY, [Crash|Faults]}), {reply, ok, State}; handle_call({fix_crash, Ref}, _From, State) -> [{_, Faults0}] = ets:lookup(?ERROR_TAB, ?SINGLETON_KEY), Faults = lists:keydelete(Ref, #fault.reference, Faults0), ets:insert(?ERROR_TAB, {?SINGLETON_KEY, Faults}), {reply, ok, State}; handle_call({force_ordering, Delay}, _From, State) -> [{_, Delays}] = ets:lookup(?DELAY_TAB, ?SINGLETON_KEY), ets:insert(?DELAY_TAB, {?SINGLETON_KEY, [Delay|Delays]}), {reply, ok, State}; handle_call(cleanup, _From, State) -> ets:delete_all_objects(?ERROR_TAB), ets:delete_all_objects(?DELAY_TAB), ets:delete_all_objects(?STATE_TAB), init_data(), {reply, ok, State}; handle_call(_Request, _From, State) -> {reply, ok, State}. %% @private handle_cast(_Request, State) -> {noreply, State}. %% @private handle_info(_Info, State) -> {noreply, State}. %% @private terminate(_Reason, _State) -> ok. %%%=================================================================== %%% Internal functions %%%=================================================================== -spec lookup_singleton(atom()) -> [{?SINGLETON_KEY, term()}]. lookup_singleton(Table) -> try ets:lookup(Table, ?SINGLETON_KEY) of Val -> Val catch error:badarg -> %% Table doesn't exist. Probably it happens because snabbkaffe %% nemesis was stopped while the SUT is running. Ignore it and %% let the SUT do its thing. [] end. -spec init_data() -> ok. init_data() -> ets:insert(?ERROR_TAB, {?SINGLETON_KEY, []}), ets:insert(?DELAY_TAB, {?SINGLETON_KEY, []}), ok.