defmodule Retryable.Work do require Logger defstruct [ :id, :work, :on_success, :on_error, :on_timeout, :notify, :timeout, :attempts, :enqueued_at, :started_at, :ended_at, :timer, :monitor, :pid ] def new(opts) do %__MODULE__{ id: UUID.uuid4, attempts: 0, work: Keyword.fetch!(opts, :work), notify: Keyword.get(opts, :notify), timeout: Keyword.get(opts, :timeout), on_success: Keyword.get(opts, :on_success, fn (_, result) -> {:return, result} end), on_error: Keyword.get(opts, :on_error, fn (_, error) -> {:fail, error} end), on_timeout: Keyword.get(opts, :on_timeout, fn (_, :timeout) -> {:fail, :timeout} end) } end def enqueued(work) do %{work | enqueued_at: :erlang.monotonic_time(:millisecond) } end def started(work) do %{work | started_at: :erlang.monotonic_time(:millisecond) } end def ended(work) do %{work | ended_at: :erlang.monotonic_time(:millisecond) } end def next_attempt(work) do %{work | attempts: work.attempts + 1, monitor: nil, pid: nil, timer: nil } end def process(work) do log(work, "Starting work, attempt ##{work.attempts + 1}.") my_pid = self() {pid, monitor} = spawn_monitor(fn -> receive do :begin_work -> result = work.work.() send(my_pid, {:finished, result}) end end) timer = if work.timeout do Process.send_after(my_pid, :timeout, work.timeout) end work = %{work | timer: timer, pid: pid, monitor: monitor} send(work.pid, :begin_work) response = await_response(work) {work, result} = process_response(work, response) process_result(work, result) end defp await_response(work) do monitor = work.monitor pid = work.pid receive do {:finished, {:ok, result}} -> cancel_timer(work) await_monitor_down(work, :normal) {:ok, result} {:finished, {:error, error}} -> cancel_timer(work) await_monitor_down(work, :normal) {:error, error} {:finished, unexpected_response} -> raise "Expected `work` to return with either `{:ok, result}` or `{:error, error}`, got: #{inspect unexpected_response}." :timeout -> Process.exit(work.pid, :kill) await_monitor_down(work, :killed) :timeout {:DOWN, ^monitor, :process, ^pid, reason} -> cancel_timer(work) {:error, reason} end end defp process_response(work, {:ok, result}) do work = next_attempt(work) {work, work.on_success.(work.attempts, result)} end defp process_response(work, {:error, error}) do work = next_attempt(work) {work, work.on_error.(work.attempts, error)} end defp process_response(work, :timeout) do work = next_attempt(work) {work, work.on_timeout.(work.attempts, :timeout)} end defp process_result(work, {:return, result}) do work = ended(work) log(work, "Finished. Queuing for: #{work.started_at - work.enqueued_at}ms Processing for: #{work.ended_at - work.started_at}ms Attempts: #{work.attempts}.") notify(work, {:ok, result}) end defp process_result(work, {:retry, time_in_ms}) do log(work, "Retrying in #{time_in_ms}ms.") :timer.sleep(time_in_ms) process(work) end defp process_result(work, {:fail, error}) do log(work, "Failed, error: #{inspect error}.") notify(work, {:error, error}) end defp process_result(_work, invalid_response) do raise "Expected either: {:return, result}, {:retry, time_in_ms} or {:fail, error}, got: #{inspect invalid_response}." end defp await_monitor_down(work, reason) do monitor = work.monitor pid = work.pid receive do {:DOWN, ^monitor, :process, ^pid, ^reason} -> nil other_message -> log(work, "Unexpectedly recieved: #{inspect other_message}") await_monitor_down(work, reason) end end defp log(work, message) do Logger.info("[Retryable][#{work.id}] #{message}") end defp cancel_timer(%Retryable.Work{timer: nil}), do: nil defp cancel_timer(%Retryable.Work{timer: timer}), do: Process.cancel_timer(timer) defp notify(work, result) do if is_pid(work.notify) do send(work.notify, {work.id, result}) end end end