ExternalService Cheatsheet

Module front door (recommended)

defmodule MyApp.Api do
  use ExternalService,
    circuit_breaker: [tolerate: 5, within: :timer.seconds(1), reset: :timer.seconds(5)],
    rate_limit: [limit: 100, per: :timer.seconds(1), wait: :timer.seconds(1)],
    retry: [max_attempts: 5, backoff: :exponential, jitter: true]

  def fetch(id), do: call(fn -> HTTP.get("/things/#{id}") end)
end

Start it under a supervisor:

children = [MyApp.Api]
Supervisor.start_link(children, strategy: :one_for_one)

Functional API

ExternalService.start(:payments,
  circuit_breaker: [tolerate: 5, within: 1_000, reset: 5_000],
  retry: [max_attempts: 3]
)

ExternalService.call(:payments, fn -> charge() end)

Synchronous

MyApp.Api.call(fn -> work() end)
MyApp.Api.call([max_attempts: 2], fn -> work() end)  # per-call retry opts
MyApp.Api.call!(fn -> work() end)                     # raises on failure

Async / parallel

task = MyApp.Api.call_async(fn -> work() end)
Task.await(task)

ids
|> MyApp.Api.call_async_stream(fn id -> fetch(id) end)
|> Enum.to_list()

Return values

call fn ->
  case HTTP.get(url) do
    {:ok, %{status: 200} = r}            -> {:ok, r}
    {:ok, %{status: s}} when s in 500..599 -> {:retry, s}  # retry
    {:ok, %{status: 429}}                -> :retry          # retry
    other                                -> other           # success
  end
end

Key rule

Options

circuit_breaker: [
  tolerate: 5,                # failed ATTEMPTS per window (default 10);
                              # retries melt too, so tolerate ≈
                              # failing calls × max_attempts
  within: :timer.seconds(1),  # window ms (default 10_000)
  reset: :timer.seconds(5)    # ms open before reset (default 60_000)
]

circuit_breaker: [tolerate: :infinity]   # no breaker at all;
                                         # never opens, holds no state

Trip the cluster together

circuit_breaker: [
  tolerate: 5,
  backend: ExternalService.CircuitBreaker.Cluster
]

# narrow the broadcast:
backend: {ExternalService.CircuitBreaker.Cluster,
          nodes: &MyApp.api_nodes/0}

Default breaker is node-local (each node trips on its own).

Introspect & reset

MyApp.Api.available?()   # breaker closed?
MyApp.Api.blown?()       # breaker open?
MyApp.Api.reset()        # force closed

ExternalService.all_available?([:a, :b])

Drive it directly

# count a failure that happened outside call/3
ExternalService.CircuitBreaker.melt(:api)

# :ok | :blown | :not_started
ExternalService.CircuitBreaker.ask(:api)

Options

rate_limit: [
  limit: 100,               # calls per window
  per: :timer.seconds(1),   # window ms
  wait: :timer.seconds(1)   # :infinity | ms | false
]                           # unset waits forever, and warns

rate_limit: [limit: :infinity, per: 1_000]   # no limiter at all

Burst up to limit, then paced at per / limit.

Bound the wait

rate_limit: [limit: 100, per: 1_000, wait: 2_000]

# budget exhausted -> function never runs:
{:error, %ExternalService.RateLimited{
  context: %{retry_after: ms}}}   # http_status 429

wait: false fails immediately. Never melts the breaker; never retried.

Share a limit across a cluster

defmodule MyApp.RateLimit do
  use Hammer, backend: Hammer.Redis
end

rate_limit: [
  limit: 100, per: 1_000,
  backend: {ExternalService.RateLimiter.Hammer,
            module: MyApp.RateLimit}
]

Default backend is node-local: N nodes ⇒ up to N × limit.

Ask, and spend, directly

ExternalService.rate_limited?(:api)      # boolean, consumes nothing

ExternalService.RateLimiter.peek(:api)   # :ok | {:wait, ms}

# spend budget for a call made some other way
ExternalService.RateLimiter.request(:api)

Options

concurrency: [
  limit: 25,                        # calls in flight at once
  reclaim_after: :timer.seconds(30),# slot expiry; must exceed
                                    # your client timeout
  wait: 50                          # ms | false (default)
]

Over the limit, calls shed rather than queue. A short :wait absorbs bursts; :infinity is rejected.

Errors & introspection

{:error, %ExternalService.ServiceSaturated{
  context: %{limit: l, in_flight: n}}}   # http_status 503

ExternalService.saturated?(:svc)
ExternalService.Concurrency.in_flight(:svc)

Saturation does not melt the breaker and is not retried.

All options

retry: [
  backoff: :exponential,   # or :linear
  base: 100,               # initial delay ms (default 10)
  factor: 1,               # :linear growth factor
  cap: :timer.seconds(2),  # max single delay
  max_attempts: 5,         # attempt count bound (or :infinity)
  expiry: :timer.seconds(10), # time budget ms (or :infinity)
  jitter: true,            # ±10%, or a float proportion
  retry_on: &match?({:error, _}, &1), # predicate over the result
  retry_exceptions: []     # exception modules to retry
]

Recipes

# Fast, bounded
retry: [max_attempts: 3, backoff: :linear, base: 50]

# Resilient HTTP default
retry: [backoff: :exponential, base: 100, cap: 2_000,
        max_attempts: 5, jitter: true]

# Retry a transient exception
retry: [retry_exceptions: [MyApp.TransientError]]

# Retry on the result of an unmodified function
retry: [retry_on: &match?({:error, %{status: 500}}, &1)]

# Deliberately unbounded (silences the start/2 warning)
retry: [max_attempts: :infinity, cap: 30_000]

Returned by call

case MyApp.Api.fetch(id) do
  {:ok, v} -> v
  {:error, %ExternalService.RetriesExhausted{}} -> degrade()
  {:error, %ExternalService.CircuitBreakerOpen{}} -> degrade()
  {:error, %ExternalService.RateLimited{}} -> shed()
  {:error, reason} -> {:error, reason}  # your own error
end

Raised by call!

rescue
  e in [ExternalService.RetriesExhausted,
        ExternalService.CircuitBreakerOpen] ->
    send_resp(conn, 503, "")

Events

[:external_service, :call, :start]
[:external_service, :call, :stop]
[:external_service, :call, :exception]
[:external_service, :call, :retry]
[:external_service, :circuit_breaker, :blown]
[:external_service, :rate_limit, :sleep]
[:external_service, :concurrency, :rejected]
[:external_service, :concurrency, :waited]

Attach

:telemetry.attach_many(
  "es-handler",
  [[:external_service, :call, :retry],
   [:external_service, :circuit_breaker, :blown]],
  &MyApp.Telemetry.handle/4,
  nil
)

Make a service inert

# test.exs child spec override
{MyApp.Api,
 circuit_breaker: [tolerate: :infinity],
 rate_limit: [limit: :infinity],
 retry: [max_attempts: 1]}

Both :infinity keys remove state, so nothing leaks between tests.

Isolate per test

setup context do
  service = :"#{context.module}.#{context.test}"
  on_exit(fn -> ExternalService.stop(service) end)
  {:ok, service: service}
end

Or reset shared state

setup do
  MyApp.Api.reset_all()   # breaker + limiter
  :ok
end

reset/0 clears only the breaker.

Force the failure paths

# open the breaker (tolerate + 1 melts)
Enum.each(0..tolerate, fn _ ->
  ExternalService.CircuitBreaker.melt(svc)
end)

# spend the rate limit budget
ExternalService.RateLimiter.request(svc)

# fail a fraction of calls
circuit_breaker: [fault_injection: 0.25]

Keep it off the clock

retry: [max_attempts: 3, base: 0]   # instant retries
rate_limit: [wait: false]           # shed, don't wait

Never sleep_function: fn _ -> :ok end — it busy-waits.