defmodule Aerospike.Router do @moduledoc false # Routes a wire-encoded request to the correct node based on the key's partition. # # Flow: key -> partition_id -> ETS partition table -> node_name -> ETS nodes table # -> pool_pid -> NimblePool.checkout! -> send wire + receive response. require Logger alias Aerospike.CircuitBreaker alias Aerospike.Connection alias Aerospike.Error alias Aerospike.Key alias Aerospike.PartitionFilter alias Aerospike.Tables @doc """ Resolves routing for `key`, checks out a pooled connection, sends `wire`, and returns the AS_MSG body. Returns `{:ok, body, node_name}` on success or `{:error, %Aerospike.Error{}}`. `node_name` is the Aerospike cluster node string (e.g. from the `node` info field). """ @spec run(atom(), Key.t(), iodata(), keyword()) :: {:ok, binary(), String.t()} | {:error, Error.t()} def run(conn_name, %Key{} = key, wire, opts \\ []) when is_atom(conn_name) do ro = routing_opts(opts) checkout_timeout = Keyword.get(ro, :pool_checkout_timeout, 5_000) replica_index = Keyword.get(ro, :replica_index, 0) with :ok <- check_ready(conn_name), {:ok, pool_pid, node_name} <- resolve_pool(conn_name, key, replica_index), :ok <- CircuitBreaker.allow_request?(conn_name, node_name), {:ok, body} <- checkout_and_request(pool_pid, wire, checkout_timeout, {conn_name, node_name}) do {:ok, body, node_name} else {:error, %Error{} = e} -> {:error, e} end end @doc """ Groups `keys` by the cluster node that owns each key's partition (for batch fan-out). Returns `{:ok, %{node_name => [{original_index, key}, ...]}}` preserving list order via `original_index` (0-based), or `{:error, %Aerospike.Error{}}`. """ @typedoc """ Keys grouped per node with the `NimblePool` pid used to reach that node. Invariant: for a given node name, every entry in the map shares the same `pool_pid` (enforced when building the map; mismatch yields `{:error, %Aerospike.Error{}}`). """ @type node_batch_group :: %{ pool_pid: pid(), entries: [{non_neg_integer(), Key.t()}] } @spec group_by_node(atom(), [Key.t()], keyword()) :: {:ok, %{String.t() => node_batch_group()}} | {:error, Error.t()} def group_by_node(conn_name, keys, opts \\ []) when is_atom(conn_name) and is_list(keys) do replica_index = Keyword.get(routing_opts(opts), :replica_index, 0) with :ok <- check_ready(conn_name), {:ok, acc} <- fold_keys_into_node_groups(conn_name, keys, replica_index) do {:ok, reverse_group_entries(acc)} end end defp fold_keys_into_node_groups(conn_name, keys, replica_index) do keys |> Enum.with_index() |> Enum.reduce_while({:ok, %{}}, fn {key, idx}, {:ok, acc} -> case resolve_pool(conn_name, key, replica_index) do {:ok, pool_pid, node_name} -> {:cont, {:ok, insert_batch_group_entry(acc, node_name, pool_pid, {idx, key})}} {:error, _} = err -> {:halt, err} end end) end defp insert_batch_group_entry(acc, node_name, pool_pid, entry) do Map.update(acc, node_name, %{pool_pid: pool_pid, entries: [entry]}, fn group -> %{group | entries: [entry | group.entries]} end) end defp reverse_group_entries(acc) do Map.new(acc, fn {node, %{pool_pid: p, entries: es}} -> {node, %{pool_pid: p, entries: Enum.reverse(es)}} end) end @doc """ Expands a `%PartitionFilter{}` into full partition IDs and cursor partial entries. When `pf.partitions` is non-empty (pagination resume), entries with a non-nil `digest` are returned in `partial` for DIGEST_ARRAY wire encoding; others are full partitions (PID_ARRAY). When `partitions` is empty, IDs are `pf.begin` through `pf.begin + pf.count - 1`. Passing `nil` is equivalent to all partitions (`0..4095`). """ @spec expand_partition_filter(PartitionFilter.t() | nil) :: {full_ids :: [non_neg_integer()], partial :: [PartitionFilter.partition_entry()]} def expand_partition_filter(nil) do last = PartitionFilter.partition_count() - 1 {Enum.to_list(0..last), []} end def expand_partition_filter(%PartitionFilter{} = pf) do case pf.partitions do [] -> last = pf.begin + pf.count - 1 {Enum.to_list(pf.begin..last), []} parts when is_list(parts) -> Enum.split_with(parts, &partition_entry_full?/1) |> then(fn {full_entries, partial_entries} -> {Enum.map(full_entries, &entry_id/1), partial_entries} end) end end defp partition_entry_full?(entry) when is_map(entry) do Map.get(entry, :digest) == nil end defp entry_id(entry) when is_map(entry) do Map.fetch!(entry, :id) end @typedoc """ Scan/query partitions grouped per node with pool PID and PID vs digest partition lists. Invariant: for a given node name, `parts_full` and `parts_partial` belong to the same `pool_pid` (enforced when building the map; mismatch yields `{:error, %Aerospike.Error{}}`). """ @type node_partition_group :: %{ pool_pid: pid(), parts_full: [non_neg_integer()], parts_partial: [PartitionFilter.partition_entry()] } @doc """ Groups partition IDs by the cluster node that owns each partition. Used by scan/query to fan-out requests to the correct nodes. Each node receives only the partition IDs it owns in `parts_full`. `parts_partial` is reserved for cursor resume (populate via a follow-up merge or future API). Returns `{:ok, groups}` where `groups` maps node name to node info: %{ "node1" => %{pool_pid: pid, parts_full: [0, 1, 2], parts_partial: []}, "node2" => %{pool_pid: pid, parts_full: [3, 4, 5], parts_partial: []} } """ @spec group_partitions_by_node(atom(), String.t(), [non_neg_integer()], non_neg_integer()) :: {:ok, %{String.t() => node_partition_group()}} | {:error, Error.t()} def group_partitions_by_node(conn_name, namespace, partition_ids, replica_index \\ 0) when is_atom(conn_name) and is_binary(namespace) and is_list(partition_ids) and is_integer(replica_index) and replica_index >= 0 do with :ok <- check_ready(conn_name), {:ok, acc} <- fold_partition_ids_into_node_groups(conn_name, namespace, partition_ids, replica_index) do {:ok, finalize_partition_groups(acc)} end end defp fold_partition_ids_into_node_groups(conn_name, namespace, partition_ids, replica_index) do Enum.reduce_while(partition_ids, {:ok, %{}}, fn part_id, {:ok, acc} -> fold_one_partition(conn_name, namespace, part_id, replica_index, acc) end) end defp fold_one_partition(conn_name, namespace, part_id, replica_index, acc) do case resolve_pool_for_partition(conn_name, namespace, part_id, replica_index) do {:ok, pool_pid, node_name} -> case insert_partition_group(acc, node_name, pool_pid, part_id) do {:ok, new_acc} -> {:cont, {:ok, new_acc}} {:error, _} = err -> {:halt, err} end {:error, _} = err -> {:halt, err} end end defp insert_partition_group(acc, node_name, pool_pid, partition_id) do case Map.get(acc, node_name) do nil -> {:ok, Map.put(acc, node_name, %{ pool_pid: pool_pid, parts_full: [partition_id], parts_partial: [] })} %{pool_pid: ^pool_pid} = g -> {:ok, Map.put(acc, node_name, %{g | parts_full: [partition_id | g.parts_full]})} %{pool_pid: _} -> {:error, Error.from_result_code(:invalid_node, message: "inconsistent pool_pid for node #{inspect(node_name)} in partition map" )} end end defp finalize_partition_groups(acc) do Map.new(acc, fn {node, %{pool_pid: p, parts_full: ids, parts_partial: partial}} -> {node, %{ pool_pid: p, parts_full: Enum.reverse(ids), parts_partial: partial }} end) end @doc false @spec resolve_pool_for_partition(atom(), String.t(), non_neg_integer(), non_neg_integer()) :: {:ok, pid(), String.t()} | {:error, Error.t()} def resolve_pool_for_partition(name, namespace, partition_id, replica_index) do case :ets.lookup(Tables.partitions(name), {namespace, partition_id, replica_index}) do [] -> {:error, Error.from_result_code(:invalid_cluster_partition_map)} [{_, node_name}] -> lookup_pool(name, node_name) end end @doc """ Returns `{:ok, pool_pid, node_name}` for a randomly selected active node in the cluster. Returns `{:error, %Aerospike.Error{code: :invalid_node}}` if no active nodes exist. """ @spec random_node_pool(atom()) :: {:ok, pid(), String.t()} | {:error, Error.t()} def random_node_pool(conn_name) when is_atom(conn_name) do with :ok <- check_ready(conn_name) do # ETS row shape: `t:Aerospike.Cluster.node_row/0` (`insert_node_registry/5`). conn_name |> Tables.nodes() |> :ets.tab2list() |> Enum.filter(fn {_name, row} -> Map.get(row, :active) end) |> case do [] -> {:error, Error.from_result_code(:invalid_node)} active_nodes -> {node_name, %{pool_pid: pool_pid}} = Enum.random(active_nodes) {:ok, pool_pid, node_name} end end end @doc """ Returns `{:ok, pool_pid, node_name}` for the named node in the cluster. Returns `{:error, %Aerospike.Error{code: :invalid_node}}` if the node is not found. """ @spec node_pool(atom(), String.t()) :: {:ok, pid(), String.t()} | {:error, Error.t()} def node_pool(conn_name, node_name) when is_atom(conn_name) and is_binary(node_name) do with :ok <- check_ready(conn_name) do lookup_pool(conn_name, node_name) end end @doc """ Checks out a connection from `pool_pid`, sends `commands` as an info request, and returns the response map. """ @type breaker_ctx :: {atom(), String.t()} | nil @spec checkout_and_info(pid(), [String.t()], non_neg_integer(), breaker_ctx()) :: {:ok, map()} | {:error, Error.t()} def checkout_and_info(pool_pid, commands, checkout_timeout, breaker_ctx \\ nil) when is_pid(pool_pid) and is_list(commands) and is_integer(checkout_timeout) and checkout_timeout >= 0 do do_checkout( pool_pid, checkout_timeout, fn conn -> case Connection.request_info(conn, commands) do {:ok, conn2, map} -> {{:ok, map}, conn2} {:error, reason} -> maybe_record_breaker(breaker_ctx, :network_error) e = Error.from_result_code(:network_error, message: inspect(reason)) {{:error, e}, :close} end end, breaker_ctx ) end @doc """ Checks out a connection from `pool_pid`, sends `wire`, reads one response body. For single-record commands (put, get, delete, exists, touch, operate). """ @spec checkout_and_request(pid(), iodata(), non_neg_integer(), breaker_ctx()) :: {:ok, binary()} | {:error, Error.t()} def checkout_and_request(pool_pid, wire, checkout_timeout, breaker_ctx \\ nil) when is_pid(pool_pid) and is_integer(checkout_timeout) and checkout_timeout >= 0 do do_checkout( pool_pid, checkout_timeout, fn conn -> case Connection.request(conn, wire) do {:ok, conn2, _v, _t, body} -> {{:ok, body}, conn2} {:error, reason} -> maybe_record_breaker(breaker_ctx, :network_error) e = Error.from_result_code(:network_error, message: inspect(reason)) {{:error, e}, :close} end end, breaker_ctx ) end @doc """ Like `checkout_and_request/3` but reads a multi-frame (streaming) response. Batch, scan, and query commands return multiple proto frames per request. All frames are read until the INFO3_LAST sentinel and concatenated. """ @spec checkout_and_request_stream(pid(), iodata(), non_neg_integer(), breaker_ctx()) :: {:ok, binary()} | {:error, Error.t()} def checkout_and_request_stream(pool_pid, wire, checkout_timeout, breaker_ctx \\ nil) when is_pid(pool_pid) and is_integer(checkout_timeout) and checkout_timeout >= 0 do do_checkout( pool_pid, checkout_timeout, fn conn -> case Connection.request_stream(conn, wire) do {:ok, conn2, body} -> {{:ok, body}, conn2} {:error, reason} -> maybe_record_breaker(breaker_ctx, :network_error) e = Error.from_result_code(:network_error, message: inspect(reason)) {{:error, e}, :close} end end, breaker_ctx ) end defp do_checkout(pool_pid, checkout_timeout, fun, breaker_ctx) do NimblePool.checkout!( pool_pid, :checkout, fn _from, conn -> fun.(conn) end, checkout_timeout ) catch :exit, {:timeout, {NimblePool, :checkout, _}} -> maybe_record_breaker(breaker_ctx, :pool_timeout) {:error, Error.from_result_code(:pool_timeout)} :exit, {:noproc, {NimblePool, :checkout, _}} -> {:error, Error.from_result_code(:invalid_node)} :exit, reason -> maybe_record_breaker(breaker_ctx, :network_error) {:error, Error.from_result_code(:network_error, message: inspect(reason))} end defp maybe_record_breaker(nil, _reason), do: :ok defp maybe_record_breaker({conn_name, node_name}, reason) do CircuitBreaker.record_error(conn_name, node_name, reason) end defp routing_opts(opts) when is_list(opts) do base = Keyword.take(opts, [:pool_checkout_timeout]) ri = Keyword.get(opts, :replica_index) if is_integer(ri) and ri >= 0 do Keyword.put(base, :replica_index, ri) else put_replica_index(base, Keyword.get(opts, :replica)) end end defp put_replica_index(kw, nil), do: kw defp put_replica_index(kw, r) when is_integer(r) and r >= 0 do Keyword.put(kw, :replica_index, r) end defp put_replica_index(kw, a) when a in [:master, :sequence, :any] do Keyword.put(kw, :replica_index, replica_atom_to_index(a)) end defp replica_atom_to_index(:master), do: 0 defp replica_atom_to_index(:sequence) do Logger.warning("replica: :sequence not yet implemented, routing to master") 0 end defp replica_atom_to_index(:any) do Logger.warning("replica: :any not yet implemented, routing to master") 0 end # Guards against requests before the cluster has finished its initial tend. defp check_ready(name) do case :ets.lookup(Tables.meta(name), Tables.ready_key()) do [{_, true}] -> :ok _ -> {:error, Error.from_result_code(:cluster_not_ready)} end end # Looks up which node owns {namespace, partition_id, replica_index} in the # partitions ETS table, then finds that node's pool PID. defp resolve_pool(name, key, replica_index) do partition_id = Key.partition_id(key) ns = key.namespace case :ets.lookup(Tables.partitions(name), {ns, partition_id, replica_index}) do [] -> {:error, Error.from_result_code(:invalid_cluster_partition_map)} [{_, node_name}] -> lookup_pool(name, node_name) end end # ETS row shape: `t:Aerospike.Cluster.node_row/0` (`insert_node_registry/5`). defp lookup_pool(name, node_name) do case :ets.lookup(Tables.nodes(name), node_name) do [] -> {:error, Error.from_result_code(:invalid_node)} [{_, %{pool_pid: pid}}] when is_pid(pid) -> {:ok, pid, node_name} _ -> {:error, Error.from_result_code(:invalid_node)} end end end