Electric.ShapeCache.Storage behaviour (electric v1.0.17)

View Source

Summary

Callbacks

Append log items from one transaction to the log.

Clean up snapshots/logs for a shape handle

Initialise shape-specific opts from the shared, global, configuration

Retrieve all stored shapes

Get the last exclusive offset of the chunk starting from the given offset.

Get the current pg_snapshot and offset for the shape storage.

Get stream of the log for a shape since a given offset

Get the total disk usage for all shapes

Run any initial setup tasks

Make a new snapshot for a shape handle based on the meta information about the table and a stream of plain string rows

Store the shape definition

Validate and initialise storage base configuration from application configuration

Check if snapshot for a given shape handle already exists

Start any processes required to run the storage backend

Clean up snapshots/logs for a shape handle by deleting whole directory.

Types

compiled_opts()

@type compiled_opts() :: term()

log()

log_item()

@type log_item() ::
  {Electric.Replication.LogOffset.t(), key :: String.t(),
   operation_type :: operation_type(), Electric.Shapes.Querying.json_iodata()}
  | {:chunk_boundary | Electric.Replication.LogOffset.t()}

offset()

@type offset() :: Electric.Replication.LogOffset.t()

operation_type()

@type operation_type() :: :insert | :update | :delete

pg_snapshot()

@type pg_snapshot() :: %{
  xmin: pos_integer(),
  xmax: pos_integer(),
  xip_list: [pos_integer()],
  filter_txns?: boolean()
}

row()

@type row() :: list()

shape_handle()

@type shape_handle() :: Electric.ShapeCacheBehaviour.shape_handle()

shape_opts()

@type shape_opts() :: term()

shape_storage()

@type shape_storage() :: {module(), shape_opts()}

storage()

@type storage() :: {module(), compiled_opts()}

xmin()

@type xmin() :: Electric.ShapeCacheBehaviour.xmin()

Callbacks

append_to_log!(t, shape_opts)

@callback append_to_log!(Enumerable.t(log_item()), shape_opts()) :: :ok | no_return()

Append log items from one transaction to the log.

Each storage implementation is responsible for handling transient errors using some retry strategy.

If the backend fails to write within the expected time, or some other error occurs, then this should raise.

cleanup!(shape_opts)

@callback cleanup!(shape_opts()) :: :ok

Clean up snapshots/logs for a shape handle

for_shape(shape_handle, compiled_opts)

@callback for_shape(shape_handle(), compiled_opts()) :: shape_opts()

Initialise shape-specific opts from the shared, global, configuration

get_all_stored_shapes(compiled_opts)

@callback get_all_stored_shapes(compiled_opts()) ::
  {:ok, %{required(shape_handle()) => Electric.Shapes.Shape.t()}}
  | {:error, term()}

Retrieve all stored shapes

get_chunk_end_log_offset(t, shape_opts)

@callback get_chunk_end_log_offset(Electric.Replication.LogOffset.t(), shape_opts()) ::
  Electric.Replication.LogOffset.t() | nil

Get the last exclusive offset of the chunk starting from the given offset.

If chunk has not finished accumulating, nil is returned.

If chunk has finished accumulating, the last offset of the chunk is returned.

get_current_position(shape_opts)

@callback get_current_position(shape_opts()) ::
  {:ok, offset(), pg_snapshot() | nil} | {:error, term()}

Get the current pg_snapshot and offset for the shape storage.

If the instance is new, then it MUST return {LogOffset.first(), nil}.

get_log_stream(offset, max_offset, shape_opts)

@callback get_log_stream(
  offset :: Electric.Replication.LogOffset.t(),
  max_offset :: Electric.Replication.LogOffset.t(),
  shape_opts()
) :: log()

Get stream of the log for a shape since a given offset

get_total_disk_usage(compiled_opts)

@callback get_total_disk_usage(compiled_opts()) :: non_neg_integer()

Get the total disk usage for all shapes

initialise(shape_opts)

@callback initialise(shape_opts()) :: :ok

Run any initial setup tasks

make_new_snapshot!(json_result_stream, shape_opts)

@callback make_new_snapshot!(
  Electric.Shapes.Querying.json_result_stream(),
  shape_opts()
) :: :ok

Make a new snapshot for a shape handle based on the meta information about the table and a stream of plain string rows

Should raise an error if making the snapshot had failed for any reason.

mark_snapshot_as_started(shape_opts)

@callback mark_snapshot_as_started(shape_opts()) :: :ok

set_pg_snapshot(pg_snapshot, shape_opts)

@callback set_pg_snapshot(pg_snapshot(), shape_opts()) :: :ok

set_shape_definition(t, shape_opts)

@callback set_shape_definition(Electric.Shapes.Shape.t(), shape_opts()) :: :ok

Store the shape definition

shared_opts(term)

@callback shared_opts(term()) :: compiled_opts()

Validate and initialise storage base configuration from application configuration

snapshot_started?(shape_opts)

@callback snapshot_started?(shape_opts()) :: boolean()

Check if snapshot for a given shape handle already exists

start_link(shape_opts)

@callback start_link(shape_opts()) :: GenServer.on_start()

Start any processes required to run the storage backend

unsafe_cleanup!(shape_opts)

@callback unsafe_cleanup!(shape_opts()) :: :ok

Clean up snapshots/logs for a shape handle by deleting whole directory.

Does not require any extra storage processes to be running, but should only be used if the shape is known to not be in use to avoid concurrency issues.

Functions

child_spec(arg)

@spec child_spec(shape_storage()) :: Supervisor.child_spec()

compact(arg)

compact(arg, offset)