ExZarr.Storage.Backend behaviour (ExZarr v1.1.0)

Behavior for implementing custom storage backends.

Storage backends handle the persistence and retrieval of Zarr array chunks and metadata. ExZarr provides built-in backends (:memory, :filesystem, :zip), and you can implement custom backends for other storage systems (S3, GCS, databases, etc.).

Behavior Callbacks

All storage backends must implement:

backend_id/0 - Returns the unique identifier atom for this backend
init/1 - Initializes the storage backend with configuration
open/1 - Opens an existing storage location
read_chunk/2 - Reads a chunk from storage
write_chunk/3 - Writes a chunk to storage
read_metadata/1 - Reads array metadata
write_metadata/3 - Writes array metadata
list_chunks/1 - Lists all chunk indices
delete_chunk/2 - Deletes a chunk (optional)
exists?/1 - Checks if storage location exists

Example: Custom Database Backend

defmodule MyApp.DatabaseStorage do
  @behaviour ExZarr.Storage.Backend

  @impl true
  def backend_id, do: :database

  @impl true
  def init(config) do
    # Initialize database connection
    {:ok, conn} = MyApp.DB.connect(config[:connection_string])
    {:ok, %{conn: conn, table: config[:table]}}
  end

  @impl true
  def read_chunk(state, chunk_index) do
    # Read chunk from database
    query = "SELECT data FROM #{state.table} WHERE chunk_index = $1"
    case MyApp.DB.query(state.conn, query, [encode_index(chunk_index)]) do
      {:ok, [row]} -> {:ok, row.data}
      {:ok, []} -> {:error, :not_found}
      {:error, reason} -> {:error, reason}
    end
  end

  @impl true
  def write_chunk(state, chunk_index, data) do
    # Write chunk to database
    query = "INSERT INTO #{state.table} (chunk_index, data) VALUES ($1, $2)
             ON CONFLICT (chunk_index) DO UPDATE SET data = $2"
    case MyApp.DB.query(state.conn, query, [encode_index(chunk_index), data]) do
      {:ok, _} -> :ok
      {:error, reason} -> {:error, reason}
    end
  end

  # ... implement other callbacks
end

# Register the custom backend
ExZarr.Storage.Registry.register(MyApp.DatabaseStorage)

# Use the custom backend
{:ok, array} = ExZarr.create(
  shape: {1000},
  chunks: {100},
  dtype: :float64,
  storage: :database,
  connection_string: "postgresql://...",
  table: "zarr_chunks"
)

Thread Safety

Backend implementations must be thread-safe. If your backend maintains mutable state, use appropriate synchronization mechanisms (Agent, GenServer, ETS, etc.).

Error Handling

Callbacks should return {:ok, result} on success or {:error, reason} on failure. The :not_found error is reserved for missing chunks/metadata.

Summary

Types

chunk_index()

config()

metadata()

state()

Callbacks

backend_id()

Returns the unique identifier for this storage backend.

delete_chunk(state, chunk_index)

Deletes a chunk from storage (optional).

exists?(config)

Checks if storage location exists.

init(config)

Initializes the storage backend with the given configuration.

list_chunks(state)

Lists all chunk indices in storage.

open(config)

Opens an existing storage location.

read_chunk(state, chunk_index)

Reads a chunk from storage.

read_metadata(state)

Reads array metadata from storage.

write_chunk(state, chunk_index, binary)

Writes a chunk to storage.

write_metadata(state, metadata, keyword)

Writes array metadata to storage.

Functions

implements?(module)

Helper function to check if a module implements the Backend behavior.

Types

chunk_index()

@type chunk_index() :: tuple()

config()

@type config() :: keyword() | map()

metadata()

@type metadata() :: ExZarr.Metadata.t()

state()

@type state() :: term()

Callbacks

backend_id()

@callback backend_id() :: atom()

Returns the unique identifier for this storage backend.

This atom is used when specifying storage: :backend_id in array creation.

Examples

def backend_id, do: :s3
def backend_id, do: :database

delete_chunk(state, chunk_index)

@callback delete_chunk(state(), chunk_index()) :: :ok | {:error, term()}

Deletes a chunk from storage (optional).

Parameters

state - Backend state
chunk_index - Tuple identifying the chunk

Returns

:ok - Delete successful or chunk didn't exist
{:error, reason} - Delete failed

Examples

def delete_chunk(state, chunk_index) do
  path = build_chunk_path(state.path, chunk_index)
  File.rm(path)
  :ok
end

exists?(config)

@callback exists?(config()) :: boolean()

Checks if storage location exists.

Parameters

config - Configuration map/keyword list

Returns

true if storage exists
false if storage doesn't exist

Examples

def exists?(config) do
  path = Keyword.fetch!(config, :path)
  File.exists?(path)
end

init(config)

@callback init(config()) :: {:ok, state()} | {:error, term()}

Initializes the storage backend with the given configuration.

Called when creating a new array. Should set up any necessary connections, create directories/tables, allocate resources, etc.

Parameters

config - Configuration map/keyword list containing backend-specific options

Returns

{:ok, state} - Initialization successful, returns backend state
{:error, reason} - Initialization failed

Examples

def init(config) do
  path = Keyword.fetch!(config, :path)
  File.mkdir_p!(path)
  {:ok, %{path: path}}
end

list_chunks(state)

@callback list_chunks(state()) :: {:ok, [chunk_index()]} | {:error, term()}

Lists all chunk indices in storage.

Parameters

state - Backend state

Returns

{:ok, [chunk_indices]} - List of chunk index tuples
{:error, reason} - List failed

Examples

def list_chunks(state) do
  {:ok, files} = File.ls(state.path)
  chunks = files
    |> Enum.filter(&is_chunk_file?/1)
    |> Enum.map(&parse_chunk_index/1)
  {:ok, chunks}
end

open(config)

@callback open(config()) :: {:ok, state()} | {:error, term()}

Opens an existing storage location.

Called when opening an existing array. Should verify the location exists and set up necessary connections.

Parameters

config - Configuration map/keyword list

Returns

{:ok, state} - Successfully opened, returns backend state
{:error, :not_found} - Storage location doesn't exist
{:error, reason} - Other error

Examples

def open(config) do
  path = Keyword.fetch!(config, :path)
  if File.exists?(path) do
    {:ok, %{path: path}}
  else
    {:error, :not_found}
  end
end

read_chunk(state, chunk_index)

@callback read_chunk(state(), chunk_index()) :: {:ok, binary()} | {:error, term()}

Reads a chunk from storage.

Parameters

state - Backend state from init/open
chunk_index - Tuple identifying the chunk (e.g., {0, 1})

Returns

{:ok, binary} - Chunk data (compressed)
{:error, :not_found} - Chunk doesn't exist
{:error, reason} - Read error

Examples

def read_chunk(state, chunk_index) do
  path = build_chunk_path(state.path, chunk_index)
  case File.read(path) do
    {:ok, data} -> {:ok, data}
    {:error, :enoent} -> {:error, :not_found}
    {:error, reason} -> {:error, reason}
  end
end

read_metadata(state)

@callback read_metadata(state()) :: {:ok, metadata()} | {:error, term()}

Reads array metadata from storage.

Parameters

state - Backend state

Returns

{:ok, metadata} - Metadata struct
{:error, :not_found} - Metadata doesn't exist
{:error, reason} - Read error

Examples

def read_metadata(state) do
  path = Path.join(state.path, ".zarray")
  case File.read(path) do
    {:ok, json} ->
      {:ok, parsed} = Jason.decode(json, keys: :atoms)
      {:ok, parse_metadata(parsed)}
    {:error, :enoent} ->
      {:error, :not_found}
  end
end

write_chunk(state, chunk_index, binary)

@callback write_chunk(state(), chunk_index(), binary()) :: :ok | {:error, term()}

Writes a chunk to storage.

Parameters

state - Backend state
chunk_index - Tuple identifying the chunk
data - Binary chunk data (compressed)

Returns

:ok - Write successful
{:error, reason} - Write failed

Examples

def write_chunk(state, chunk_index, data) do
  path = build_chunk_path(state.path, chunk_index)
  File.write(path, data)
end

write_metadata(state, metadata, keyword)

@callback write_metadata(state(), metadata(), keyword()) :: :ok | {:error, term()}

Writes array metadata to storage.

Parameters

state - Backend state
metadata - Metadata struct
opts - Additional options (backend-specific)

Returns

:ok - Write successful
{:error, reason} - Write failed

Examples

def write_metadata(state, metadata, _opts) do
  json = encode_metadata(metadata)
  path = Path.join(state.path, ".zarray")
  File.write(path, json)
end

Functions

implements?(module)

@spec implements?(module()) :: boolean()

Helper function to check if a module implements the Backend behavior.