Bedrock.DataPlane.Materializer.Olivine.IndexDatabase (bedrock v0.6.0)

Append-only file-based storage for Olivine index version blocks.

File Format

Each record represents a version block containing modified pages:

[Header: 16 bytes]
- magic: 0x4F4C5644 (4 bytes) - "OLVD"
- version: (8 bytes) - commit version (binary)
- payload_size: (4 bytes) - size of payload

[Payload: variable bytes]
- :erlang.term_to_binary({previous_version, pages_map}, [:compressed])

[Footer: 4 bytes]
- payload_size: (4 bytes) - repeated for backward scan

Recovery

The durable version is the version of the last complete record in the file. Recovery reads the last 20 bytes (footer + header start) to extract it.

Reading

Page blocks are loaded by scanning backward from EOF until the target version is found. This supports the iterative page chain loading pattern.

Summary

Types

t()

Functions

build_snapshot_record(version, pages_map)

Builds a snapshot record as iodata without writing to a file. Used by CompactionWriter implementations to get the index bytes. The previous_version equals version (self-loop) to terminate the chain.

close(index_db)

durable_version(index_db)

flush(index_db, new_durable_version, previous_durable_version, collected_pages)

info(index_db, stat)

load_durable_version(map)

load_page_block(index_db, target_version)

Load the page block for a given version, returning the pages and the next version in the chain. Returns {pages_map, next_version} where next_version is the previous version that forms a chain.

load_pages_from_version(index_db, version)

open(otp_name, file_path)

sync(index_db)

write_snapshot_block(file_fd, version, pages_map)

Writes a snapshot block to a file descriptor. Used during compaction to create a single version block with all current pages. The previous_version equals version (self-loop) to terminate the chain.

Types

t()

@type t() :: %Bedrock.DataPlane.Materializer.Olivine.IndexDatabase{
  durable_version: Bedrock.version(),
  file: :file.fd(),
  file_name: [char()],
  file_offset: non_neg_integer(),
  last_block_empty: boolean(),
  last_block_offset: non_neg_integer(),
  last_block_previous_version: Bedrock.version() | nil
}

Functions

build_snapshot_record(version, pages_map)

@spec build_snapshot_record(Bedrock.version(), %{
  required(Bedrock.DataPlane.Materializer.Olivine.Index.Page.id()) =>
    {Bedrock.DataPlane.Materializer.Olivine.Index.Page.t(),
     Bedrock.DataPlane.Materializer.Olivine.Index.Page.id()}
}) :: iodata()

Builds a snapshot record as iodata without writing to a file. Used by CompactionWriter implementations to get the index bytes. The previous_version equals version (self-loop) to terminate the chain.

close(index_db)

@spec close(t()) :: :ok

durable_version(index_db)

@spec durable_version(t()) :: Bedrock.version()

flush(index_db, new_durable_version, previous_durable_version, collected_pages)

@spec flush(
  t(),
  new_durable_version :: Bedrock.version(),
  previous_durable_version :: Bedrock.version(),
  collected_pages :: [
    %{
      required(Bedrock.DataPlane.Materializer.Olivine.Index.Page.id()) =>
        {Bedrock.DataPlane.Materializer.Olivine.Index.Page.t(),
         Bedrock.DataPlane.Materializer.Olivine.Index.Page.id()}
    }
  ]
) ::
  {:ok, t()}
  | {:error, {:index_file_write_failed, File.posix()}}
  | {:error, {:index_file_truncate_failed, File.posix()}}
  | {:error, {:index_file_sync_failed, File.posix()}}

info(index_db, stat)

@spec info(t(), :n_keys | :utilization | :size_in_bytes | :key_ranges) ::
  any() | :undefined

load_durable_version(map)

@spec load_durable_version(t()) :: {:ok, Bedrock.version()} | {:error, :not_found}

load_page_block(index_db, target_version)

@spec load_page_block(t(), Bedrock.version()) ::
  {:ok,
   %{
     required(Bedrock.DataPlane.Materializer.Olivine.Index.Page.id()) =>
       {Bedrock.DataPlane.Materializer.Olivine.Index.Page.t(),
        Bedrock.DataPlane.Materializer.Olivine.Index.Page.id()}
   }, Bedrock.version() | nil}
  | {:error, :not_found}

Load the page block for a given version, returning the pages and the next version in the chain. Returns {pages_map, next_version} where next_version is the previous version that forms a chain.

load_pages_from_version(index_db, version)

@spec load_pages_from_version(t(), Bedrock.version()) :: %{
  required(Bedrock.DataPlane.Materializer.Olivine.Index.Page.id()) =>
    {Bedrock.DataPlane.Materializer.Olivine.Index.Page.t(),
     Bedrock.DataPlane.Materializer.Olivine.Index.Page.id()}
}

open(otp_name, file_path)

@spec open(otp_name :: atom(), file_path :: String.t()) ::
  {:ok, t()} | {:error, :system_limit | :badarg | File.posix()}

sync(index_db)

@spec sync(t()) :: :ok

write_snapshot_block(file_fd, version, pages_map)

@spec write_snapshot_block(:file.fd(), Bedrock.version(), %{
  required(Bedrock.DataPlane.Materializer.Olivine.Index.Page.id()) =>
    {Bedrock.DataPlane.Materializer.Olivine.Index.Page.t(),
     Bedrock.DataPlane.Materializer.Olivine.Index.Page.id()}
}) :: :ok

Writes a snapshot block to a file descriptor. Used during compaction to create a single version block with all current pages. The previous_version equals version (self-loop) to terminate the chain.