lightspeed/data_plane_huge

Data-plane 2.0 contracts for million-row and burst-analytics workloads (M51).

Types

Benchmark

Deterministic heavy-data benchmark point.

pub type Benchmark {
  Benchmark(
    workload: String,
    p50_latency_ms: Int,
    p95_latency_ms: Int,
    payload_bytes: Int,
    patch_ops: Int,
    throughput_events_per_second: Int,
    full_root_churn_events: Int,
    ordering_violations: Int,
  )
}

Constructors

Benchmark(
  workload: String,
  p50_latency_ms: Int,
  p95_latency_ms: Int,
  payload_bytes: Int,
  patch_ops: Int,
  throughput_events_per_second: Int,
  full_root_churn_events: Int,
  ordering_violations: Int,
)

Budget

Deterministic heavy-data budget profile.

pub type Budget {
  Budget(
    workload: String,
    max_p95_latency_ms: Int,
    max_payload_bytes: Int,
    max_patch_ops: Int,
    min_throughput_events_per_second: Int,
    max_full_root_churn_events: Int,
    max_ordering_violations: Int,
  )
}

Constructors

Budget(
  workload: String,
  max_p95_latency_ms: Int,
  max_payload_bytes: Int,
  max_patch_ops: Int,
  min_throughput_events_per_second: Int,
  max_full_root_churn_events: Int,
  max_ordering_violations: Int,
)

BudgetResult

Heavy-data budget evaluation result.

pub type BudgetResult {
  BudgetResult(workload: String, passed: Bool, reason: String)
}

Constructors

BudgetResult(workload: String, passed: Bool, reason: String)

BurstBatch

Burst update batch.

pub type BurstBatch {
  BurstBatch(
    batch_id: String,
    upserts: Int,
    deletes: Int,
    max_sequence: Int,
  )
}

Constructors

BurstBatch(
  batch_id: String,
  upserts: Int,
  deletes: Int,
  max_sequence: Int,
)

Dataset

Huge-dataset profile.

pub type Dataset {
  MillionRowList
  MillionRowGrid(columns: Int)
  BurstAnalytics(cubes: Int)
}

Constructors

```
MillionRowList
```
```
MillionRowGrid(columns: Int)
```
```
BurstAnalytics(cubes: Int)
```

PatchSummary

Compact patch summary for one huge-data update pass.

pub type PatchSummary {
  PatchSummary(
    patch_ops: Int,
    payload_bytes: Int,
    full_root_churn: Bool,
    compact_profile: String,
  )
}

Constructors

PatchSummary(
  patch_ops: Int,
  payload_bytes: Int,
  full_root_churn: Bool,
  compact_profile: String,
)

Plane

opaque

Data-plane 2.0 runtime.

pub opaque type Plane

QueryError

Query failures.

pub type QueryError {
  InvalidWindow(offset: Int, limit: Int)
  SequenceRegressed(expected_min: Int, provided: Int)
}

Constructors

```
InvalidWindow(offset: Int, limit: Int)
```

SequenceRegressed(expected_min: Int, provided: Int)

TransportProfile

Compact transport/data profile for high-volume views.

pub type TransportProfile {
  TransportProfile(
    name: String,
    encoding: String,
    max_frame_bytes: Int,
    window_chunk_size: Int,
    compact_mode: String,
  )
}

Constructors

TransportProfile(
  name: String,
  encoding: String,
  max_frame_bytes: Int,
  window_chunk_size: Int,
  compact_mode: String,
)

Window

Window result summary.

pub type Window {
  Window(
    offset: Int,
    limit: Int,
    total: Int,
    first_index: Int,
    last_index: Int,
    version: Int,
    sequence: Int,
  )
}

Constructors

Window(
  offset: Int,
  limit: Int,
  total: Int,
  first_index: Int,
  last_index: Int,
  version: Int,
  sequence: Int,
)

WindowRequest

Window request over a huge dataset.

pub type WindowRequest {
  WindowRequest(offset: Int, limit: Int, sequence: Int)
}

Constructors

WindowRequest(offset: Int, limit: Int, sequence: Int)

Values

apply_burst_updates

pub fn apply_burst_updates(
  plane: Plane,
  request: WindowRequest,
  batch: BurstBatch,
) -> #(Plane, Result(Window, QueryError), PatchSummary)

Apply one burst update batch and emit compact patch summary.

benchmark_signature

pub fn benchmark_signature(benchmark: Benchmark) -> String

Stable benchmark signature.

budget_failures

pub fn budget_failures(results: List(BudgetResult)) -> Int

Count failing huge-data budgets.

budget_result_signature

pub fn budget_result_signature(result: BudgetResult) -> String

Stable budget-result signature.

burst_batch

pub fn burst_batch(
  batch_id: String,
  upserts: Int,
  deletes: Int,
  max_sequence: Int,
) -> BurstBatch

Build one burst update batch.

compact_transport_profiles

pub fn compact_transport_profiles() -> List(TransportProfile)

Deterministic compact transport profile matrix.

compact_transport_valid

pub fn compact_transport_valid(profile: TransportProfile) -> Bool

Validate one compact transport profile.

dataset

pub fn dataset(plane: Plane) -> Dataset

Plane dataset accessor.

dataset_label

pub fn dataset_label(dataset: Dataset) -> String

Stable dataset label.

default_huge_budgets

pub fn default_huge_budgets() -> List(Budget)

Default deterministic M51 budgets.

default_transport_profile

pub fn default_transport_profile() -> TransportProfile

Default transport profile for huge-data workloads.

evaluate_huge_budgets

pub fn evaluate_huge_budgets(
  benchmarks: List(Benchmark),
  budgets: List(Budget),
) -> List(BudgetResult)

Evaluate huge-data benchmarks against one budget profile.

million_row_plane

pub fn million_row_plane(
  target: String,
  dataset: Dataset,
  transport: TransportProfile,
) -> Plane

Build the default million-row plane for one dataset profile.

new

pub fn new(
  target: String,
  dataset: Dataset,
  total_rows: Int,
  window_limit: Int,
  ordering_key: String,
  transport: TransportProfile,
) -> Plane

Build a huge-data plane.

ordering_invariant

pub fn ordering_invariant(window: Window) -> Bool

True when ordering and sequence invariants hold for one window.

patch_summary_signature

pub fn patch_summary_signature(summary: PatchSummary) -> String

Stable patch summary signature.

query_error_label

pub fn query_error_label(error: QueryError) -> String

Stable query-error label.

query_window

pub fn query_window(
  plane: Plane,
  request: WindowRequest,
) -> Result(Window, QueryError)

Query one huge-data window.

request

pub fn request(
  offset: Int,
  limit: Int,
  sequence: Int,
) -> WindowRequest

Build one window request.

run_huge_benchmarks

pub fn run_huge_benchmarks() -> List(Benchmark)

Run deterministic heavy-data benchmarks.

signature

pub fn signature(plane: Plane) -> String

Stable huge-plane signature.

steady_state_no_full_root_churn

pub fn steady_state_no_full_root_churn(
  summary: PatchSummary,
) -> Bool

True when the patch summary avoids full-root churn.

target

pub fn target(plane: Plane) -> String

Plane target accessor.

total_rows

pub fn total_rows(plane: Plane) -> Int

Plane total rows accessor.

transport

pub fn transport(plane: Plane) -> TransportProfile

Plane transport profile accessor.

transport_label

pub fn transport_label(profile: TransportProfile) -> String

Stable transport-profile label.

valid

pub fn valid(plane: Plane) -> Bool

Validate one huge-data plane.

window_sequence

pub fn window_sequence(window: Window) -> Int

Window sequence accessor.

window_signature

pub fn window_signature(window: Window) -> String

Stable window signature.

window_version

pub fn window_version(window: Window) -> Int

Window version accessor.