lightspeed/pipeline/slo

ETL SLO budget contracts for lag/throughput/retry/replay recovery.

Types

Budget

One ETL SLO budget profile.

pub type Budget {
  Budget(
    profile: String,
    max_lag_ms: Int,
    min_throughput_records_per_sec: Int,
    max_retry_rate_percent: Int,
    max_replay_recovery_ms: Int,
  )
}

Constructors

Budget(
  profile: String,
  max_lag_ms: Int,
  min_throughput_records_per_sec: Int,
  max_retry_rate_percent: Int,
  max_replay_recovery_ms: Int,
)

BudgetResult

One budget check result.

pub type BudgetResult {
  BudgetResult(profile: String, passed: Bool, reason: String)
}

Constructors

BudgetResult(profile: String, passed: Bool, reason: String)

Observation

One measured ETL observation.

pub type Observation {
  Observation(
    profile: String,
    lag_ms: Int,
    throughput_records_per_sec: Int,
    retry_count: Int,
    processed_records: Int,
    replay_recovery_ms: Int,
  )
}

Constructors

Observation(
  profile: String,
  lag_ms: Int,
  throughput_records_per_sec: Int,
  retry_count: Int,
  processed_records: Int,
  replay_recovery_ms: Int,
)

Values

budget

pub fn budget(
  profile: String,
  max_lag_ms: Int,
  min_throughput_records_per_sec: Int,
  max_retry_rate_percent: Int,
  max_replay_recovery_ms: Int,
) -> Budget

Build one SLO budget profile.

budget_failures

pub fn budget_failures(results: List(BudgetResult)) -> Int

Count budget failures.

budget_result_label

pub fn budget_result_label(result: BudgetResult) -> String

Stable budget-result label.

budget_version

pub const budget_version: Int

budget_version_label

pub fn budget_version_label() -> String

Versioned M33 budget profile label.

default_budget

pub fn default_budget() -> List(Budget)

Default M33 ETL SLO budget profile list.

evaluate

pub fn evaluate(
  observations: List(Observation),
  budgets: List(Budget),
) -> List(BudgetResult)

Evaluate one observation set against one budget list.

observation

pub fn observation(
  profile: String,
  lag_ms: Int,
  throughput_records_per_sec: Int,
  retry_count: Int,
  processed_records: Int,
  replay_recovery_ms: Int,
) -> Observation

Build one ETL observation.

observation_label

pub fn observation_label(observation: Observation) -> String

Stable observation label.

retry_rate_percent

pub fn retry_rate_percent(observation: Observation) -> Int

Retry-rate percentage derived from retry count and processed records.

valid_budget

pub fn valid_budget(budget: Budget) -> Bool

Validate one budget profile.

valid_budgets

pub fn valid_budgets(budgets: List(Budget)) -> Bool

Validate a budget list and enforce unique profile names.

valid_observation

pub fn valid_observation(observation: Observation) -> Bool

Validate one observation.