ExKpl v0.1.0 ExKpl View Source

Elixir implementation of the Kinesis Producer Library

This is a port of the Erlang implementation of the KPL included in adroll/erlmld

Basic usage: iex> {_, aggregator} = ExKpl.add(ExKpl.new(), {“partition_key”, “data”}) …> ExKpl.finish(aggregator) {{“partition_key”,

<<243, 137, 154, 194, 10, 13, 112, 97, 114, 116, 105, 116, 105, 111, 110, 95, 107, 101, 121,
26, 8, 8, 0, 26, 4, 100, 97, 116, 97, 208, 54, 153, 218, 90, 34, 47, 163, 33, 8, 173, 27,
217, 85, 161, 78>>, :undefined},
%ExKpl{
  agg_explicit_hash_key: :undefined,
  agg_partition_key: :undefined,
  agg_size_bytes: 0,
  explicit_hash_keyset: %ExKpl.Keyset{key_to_index: %{}, rev_keys: []},
  num_user_records: 0,
  partition_keyset: %ExKpl.Keyset{key_to_index: %{}, rev_keys: []},
  rev_records: []

}}

Typically you will use it like:

case ExKpl.add(aggregator, {partition_key, data}) do

{:undefined, aggregator} ->
  aggregator

{full_record, aggregator}
  send_record_to_kinesis(full_record)
  aggregator

end

You can force the current records to be aggregated with finish/1

Link to this section Summary

Link to this section Types

Link to this type aggregated_record() View Source
aggregated_record() :: {key(), binary(), key()}
Link to this type t() View Source
t() :: %ExKpl{
  agg_explicit_hash_key: :undefined | binary(),
  agg_partition_key: :undefined | binary(),
  agg_size_bytes: non_neg_integer(),
  explicit_hash_keyset: ExKpl.Keyset.t(),
  num_user_records: non_neg_integer(),
  partition_keyset: ExKpl.Keyset.t(),
  rev_records: [ExKpl.Proto.Record.t()]
}
Link to this type user_record() View Source
user_record() :: {key(), binary(), key()}

Link to this section Functions

Link to this function add(agg, arg) View Source
add(t(), {key(), binary()}) :: {user_record() | :undefined, t()}
add(t(), {key(), binary(), key()}) :: {user_record() | :undefined, t()}
Link to this function add_all(agg, records) View Source
add_all(t(), [user_record()]) :: {[aggregated_record()], t()}
Link to this function finish(agg) View Source
finish(t()) :: {aggregated_record() | :undefined, t()}
Link to this function finish(agg, should_deflate?) View Source
finish(t(), boolean()) :: {:undefined, t()}
finish(t(), boolean()) :: {aggregated_record(), t()}
Link to this function size_bytes(ex_kpl) View Source
size_bytes(t()) :: non_neg_integer()