ExKpl v0.1.3 ExKpl View Source
Elixir implementation of the Kinesis Producer Library record aggregator.
This is a port of the Erlang implementation included in adroll/erlmld
Basic usage:
iex> {_, aggregator} = ExKpl.add(ExKpl.new(), {"partition_key", "data"})
...> ExKpl.finish(aggregator)
{{"partition_key", <<0, 137, 154, 194, 10, 13, 112, 97, 114, 116, 105, 116, 105, 111, 110, 95, 107, 101, 121, 26, 8, 8, 0, 26, 4, 100, 97, 116, 97, 208, 54, 153, 218, 90, 34, 47, 163, 33, 8, 173, 27, 217, 85, 161, 78>>, :undefined}, %ExKpl{agg_explicit_hash_key: :undefined, agg_partition_key: :undefined, agg_size_bytes: 0, explicit_hash_keyset: %ExKpl.Keyset{key_to_index: %{}, rev_keys: []}, num_user_records: 0, partition_keyset: %ExKpl.Keyset{key_to_index: %{}, rev_keys: []}, rev_records: []}}
Typically you will use it like:
case ExKpl.add(aggregator, {partition_key, data}) do
{:undefined, aggregator} ->
aggregator
{full_record, aggregator} ->
send_record_to_kinesis(full_record)
aggregator
end
You can force the current records to be aggregated with finish/1,2
Link to this section Summary
Link to this section Types
Link to this type
aggregated_record()
View Source
aggregated_record()
View Source
aggregated_record() :: {key(), serialized_data(), key()}
aggregated_record() :: {key(), serialized_data(), key()}
Link to this type
key()
View Source
key()
View Source
key() :: binary()
key() :: binary()
Link to this type
raw_data()
View Source
raw_data()
View Source
raw_data() :: binary()
raw_data() :: binary()
Link to this type
serialized_data()
View Source
serialized_data()
View Source
serialized_data() :: binary()
serialized_data() :: binary()
Link to this type
t()
View Source
t()
View Source
t() :: %ExKpl{
agg_explicit_hash_key: :undefined | binary(),
agg_partition_key: :undefined | binary(),
agg_size_bytes: non_neg_integer(),
explicit_hash_keyset: ExKpl.Keyset.t(),
num_user_records: non_neg_integer(),
partition_keyset: ExKpl.Keyset.t(),
rev_records: [ExKpl.Proto.Record.t()]
}
t() :: %ExKpl{ agg_explicit_hash_key: :undefined | binary(), agg_partition_key: :undefined | binary(), agg_size_bytes: non_neg_integer(), explicit_hash_keyset: ExKpl.Keyset.t(), num_user_records: non_neg_integer(), partition_keyset: ExKpl.Keyset.t(), rev_records: [ExKpl.Proto.Record.t()] }
Link to this type
user_record() View Source
Link to this section Functions
Link to this function
add(agg, arg) View Source
Link to this function
add_all(agg, records)
View Source
add_all(agg, records)
View Source
add_all(t(), [user_record()]) :: {[aggregated_record()], t()}
add_all(t(), [user_record()]) :: {[aggregated_record()], t()}
Link to this function
count(ex_kpl)
View Source
count(ex_kpl)
View Source
count(t()) :: non_neg_integer()
count(t()) :: non_neg_integer()
Link to this function
finish(agg)
View Source
finish(agg)
View Source
finish(t()) :: {aggregated_record() | :undefined, t()}
finish(t()) :: {aggregated_record() | :undefined, t()}
Link to this function
finish(agg, should_deflate?) View Source
Link to this function
new()
View Source
new()
View Source
new() :: t()
new() :: t()
Link to this function
size_bytes(ex_kpl)
View Source
size_bytes(ex_kpl)
View Source
size_bytes(t()) :: non_neg_integer()
size_bytes(t()) :: non_neg_integer()