if Mix.env() in [:dev] do defmodule Mix.Tasks.Benchmark do use Mix.Task alias Klife.TestUtils.AsyncProducerBenchmark alias Klife.Record def run(args) do Application.ensure_all_started(:klife) {:ok, _topics} = Klife.Utils.create_topics() opts = [strategy: :one_for_one, name: Benchmark.Supervisor] {:ok, _} = Supervisor.start_link([MyClient], opts) :ok = Klife.Testing.setup(MyClient) Process.sleep(1_000) apply(Mix.Tasks.Benchmark, :do_run_bench, args) end defp generate_data() do topic0 = "benchmark_topic_0" topic1 = "benchmark_topic_1" topic2 = "benchmark_topic_2" max_partition = 30 records_0 = Enum.map(0..(max_partition - 1), fn p -> %Klife.Record{ value: :rand.bytes(1_000), key: :rand.bytes(50), topic: topic0, partition: p } end) records_1 = Enum.map(0..(max_partition - 1), fn p -> %Klife.Record{ value: :rand.bytes(1_000), key: :rand.bytes(50), topic: topic1, partition: p } end) records_2 = Enum.map(0..(max_partition - 1), fn p -> %Klife.Record{ value: :rand.bytes(1_000), key: :rand.bytes(50), topic: topic2, partition: p } end) %{ records_0: records_0, records_1: records_1, records_2: records_2, max_partition: max_partition } end def do_run_bench("test", parallel) do client = MyConsumerGroup.klife_client() :persistent_term.put(MyConsumerGroup, client) Benchee.run( %{ "persistent term" => fn -> Enum.map(1..1000, fn _ -> :persistent_term.get(MyConsumerGroup) end) end, "defmodule" => fn -> Enum.map(1..1000, fn _ -> MyConsumerGroup.klife_client() end) end }, time: 10, memory_time: 2, parallel: parallel |> String.to_integer() ) end def do_run_bench("consumer_klife_only", parallel) do base_topic = Base.encode16(:rand.bytes(20)) partitions = 30 topics = Enum.map(1..30, fn i -> base_topic <> "_#{i}" end) {:ok, sup_pid} = DynamicSupervisor.start_link(name: :benchmark_supervisor) targets = Task.async_stream( topics, fn t -> {:ok, target} = prepare_topic(t, partitions, 10_000, 1000) target end, timeout: 300_000 ) |> Enum.map(fn {:ok, tgt} -> tgt end) :persistent_term.put(:bench_counter, :atomics.new(1, [])) Benchee.run( %{ "klife - exclusive" => fn -> run_consumer_bench(:klife_exclusive, topics, Enum.sum(targets), sup_pid) end, "klife - shared" => fn -> run_consumer_bench(:klife_shared, topics, Enum.sum(targets), sup_pid) end }, time: 60, memory_time: 2, parallel: parallel |> String.to_integer() ) end def do_run_bench("consumer", client) do base_topic = Base.encode16(:rand.bytes(20)) partitions = 30 # brod 3t e 1kb: [total_time: "653 ms", since_first_cb_call: "361 ms (55.28%)"] # klife 3t e 1kb: [total_time: "1040 ms", since_first_cb_call: "826 ms (79.42%)"] # # brod 6t e 1kb: [total_time: "1200 ms", since_first_cb_call: "1121 ms (93.42%)"] # klife 6t e 1kb: [total_time: "1868 ms", since_first_cb_call: "1540 ms (82.44%)"] # # brod 12t e 1kb: [total_time: "2785 ms", since_first_cb_call: "2688 ms (96.52%)"] # klife 12t e 1kb: [total_time: "4551 ms", since_first_cb_call: "2314 ms (50.85%)"] topics = Enum.map(1..12, fn i -> base_topic <> "_#{i}" end) {:ok, sup_pid} = DynamicSupervisor.start_link(name: :benchmark_supervisor) targets = Task.async_stream( topics, fn t -> {:ok, target} = prepare_topic(t, partitions, 10_000, 1000) target end, timeout: 300_000 ) |> Enum.map(fn {:ok, tgt} -> tgt end) :persistent_term.put(:bench_counter, :atomics.new(1, [])) run_consumer_bench( String.to_existing_atom(client), topics, Enum.sum(targets), sup_pid ) |> IO.inspect(label: client, charlists: :as_lists) end def do_run_bench("producer_async", parallel) do AsyncProducerBenchmark.run(["klife"], String.to_integer(parallel)) end def do_run_bench("producer_sync", parallel) do %{ records_0: records_0, records_1: records_1, records_2: records_2, max_partition: max_partition } = generate_data() # Warmup brod Enum.map(0..(max_partition - 1), fn i -> :brod.produce_sync_offset(:kafka_client, List.first(records_0).topic, i, "key", "warmup") :brod.produce_sync_offset(:kafka_client, List.first(records_1).topic, i, "key", "warmup") :brod.produce_sync_offset(:kafka_client, List.first(records_2).topic, i, "key", "warmup") end) Benchee.run( %{ "klife" => fn -> rec0 = Enum.random(records_0) rec1 = Enum.random(records_1) rec2 = Enum.random(records_2) t0 = Task.async(fn -> MyClient.produce(rec0) end) t1 = Task.async(fn -> MyClient.produce(rec1) end) t2 = Task.async(fn -> MyClient.produce(rec2) end) [{:ok, _}, {:ok, _}, {:ok, _}] = Task.await_many([t0, t1, t2]) end, "kafka_ex" => fn -> rec0 = Enum.random(records_0) rec1 = Enum.random(records_1) rec2 = Enum.random(records_2) t0 = Task.async(fn -> KafkaEx.produce(rec0.topic, rec0.partition, rec0.value, key: rec0.key, required_acks: -1 ) end) t1 = Task.async(fn -> KafkaEx.produce(rec1.topic, rec1.partition, rec1.value, key: rec1.key, required_acks: -1 ) end) t2 = Task.async(fn -> KafkaEx.produce(rec2.topic, rec2.partition, rec2.value, key: rec2.key, required_acks: -1 ) end) [{:ok, _}, {:ok, _}, {:ok, _}] = Task.await_many([t0, t1, t2]) end, "brod" => fn -> rec0 = Enum.random(records_0) rec1 = Enum.random(records_1) rec2 = Enum.random(records_2) t0 = Task.async(fn -> :brod.produce_sync_offset( :kafka_client, rec0.topic, rec0.partition, rec0.key, rec0.value ) end) t1 = Task.async(fn -> :brod.produce_sync_offset( :kafka_client, rec1.topic, rec1.partition, rec1.key, rec1.value ) end) t2 = Task.async(fn -> :brod.produce_sync_offset( :kafka_client, rec2.topic, rec2.partition, rec2.key, rec2.value ) end) [{:ok, _}, {:ok, _}, {:ok, _}] = Task.await_many([t0, t1, t2]) end }, time: 15, memory_time: 2, parallel: parallel |> String.to_integer() ) end def do_run_bench("producer_txn", parallel) do %{ records_0: records_0, records_1: records_1, records_2: records_2 } = generate_data() Benchee.run( %{ "produce_batch" => fn -> rec0 = Enum.random(records_0) rec1 = Enum.random(records_1) rec2 = Enum.random(records_2) [{:ok, _}, {:ok, _}, {:ok, _}] = MyClient.produce_batch([rec0, rec1, rec2]) end, "produce_batch_txn" => fn -> rec0 = Enum.random(records_0) rec1 = Enum.random(records_1) rec2 = Enum.random(records_2) {:ok, [_rec1, _rec2, _rec3]} = MyClient.produce_batch_txn([rec0, rec1, rec2]) end }, time: 15, memory_time: 2, parallel: parallel |> String.to_integer() ) end def do_run_bench("producer_inflight", parallel) do %{ records_0: records_0 } = generate_data() in_flight_records = Enum.map(records_0, fn r -> %{r | topic: "benchmark_topic_in_flight"} end) in_flight_linger_records = Enum.map(records_0, fn r -> %{r | topic: "benchmark_topic_in_flight_linger"} end) Benchee.run( %{ "klife" => fn -> {:ok, _rec} = MyClient.produce(Enum.random(records_0)) end, "klife multi inflight" => fn -> {:ok, _rec} = MyClient.produce(Enum.random(in_flight_records)) end, "klife multi inflight linger" => fn -> {:ok, _rec} = MyClient.produce(Enum.random(in_flight_linger_records)) end }, time: 15, memory_time: 2, parallel: parallel |> String.to_integer() ) end def do_run_bench("producer_resources", parallel, rec_qty) do %{ records_0: records_0, records_1: records_1, records_2: records_2, max_partition: max_partition } = generate_data() # Warmup brod Enum.map(0..(max_partition - 1), fn i -> :brod.produce_sync_offset(:kafka_client, List.first(records_0).topic, i, "key", "warmup") :brod.produce_sync_offset(:kafka_client, List.first(records_1).topic, i, "key", "warmup") :brod.produce_sync_offset(:kafka_client, List.first(records_2).topic, i, "key", "warmup") end) tasks_recs_to_send = Enum.map(1..parallel, fn _t -> Enum.map(1..rec_qty, fn _ -> [records_0, records_1, records_2] |> Enum.random() |> Enum.random() end) end) IO.inspect("Running klife") tasks_klife = Enum.map(tasks_recs_to_send, fn recs -> Task.async(fn -> Enum.map(recs, fn rec -> {:ok, _rec} = MyClient.produce(rec) end) end) end) start_klife = System.monotonic_time(:second) _resps = Task.await_many(tasks_klife, :timer.minutes(5)) total_time_klife = System.monotonic_time(:second) - start_klife IO.inspect("Running brod") tasks_brod = Enum.map(tasks_recs_to_send, fn recs -> Task.async(fn -> Enum.map(recs, fn rec -> :brod.produce_sync_offset( :kafka_client, rec.topic, rec.partition, rec.key, rec.value ) end) end) end) start_brod = System.monotonic_time(:second) _resps = Task.await_many(tasks_brod, :timer.minutes(5)) total_time_brod = System.monotonic_time(:second) - start_brod IO.inspect("Running kafka_ex") tasks_kafkaex = Enum.map(tasks_recs_to_send, fn recs -> Task.async(fn -> Enum.map(recs, fn rec -> KafkaEx.produce(rec.topic, rec.partition, rec.value, key: rec.key, required_acks: -1 ) end) end) end) start_kafkaex = System.monotonic_time(:second) _resps = Task.await_many(tasks_kafkaex, :timer.minutes(5)) total_time_kafkaex = System.monotonic_time(:second) - start_kafkaex IO.inspect( %{ total_time_klife_seconds: total_time_klife, total_time_brod_seconds: total_time_brod, total_time_kafkaex_seconds: total_time_kafkaex }, label: "results" ) end defp run_consumer_bench(client, topics, target, sup_pid) do counter = :persistent_term.get(:bench_counter) Enum.map(1..1, fn _ -> {:ok, pid} = case client do :brod -> DynamicSupervisor.start_child( sup_pid, {BrodBenchmarkConsumer, %{ topics: topics, consumer_config: [{:begin_offset, :earliest}], consumer_group_id: Base.encode64(:rand.bytes(10)) }} ) :klife_exclusive -> DynamicSupervisor.start_child( sup_pid, {BenchmarkConsumer, [ topics: Enum.map(topics, fn tname -> [ name: tname, offset_reset_policy: :earliest, fetch_strategy: {:exclusive, []} ] end), group_name: Base.encode64(:rand.bytes(10)) ]} ) :klife_shared -> DynamicSupervisor.start_child( sup_pid, {BenchmarkConsumer, [ topics: Enum.map(topics, fn tname -> [ name: tname, offset_reset_policy: :earliest ] end), group_name: Base.encode64(:rand.bytes(10)), fetch_strategy: {:shared, MyClient.get_default_fetcher()} ]} ) end t0 = System.monotonic_time(:millisecond) :ok = wait_consumption( counter, target, System.monotonic_time(:millisecond) + 120_000 ) tf = System.monotonic_time(:millisecond) t_first_call = :persistent_term.get(:first_call) :persistent_term.erase(:first_call) :atomics.put(counter, 1, 0) :ok = DynamicSupervisor.terminate_child(sup_pid, pid) total = tf - t0 first_call = tf - t_first_call [ total_time: "#{total} ms", since_first_cb_call: "#{first_call} ms (#{Float.round(first_call / total * 100, 2)}%)" ] |> IO.inspect() end) end defp prepare_topic(topic, partitions, recs_per_partition, rec_size) do :ok = Klife.Utils.create_topics(MyClient, [%{name: topic, partitions: partitions}]) Process.sleep(5000) bytes = :rand.bytes(rec_size) Enum.map(0..(partitions - 1), fn p -> Task.async(fn -> Enum.map(1..recs_per_partition, fn _i -> %Record{ value: bytes, topic: topic, partition: p } end) |> Enum.chunk_every(5) |> Enum.each(fn rec_batch -> MyClient.produce_batch(rec_batch) end) end) end) |> Task.await_many(300_000) target_val = round(partitions * recs_per_partition * rec_size) {:ok, target_val} end defp wait_consumption(atomic_ref, target, deadline) do if System.monotonic_time(:millisecond) > deadline do raise "Timeout on wait consumption!" end curr_val = :atomics.get(atomic_ref, 1) cond do curr_val < target -> Process.sleep(1) wait_consumption(atomic_ref, target, deadline) curr_val == target -> :ok curr_val > target -> raise "Double count!" end end end defmodule BenchmarkConsumer do @moduledoc false require Logger use Klife.Consumer.ConsumerGroup, client: MyClient @impl true def handle_record_batch(_topic, _partition, _cg_name, recs) do if :persistent_term.get(:first_call, nil) == nil do :persistent_term.put(:first_call, System.monotonic_time(:millisecond)) end counter = :persistent_term.get(:bench_counter) Enum.each(recs, fn %Klife.Record{} = rec -> :atomics.add(counter, 1, byte_size(rec.value)) end) :commit end end defmodule BrodBenchmarkConsumer do @moduledoc false @behaviour :brod_group_subscriber_v2 def child_spec(config) do config = %{ client: :kafka_client, group_id: config.consumer_group_id, topics: config.topics, cb_module: __MODULE__, consumer_config: config.consumer_config, init_data: [], message_type: :message_set, group_config: [ offset_commit_policy: :commit_to_kafka_v2, offset_commit_interval_seconds: 5, rejoin_delay_seconds: 60, reconnect_cool_down_seconds: 60 ] } %{ id: __MODULE__, start: {:brod_group_subscriber_v2, :start_link, [config]}, type: :worker, restart: :temporary, shutdown: 5000 } end @impl :brod_group_subscriber_v2 def init(_group_id, _init_data), do: {:ok, []} @impl :brod_group_subscriber_v2 def handle_message(message, _state) do if :persistent_term.get(:first_call, nil) == nil do :persistent_term.put(:first_call, System.monotonic_time(:millisecond)) end {:kafka_message_set, _topic, _p, _count, recs} = message counter = :persistent_term.get(:bench_counter) Enum.each(recs, fn rec -> {:kafka_message, _offset, _headers?, val, _, _, _} = rec :atomics.add(counter, 1, byte_size(val)) end) {:ok, :commit, []} end end end