Compression Fundamentals

# Use this install to work with the source code
# Mix.install(
#   [
#     {:ex_codecs, path: Path.join(__DIR__, "..")}, 
#     {:rustler, "~> 0.36"},
#     {:jason, "~> 1.4"}, 
#     {:kino, "~> 0.14"}, 
#     {:kino_vega_lite, "~> 0.1.13"}
#   ],
#   config:  [rustler_precompiled: [force_build: [ex_codecs: true]]]
# )

Mix.install( [
    {:ex_codecs, "~> 0.2.3"}, 
    {:jason, "~> 1.4"}, 
    {:kino, "~> 0.14"}, 
    {:kino_vega_lite, "~> 0.1.13"}
  ])

Series

How Compression Works

Compression algorithms exploit redundancy in data. The more patterns and repetition, the more compressible the data is — this is measured by entropy.

# Low entropy: highly repetitive
low_entropy = String.duplicate("AAAA", 4096)

# High entropy: near-random data
high_entropy = :crypto.strong_rand_bytes(16384)

# Medium entropy: natural language text
medium_entropy = String.duplicate("The quick brown fox jumps over the lazy dog. ", 200)

for {label, data} <- [
  {"Repetitive (low entropy)", low_entropy},
  {"Natural text (medium)", medium_entropy},
  {"Random bytes (high entropy)", high_entropy}
] do
  {:ok, z} = ExCodecs.encode(:zstd, data)
  ratio = Float.round(byte_size(data) / byte_size(z), 2)
  IO.puts("#{String.pad_trailing(label, 28)} | #{byte_size(data)} -> #{byte_size(z)} bytes | #{ratio}x ratio")
end

Repetitive (low entropy)     | 16384 -> 11 bytes | 1489.45x ratio
Natural text (medium)        | 9000 -> 64 bytes | 140.63x ratio
Random bytes (high entropy)  | 16384 -> 16394 bytes | 1.0x ratio

[:ok, :ok, :ok]

Lossless vs Lossy

ExCodecs provides lossless codecs — decoded data is bit-for-bit identical to the original:

data = :crypto.strong_rand_bytes(8192)

compression_codecs =
  ExCodecs.Compression.available_codecs()
  |> Enum.map(& &1.name)

for codec <- compression_codecs do
  {:ok, enc} = ExCodecs.encode(codec, data)
  {:ok, dec} = ExCodecs.decode(codec, enc)
  IO.puts("#{String.pad_trailing(inspect(codec), 10)} lossless: #{dec == data}")
end

:blosc2    lossless: true
:bzip2     lossless: true
:lz4       lossless: true
:snappy    lossless: true
:zstd      lossless: true

[:ok, :ok, :ok, :ok, :ok]

Lossy codecs (JPEG, MP3, etc.) sacrifice exact reproduction for smaller size. These are not in ExCodecs' scope but could be added via the ExCodecs.Codec behaviour.

Compression Methods

Dictionary-Based (LZ4, Snappy, Zstd)

These build a reference table of repeated substrings during compression:

# Dictionary methods excel on repeated patterns
text = String.duplicate("compression compresses compressed compressor ", 200)

{:ok, lz4_enc} = ExCodecs.encode(:lz4, text)
{:ok, zstd_enc} = ExCodecs.encode(:zstd, text)
{:ok, snappy_enc} = ExCodecs.encode(:snappy, text)

IO.puts("Original: #{byte_size(text)} bytes")
IO.puts("LZ4:     #{byte_size(lz4_enc)} bytes (#{Float.round(100 * byte_size(lz4_enc) / byte_size(text), 1)}%)")
IO.puts("Zstd:    #{byte_size(zstd_enc)} bytes (#{Float.round(100 * byte_size(zstd_enc) / byte_size(text), 1)}%)")
IO.puts("Snappy:  #{byte_size(snappy_enc)} bytes (#{Float.round(100 * byte_size(snappy_enc) / byte_size(text), 1)}%)")

Original: 9000 bytes
LZ4:     78 bytes (0.9%)
Zstd:    43 bytes (0.5%)
Snappy:  451 bytes (5.0%)

:ok

Block-Sorting (Bzip2)

Bzip2 uses the Burrows-Wheeler Transform to group similar characters, then applies Huffman coding:

# Bzip2 achieves excellent ratios on text-heavy data
{:ok, bz_enc} = ExCodecs.encode(:bzip2, text)
IO.puts("Bzip2:   #{byte_size(bz_enc)} bytes (#{Float.round(100 * byte_size(bz_enc) / byte_size(text), 1)}%)")

Bzip2:   84 bytes (0.9%)

:ok

Shuffle + Compress (Blosc2)

Blosc2 reorders bytes to create longer runs before applying an internal compressor:

# Numerical data with patterns across float64 values
floats = for i <- 1..2048, into: <<>>, do: <<i * 0.125::float-size(64)-little>>

{:ok, blosc_none} = ExCodecs.encode(:blosc2, floats, typesize: 8, shuffle: :none)
{:ok, blosc_byte} = ExCodecs.encode(:blosc2, floats, typesize: 8, shuffle: :byte)
{:ok, blosc_bit} = ExCodecs.encode(:blosc2, floats, typesize: 8, shuffle: :bit)
{:ok, zstd_plain} = ExCodecs.encode(:zstd, floats)

IO.puts("Original:        #{byte_size(floats)} bytes")
IO.puts("Blosc2 (none):   #{byte_size(blosc_none)} bytes")
IO.puts("Blosc2 (byte):   #{byte_size(blosc_byte)} bytes")
IO.puts("Blosc2 (bit):    #{byte_size(blosc_bit)} bytes")
IO.puts("Zstd (plain):    #{byte_size(zstd_plain)} bytes")

Original:        16384 bytes
Blosc2 (none):   8251 bytes
Blosc2 (byte):   675 bytes
Blosc2 (bit):    504 bytes
Zstd (plain):    1889 bytes

:ok

Speed vs Ratio Tradeoffs

# Generate test datasets
json_like = Jason.encode!(for i <- 1..500, do: %{id: i, name: "item_#{i}", value: :rand.uniform(1000)})

datasets = %{
  "Repetitive" => String.duplicate("abcdefghij", 5000),
  "Natural text" => String.duplicate("The quick brown fox jumps over the lazy dog. ", 300),
  "Semi-random" => (for _ <- 1..10000, into: <<>>, do: <<:rand.uniform(255)>>),
  "JSON-like" => json_like
}

results = for {label, data} <- datasets, codec <- [:lz4, :snappy, :zstd, :bzip2] do
  {:ok, enc} = ExCodecs.encode(codec, data)
  {time, _} = :timer.tc(fn -> for _ <- 1..50, do: ExCodecs.encode(codec, data) end)
  {time_d, _} = :timer.tc(fn -> for _ <- 1..50, do: ExCodecs.decode(codec, enc) end)
  %{
    dataset: label,
    codec: inspect(codec),
    original_size: byte_size(data),
    compressed_size: byte_size(enc),
    ratio: Float.round(100 * byte_size(enc) / byte_size(data), 1),
    encode_us: div(time, 50),
    decode_us: div(time_d, 50)
  }
end

Kino.DataTable.new(results)

[%{ratio: 29.1, dataset: "JSON-like", codec: ":lz4", original_size: 20217, compressed_size: 5878, encode_us: 40, decode_us: 32}, %{ratio: 33.1, dataset: "JSON-like", codec: ":snappy", original_size: 20217, compressed_size: 6682, encode_us: 35, decode_us: 25}, %{ratio: 14.3, dataset: "JSON-like", codec: ":zstd", original_size: 20217, compressed_size: 2895, encode_us: 140, decode_us: 64}, %{ratio: 12.6, dataset: "JSON-like", codec: ":bzip2", original_size: 20217, compressed_size: 2548, encode_us: 1169, decode_us: 247}, %{ratio: 0.8, dataset: "Natural text", codec: ":lz4", original_size: 13500, compressed_size: 113, encode_us: 9, decode_us: 11}, %{ratio: 5.0, dataset: "Natural text", codec: ":snappy", original_size: 13500, compressed_size: 681, encode_us: 9, decode_us: 9}, %{ratio: 0.5, dataset: "Natural text", codec: ":zstd", original_size: 13500, compressed_size: 64, encode_us: 11, decode_us: 13}, %{ratio: 1.2, dataset: "Natural text", codec: ":bzip2", original_size: 13500, compressed_size: 156, encode_us: 1366, decode_us: 116}, %{ratio: 0.4, dataset: "Repetitive", codec: ":lz4", original_size: 50000, compressed_size: 220, encode_us: 12, decode_us: 41}, %{ratio: 4.7, dataset: "Repetitive", codec: ":snappy", original_size: 50000, compressed_size: 2359, encode_us: 12, decode_us: 18}, %{ratio: 0.1, dataset: "Repetitive", codec: ":zstd", original_size: 50000, compressed_size: 28, encode_us: 16, decode_us: 16}, %{ratio: 0.1, dataset: "Repetitive", codec: ":bzip2", original_size: 50000, compressed_size: 68, encode_us: 3899, decode_us: 292}, %{ratio: 100.5, dataset: "Semi-random", codec: ":lz4", ...}, ...]

Compression Ratio Visualization

VegaLite.new(width: 600, height: 400)
|> VegaLite.data_from_values(results)
|> VegaLite.mark(:bar)
|> VegaLite.encode_field(:x, "codec", type: :nominal)
|> VegaLite.encode_field(:y, "ratio", type: :quantitative, title: "Compressed size (%)")
|> VegaLite.encode_field(:color, "dataset", type: :nominal)
|> VegaLite.encode_field(:column, "dataset", type: :nominal)

{"$schema":"https://vega.github.io/schema/vega-lite/v5.json","data":{"values":[{"codec":":lz4","compressed_size":5878,"dataset":"JSON-like","decode_us":32,"encode_us":40,"original_size":20217,"ratio":29.1},{"codec":":snappy","compressed_size":6682,"dataset":"JSON-like","decode_us":25,"encode_us":35,"original_size":20217,"ratio":33.1},{"codec":":zstd","compressed_size":2895,"dataset":"JSON-like","decode_us":64,"encode_us":140,"original_size":20217,"ratio":14.3},{"codec":":bzip2","compressed_size":2548,"dataset":"JSON-like","decode_us":247,"encode_us":1169,"original_size":20217,"ratio":12.6},{"codec":":lz4","compressed_size":113,"dataset":"Natural text","decode_us":11,"encode_us":9,"original_size":13500,"ratio":0.8},{"codec":":snappy","compressed_size":681,"dataset":"Natural text","decode_us":9,"encode_us":9,"original_size":13500,"ratio":5.0},{"codec":":zstd","compressed_size":64,"dataset":"Natural text","decode_us":13,"encode_us":11,"original_size":13500,"ratio":0.5},{"codec":":bzip2","compressed_size":156,"dataset":"Natural text","decode_us":116,"encode_us":1366,"original_size":13500,"ratio":1.2},{"codec":":lz4","compressed_size":220,"dataset":"Repetitive","decode_us":41,"encode_us":12,"original_size":50000,"ratio":0.4},{"codec":":snappy","compressed_size":2359,"dataset":"Repetitive","decode_us":18,"encode_us":12,"original_size":50000,"ratio":4.7},{"codec":":zstd","compressed_size":28,"dataset":"Repetitive","decode_us":16,"encode_us":16,"original_size":50000,"ratio":0.1},{"codec":":bzip2","compressed_size":68,"dataset":"Repetitive","decode_us":292,"encode_us":3899,"original_size":50000,"ratio":0.1},{"codec":":lz4","compressed_size":10045,"dataset":"Semi-random","decode_us":8,"encode_us":10,"original_size":10000,"ratio":100.5},{"codec":":snappy","compressed_size":10005,"dataset":"Semi-random","decode_us":7,"encode_us":9,"original_size":10000,"ratio":100.0},{"codec":":zstd","compressed_size":10010,"dataset":"Semi-random","decode_us":11,"encode_us":10,"original_size":10000,"ratio":100.1},{"codec":":bzip2","compressed_size":10463,"dataset":"Semi-random","decode_us":464,"encode_us":1618,"original_size":10000,"ratio":104.6}]},"encoding":{"color":{"field":"dataset","type":"nominal"},"column":{"field":"dataset","type":"nominal"},"x":{"field":"codec","type":"nominal"},"y":{"field":"ratio","title":"Compressed size (%)","type":"quantitative"}},"height":400,"mark":"bar","width":600}

Speed Visualization

VegaLite.new(width: 600, height: 400)
|> VegaLite.data_from_values(results)
|> VegaLite.mark(:bar)
|> VegaLite.encode_field(:x, "codec", type: :nominal)
|> VegaLite.encode_field(:y, "encode_us", type: :quantitative, title: "Encode time (µs)")
|> VegaLite.encode_field(:color, "dataset", type: :nominal)
|> VegaLite.encode_field(:column, "dataset", type: :nominal)

{"$schema":"https://vega.github.io/schema/vega-lite/v5.json","data":{"values":[{"codec":":lz4","compressed_size":5878,"dataset":"JSON-like","decode_us":32,"encode_us":40,"original_size":20217,"ratio":29.1},{"codec":":snappy","compressed_size":6682,"dataset":"JSON-like","decode_us":25,"encode_us":35,"original_size":20217,"ratio":33.1},{"codec":":zstd","compressed_size":2895,"dataset":"JSON-like","decode_us":64,"encode_us":140,"original_size":20217,"ratio":14.3},{"codec":":bzip2","compressed_size":2548,"dataset":"JSON-like","decode_us":247,"encode_us":1169,"original_size":20217,"ratio":12.6},{"codec":":lz4","compressed_size":113,"dataset":"Natural text","decode_us":11,"encode_us":9,"original_size":13500,"ratio":0.8},{"codec":":snappy","compressed_size":681,"dataset":"Natural text","decode_us":9,"encode_us":9,"original_size":13500,"ratio":5.0},{"codec":":zstd","compressed_size":64,"dataset":"Natural text","decode_us":13,"encode_us":11,"original_size":13500,"ratio":0.5},{"codec":":bzip2","compressed_size":156,"dataset":"Natural text","decode_us":116,"encode_us":1366,"original_size":13500,"ratio":1.2},{"codec":":lz4","compressed_size":220,"dataset":"Repetitive","decode_us":41,"encode_us":12,"original_size":50000,"ratio":0.4},{"codec":":snappy","compressed_size":2359,"dataset":"Repetitive","decode_us":18,"encode_us":12,"original_size":50000,"ratio":4.7},{"codec":":zstd","compressed_size":28,"dataset":"Repetitive","decode_us":16,"encode_us":16,"original_size":50000,"ratio":0.1},{"codec":":bzip2","compressed_size":68,"dataset":"Repetitive","decode_us":292,"encode_us":3899,"original_size":50000,"ratio":0.1},{"codec":":lz4","compressed_size":10045,"dataset":"Semi-random","decode_us":8,"encode_us":10,"original_size":10000,"ratio":100.5},{"codec":":snappy","compressed_size":10005,"dataset":"Semi-random","decode_us":7,"encode_us":9,"original_size":10000,"ratio":100.0},{"codec":":zstd","compressed_size":10010,"dataset":"Semi-random","decode_us":11,"encode_us":10,"original_size":10000,"ratio":100.1},{"codec":":bzip2","compressed_size":10463,"dataset":"Semi-random","decode_us":464,"encode_us":1618,"original_size":10000,"ratio":104.6}]},"encoding":{"color":{"field":"dataset","type":"nominal"},"column":{"field":"dataset","type":"nominal"},"x":{"field":"codec","type":"nominal"},"y":{"field":"encode_us","title":"Encode time (µs)","type":"quantitative"}},"height":400,"mark":"bar","width":600}

When Not to Compress

# Already-compressed data doesn't shrink further
compressed_png = for _ <- 1..8192, into: <<>>, do: <<:rand.uniform(255)>>
{:ok, after_zstd} = ExCodecs.encode(:zstd, compressed_png)

IO.puts("Random data:          #{byte_size(compressed_png)} bytes")
IO.puts("After Zstd compress:  #{byte_size(after_zstd)} bytes")
IO.puts("Compressed data can actually GROW due to header overhead")

Random data:          8192 bytes
After Zstd compress:  8202 bytes
Compressed data can actually GROW due to header overhead

:ok

Rules of thumb:

• Don't compress encrypted or already-compressed data — you waste CPU for no gain
• Avoid compressing tiny payloads — the codec header overhead may exceed savings
• Consider latency — LZ4/Snappy for hot paths, Bzip2 only for cold storage

CPU vs size (and where memory actually fits)

ExCodecs does not expose peak NIF memory counters, so these cells measure what we can observe on the BEAM: encode time, decode time, and compressed size. Memory notes below come from the codec designs (see the Zstd / Bzip2 guides).

Zstd levels — CPU for size; decode stays fast

Higher :level spends more encode CPU for a smaller blob. Decompression speed stays roughly flat across levels — that is the main Zstd property worth seeing.

data = String.duplicate("Hello, World! This is a compression test. ", 2000)

IO.puts(
  String.pad_trailing("Level", 8) <>
    String.pad_trailing("Size", 10) <>
    String.pad_trailing("Ratio%", 10) <>
    String.pad_trailing("Encode µs", 12) <>
    "Decode µs"
)

IO.puts(String.duplicate("-", 52))

for level <- [1, 3, 5, 9, 15, 22] do
  {enc_us, {:ok, enc}} = :timer.tc(fn -> ExCodecs.encode(:zstd, data, level: level) end)
  {dec_us, {:ok, ^data}} = :timer.tc(fn -> ExCodecs.decode(:zstd, enc) end)
  ratio = Float.round(100 * byte_size(enc) / byte_size(data), 1)

  IO.puts(
    String.pad_trailing("#{level}", 8) <>
      String.pad_trailing("#{byte_size(enc)}", 10) <>
      String.pad_trailing("#{ratio}", 10) <>
      String.pad_trailing("#{enc_us}", 12) <>
      "#{dec_us}"
  )
end

IO.puts("""

Memory (design, not measured here): higher levels tend to use larger
match windows / tables during *encode*. Decode memory stays modest.
On the BEAM these buffers live in DirtyCpu NIF memory, not the Erlang heap.
""")

Level   Size      Ratio%    Encode µs   Decode µs
----------------------------------------------------
1       64        0.1       140         67
3       64        0.1       73          50
5       64        0.1       72          55
9       64        0.1       179         97
15      64        0.1       603         107
22      61        0.1       634         117

Memory (design, not measured here): higher levels tend to use larger
match windows / tables during *encode*. Decode memory stays modest.
On the BEAM these buffers live in DirtyCpu NIF memory, not the Erlang heap.

:ok

Bzip2 block size — speed, ratio, and memory scale together

:block_size is 1..9. Each step raises the block buffer by about 100 KiB (and roughly half that on decompress). To see block size actually bind, the input below is ~1 MiB, so small block sizes split it into many blocks while large ones use one or two. With that, larger blocks improve the ratio and raise encode time (the BWT is superlinear per block); decode time is roughly flat. Numbers are the mean of 5 runs after a warmup pass.

data = String.duplicate("Hello, World! This is a compression test. ", 25_000)

IO.puts(
  String.pad_trailing("Block", 8) <>
    String.pad_trailing("Size", 10) <>
    String.pad_trailing("Ratio%", 10) <>
    String.pad_trailing("Encode µs", 12) <>
    String.pad_trailing("Decode µs", 12) <>
    "Block buf"
)

IO.puts(String.duplicate("-", 64))

for bs <- 1..9 do
  {:ok, enc} = ExCodecs.encode(:bzip2, data, block_size: bs)
  ExCodecs.decode(:bzip2, enc)

  enc_times =
    for _ <- 1..5 do
      {us, {:ok, _}} = :timer.tc(fn -> ExCodecs.encode(:bzip2, data, block_size: bs) end)
      us
    end

  dec_times =
    for _ <- 1..5 do
      {us, {:ok, _}} = :timer.tc(fn -> ExCodecs.decode(:bzip2, enc) end)
      us
    end

  enc_us = div(Enum.sum(enc_times), 5)
  dec_us = div(Enum.sum(dec_times), 5)
  ratio = Float.round(100 * byte_size(enc) / byte_size(data), 1)
  mem = "#{bs * 100} KiB"

  IO.puts(
    String.pad_trailing("#{bs}", 8) <>
      String.pad_trailing("#{byte_size(enc)}", 10) <>
      String.pad_trailing("#{ratio}", 10) <>
      String.pad_trailing("#{enc_us}", 12) <>
      String.pad_trailing("#{dec_us}", 12) <>
      mem
  )
end

IO.puts("""

Unlike Zstd, Bzip2 decode is also relatively slow — pick it for cold/archival
paths, not hot reads. The "Block buf" column is just the block buffer
(≈ 100 KiB × block_size); total compressor memory adds fixed overhead on top.
Prefer a smaller block_size when concurrent compressions would otherwise stack
many megabytes of NIF memory.
""")

Block   Size      Ratio%    Encode µs   Decode µs   Block buf
----------------------------------------------------------------
1       1622      0.2       119345      5320        100 KiB
2       926       0.1       127456      5116        200 KiB
3       604       0.1       130539      5194        300 KiB
4       492       0.0       130842      5181        400 KiB
5       466       0.0       131821      5027        500 KiB
6       329       0.0       133919      5115        600 KiB
7       328       0.0       135530      4990        700 KiB
8       336       0.0       134722      5011        800 KiB
9       327       0.0       136706      5082        900 KiB

Unlike Zstd, Bzip2 decode is also relatively slow — pick it for cold/archival
paths, not hot reads. The "Block buf" column is just the block buffer
(≈ 100 KiB × block_size); total compressor memory adds fixed overhead on top.
Prefer a smaller block_size when concurrent compressions would otherwise stack
many megabytes of NIF memory.

:ok

Decompression bombs (bounded)

A decompression bomb is a tiny compressed blob that expands into a huge payload. ExCodecs rejects that expansion when it would exceed :max_output_size (default 256 MiB).

The classic shape is a long run of zeros: cheap to compress, expensive to expand. Keep the expanded size modest in demos and set a tight bound below it so the decode fails safely instead of allocating the full output.

expanded_size = 65_536
tight_limit = 1_024
bomb_raw = :binary.copy(<<0>>, expanded_size)

codecs = [
  {:zstd, []},
  {:lz4, []},
  {:snappy, []},
  {:bzip2, []},
  {:blosc2, [cname: :lz4, shuffle: :none, typesize: 1]}
]

for {codec, encode_opts} <- codecs do
  {:ok, bomb} = ExCodecs.encode(codec, bomb_raw, encode_opts)
  ratio = Float.round(expanded_size / byte_size(bomb), 1)

  {:error, %ExCodecs.Error{reason: :output_limit_exceeded}} =
    ExCodecs.decode(codec, bomb, max_output_size: tight_limit)

  {:ok, ^bomb_raw} =
    ExCodecs.decode(codec, bomb, max_output_size: expanded_size)

  IO.puts(
    "#{String.pad_trailing(inspect(codec), 10)} bomb #{byte_size(bomb)} bytes " <>
      "(#{ratio}x) → rejected under #{tight_limit}, OK under #{expanded_size}"
  )
end

:zstd      bomb 11 bytes (5957.8x) → rejected under 1024, OK under 65536
:lz4       bomb 272 bytes (240.9x) → rejected under 1024, OK under 65536
:snappy    bomb 3077 bytes (21.3x) → rejected under 1024, OK under 65536
:bzip2     bomb 43 bytes (1524.1x) → rejected under 1024, OK under 65536
:blosc2    bomb 32 bytes (2048.0x) → rejected under 1024, OK under 65536

[:ok, :ok, :ok, :ok, :ok]

For untrusted inputs, pass an explicit tight :max_output_size. Raise the default only for trusted sources you control.

Key Takeaways

1. Entropy dominates — random data barely compresses; repetitive data compresses well
2. No single best codec — each excels for different data and latency requirements
3. Shuffle transforms (Blosc2) dramatically improve compression of typed binary data
4. Compression level is a dial — higher Zstd levels cost encode CPU; decode stays fast. Bzip2 block_size scales CPU, ratio, and ~100 KiB×N working memory together
5. Bound decompression — high-ratio “bomb” payloads are rejected via :max_output_size
6. Measure your actual data — use the Codec Comparison livebook with your own datasets

Navigation

Previous: Introduction · Next: Codec Comparison