View Source Overview

Hex Docs

SortedSetKV

An ultrafast double-ended queue, scored sorted set, and embedded key value database. Inspired by Redis's sorted sets, but quite different. Written for Elixir and backed by Rust's sled database.

Think of it as a Key Value database with an optional secondary u64 index. Perfect for a TTL or timeseries.

This is the basis of building a KV with a TTL, however I left it abstracted to be used like redis zadd with an optional value field and score field. This means you can use it like a set, use it like a KV, use it like a scored set, use it like a scored KV, or use it as a KV TTL. It is very versitile. And very fast. Everything is local, so you can get 1-4 times faster speeds than using Redis.

Also because you control the TLL, it won't auto evict things. This is very different from Redis which will start evicting keys regardless if their TTL has passed when it runs out of memory.

SortedSetKV is stored on disk and can grow beyond your RAM limit. There is no GenServer abstraction and calls are made directly to Rust.

Disclaimer

This is alpha software and the API can change in the future.

Installation

SortedSetKV can be installed by adding :sorted_set_kv to your list of dependencies in mix.exs:

def deps do
  [
    {:sorted_set_kv, "~> 0.1.0"}
  ]
end

Basic Usage

{:ok, db} = SortedSetKV.open("mypath")
# Add a key to a set, with a value and a score.
# The last parameter tells to only add if the old score is less than new score.
:ok = SortedSetKV.zadd(db, "mycollection", "hello", "world", 42, true)
:ok = SortedSetKV.zadd(db, "mycollection", "foo", "bar", 420, true)
:ok = SortedSetKV.zadd(db, "mycollection", "noscore", "", nil, true)
:ok = SortedSetKV.zadd(db, "mycollection", "novalue", nil, 100, true)
# Returns whether it exists and its score
{true, 42} = SortedSetKV.zscore(db, "mycollection", "hello")
{true, 420} = SortedSetKV.zscore(db, "mycollection", "foo")
{true, nil} = SortedSetKV.zscore(db, "mycollection", "noscore")
{true, 100} = SortedSetKV.zscore(db, "mycollection", "novalue")

# A key must have a score or a value to exist:
:ok = SortedSetKV.zadd(db, "mycollection", "noexists", nil, nil, true)
{false, nil} = SortedSetKV.zscore(db, "mycollection", "noexists")

Retrieving Values

# Get a key with a minimum score
{value, score} = SortedSetKV.zgetbykey(db, "mycollection", "hello", 0)
# A key with a score lower than the minscore will return nil
nil = SortedSetKV.zgetbykey(db, "mycollection", "foo", 500)
# see if any keys exist with the score
true = SortedSetKV.zexists(db, "mycollection", 0, 500)

Conditional Add

With zadd and zupdate, you can optionally only update the score if the new score is greater than the old score or if the old score is not set.

:ok = SortedSetKV.zadd(db, "mycollection", "hello", "world", 42, true)
{"world", 42} = SortedSetKV.zgetbykey(db, "mycollection", "hello", 0)
:ok = SortedSetKV.zadd(db, "mycollection", "hello", "value2", 0, true)
# only adds if the score is greather than
{"world", 42} = SortedSetKV.zgetbykey(db, "mycollection", "hello", 0)

:ok = SortedSetKV.zscoreupdate(db, "mycollection", "hello", 0, true)
{"world", 42} = SortedSetKV.zgetbykey(db, "mycollection", "hello", 0)

# Setting the value to false overrides this
:ok = SortedSetKV.zadd(db, "mycollection", "hello", "value2", 10, false)
{"value2", 10} = SortedSetKV.zgetbykey(db, "mycollection", "hello", 0)

:ok = SortedSetKV.zscoreupdate(db, "mycollection", "hello", 0, false)
{"value2", 0} = SortedSetKV.zgetbykey(db, "mycollection", "hello", 0)

Iterating keys with scores

offset = 0
limit = 100
["hello"] = SortedSetKV.zrangebyscore(db, "mycollection", 0, 50, offset, limit)
# Filter by prefix and score
["foo"] = SortedSetKV.zrangebyprefixscore(db, "mycollection", "fo", 0, 500, offset, limit)

Removing Values

# Remove key
:ok = SortedSetKV.zrem(db, "mycollection", "hello")
# Remove all keys by score and returns how many it deleted
_ = SortedSetKV.zrembyrangebyscore(db, "mycollection", 0, 500, limit)

Queue

:ok = SortedSetKV.rpush(db, "mylist", "value")
"value" = SortedSetKV.lpop(db, "mylist")
nil = SortedSetKV.lpop(db, "mylist")
:ok = SortedSetKV.rpush(db, "mylist", "1")
:ok = SortedSetKV.rpush(db, "mylist", "2")
:ok = SortedSetKV.lpush(db, "mylist", "0")
"0" = SortedSetKV.lpop(db, "mylist")
"2" = SortedSetKV.rpop(db, "mylist")

TTL

If you use millisecond timestamps as the score, it behaves like a TTL.

{:ok, db} = SortedSetKV.open("mypath")
# Add a key to a set, with a value and a score
:ok = SortedSetKV.zadd(db, "mycollection", "hello", "world", :os.system_time(:millisecond) + 5000)
# Get key only if it is in TTL
SortedSetKV.zgetbykey(db, "mycollection", "foo", :os.system_time(:millisecond))

# Clean up exipired Keys
SortedSetKV.zrembyrangebyscore(db, "mycollection", 0, :os.system_time(:millisecond))

You can use a GenServer like this to customize your TTL cleanup. Because Elixir executes all Rust Nifs on one thread, you will not want to block for very long. It is wise to only delete a few keys at a time.

defmodule TTLCleanup do
    use GenServer
    require Logger

    @review_time 5_000

    def start_link(conn) do
      GenServer.start_link(__MODULE__, conn, [])
    end

    def init(conn) do
      Process.send_after(self(), :review_storage, @review_time)
      {:ok, conn}
    end

    def handle_info(:review_storage, conn) do
      Logger.debug("TTL cleanup")

      :ok = scan(conn, "collection1")
      :ok = scan(conn, "collection2")
      :ok = scan(conn, "collection3")

      Process.send_after(self(), :review_storage, @review_time)

      {:noreply, conn}
    end

    def scan(conn, collection) do
      new_agg =
        SortedSetKV.zrembyrangebyscore(conn, collection, 0, :os.system_time(:millisecond), 100)

      case new_agg do
        v when is_number(v) and v <= 99 ->
          :ok

        v when is_number(v) ->
          scan(conn, collection)
      end
    end
end