-module(kura_ets_query).
-moduledoc """
Plan interpreter for the ETS backend. Executes the `{kura_ets, Plan}`
terms emitted by `kura_ets_dialect` against the pool's ETS tables and
returns pgo-shaped results (`#{command, rows, num_rows}`).
## Storage model
One `set` table per source table. Objects are `{Key, Row}` where `Row`
is a map of dumped field values (as produced by kura's type system on
the write path) and `Key` is the primary-key value — a bare value for
single-column keys, a tuple for composite keys, or a unique reference
when no primary key can be derived from the schema.
Missing columns are materialized as `null` at insert (with schema
defaults applied first), mirroring what a SQL backend would return,
so `is_nil` conditions and `kura_db:load_row/2` behave identically.
## Filtering
Queries are evaluated with a plain scan-and-filter over the table
(`ets:tab2list/1`), not with match specifications: kura's where
grammar (like/ilike, composite in, between, nested and/or/not) maps
poorly onto match-spec guards, and for the in-memory datasets this
backend targets a scan is fast enough and far simpler to verify.
One exception: when the top-level wheres pin every primary-key column
with an equality, the row is fetched with `ets:lookup/2` (the storage
key is the primary key), making point reads O(1) — the cache-aside
hot path.
Comparison values are passed through `kura_types:dump/2` when the
schema declares a type for the field, so they compare against the
stored (dumped) representation the same way SQL params are encoded by
a database client.
SQL null semantics are approximated: a comparison against a null
(`undefined`/`null`) field value is false, except for the explicit
`is_nil` / `is_not_nil` conditions.
Unsupported constructs (joins, group_by, having, CTEs, combinations,
subqueries, fragments, `matches`, locks, prefix multitenancy) yield
`{error, {kura_ets_unsupported, What}}`.
""".
-include_lib("kura/include/kura.hrl").
-export([run/2]).
-define(AGGS, [count, sum, avg, min, max]).
-spec run(atom(), kura_ets_dialect:plan()) -> dynamic().
run(Pool, Plan) ->
try
do_run(Pool, Plan)
catch
throw:{kura_ets_unsupported, _} = Err -> {error, Err}
end.
%%----------------------------------------------------------------------
%% SELECT
%%----------------------------------------------------------------------
do_run(Pool, {select, Q = #kura_query{from = From}}) ->
ok = assert_supported(Q),
with_table(Pool, From, fun(Tid) ->
Types = types_of(From),
Matched =
case pk_lookup(Tid, From, Q#kura_query.wheres, Types) of
{hit, PkRows} -> PkRows;
scan -> filter_rows(Tid, Q#kura_query.wheres, Types)
end,
Rows =
case aggregate_select(Q#kura_query.select) of
true ->
aggregate(Q#kura_query.select, Matched);
false ->
Sorted = order(Matched, Q#kura_query.order_bys),
Deduped = distinct_on(Sorted, Q#kura_query.distinct),
Projected = project(Deduped, Q#kura_query.select),
Distinct = distinct_all(Projected, Q#kura_query.distinct),
limit(offset(Distinct, Q#kura_query.offset), Q#kura_query.limit)
end,
#{command => select, rows => Rows, num_rows => length(Rows)}
end);
%%----------------------------------------------------------------------
%% INSERT
%%----------------------------------------------------------------------
do_run(Pool, {insert, Schema, Data, Opts}) ->
with_table(Pool, Schema, fun(Tid) ->
Row = fill_defaults(Schema, Data),
Key = row_key(Schema, Row),
insert_one(Tid, Key, Row, maps:get(on_conflict, Opts, undefined), Schema)
end);
do_run(Pool, {insert_all, Schema, Rows0, Opts}) ->
with_table(Pool, Schema, fun(Tid) ->
Rows = [fill_defaults(Schema, R) || R <- Rows0],
Objects = [{row_key(Schema, R), R} || R <- Rows],
case insert_many(Tid, Objects, maps:get(on_conflict, Opts, undefined), Schema) of
{ok, Inserted} ->
Base = #{command => insert, num_rows => length(Inserted)},
case maps:get(returning, Opts, false) of
false ->
Base#{rows => []};
true ->
Base#{rows => Inserted};
Fields when is_list(Fields) ->
Base#{rows => [maps:with(Fields, R) || R <- Inserted]}
end;
{error, _} = Err ->
Err
end
end);
%%----------------------------------------------------------------------
%% UPDATE / DELETE by primary key
%%----------------------------------------------------------------------
do_run(Pool, {update, Schema, Changes, KeyClauses}) ->
with_table(Pool, Schema, fun(Tid) ->
Types = types_of(Schema),
case find_by_clauses(Tid, KeyClauses, Types) of
[] ->
#{command => update, rows => [], num_rows => 0};
[{Key, Old} | _] ->
New = maps:merge(Old, Changes),
case replace_row(Tid, Key, Schema, New) of
ok -> #{command => update, rows => [New], num_rows => 1};
{error, _} = Err -> Err
end
end
end);
do_run(Pool, {delete, Schema, KeyClauses}) ->
with_table(Pool, Schema, fun(Tid) ->
Types = types_of(Schema),
case find_by_clauses(Tid, KeyClauses, Types) of
[] ->
#{command => delete, rows => [], num_rows => 0};
[{Key, Old} | _] ->
true = ets:delete(Tid, Key),
#{command => delete, rows => [Old], num_rows => 1}
end
end);
%%----------------------------------------------------------------------
%% Bulk UPDATE / DELETE
%%----------------------------------------------------------------------
do_run(Pool, {update_all, Q = #kura_query{from = From}, SetMap}) ->
ok = assert_supported(Q),
with_table(Pool, From, fun(Tid) ->
Types = types_of(From),
Changes = dump_values(SetMap, Types),
Matched = filter_objects(Tid, Q#kura_query.wheres, Types),
Result = lists:foldl(
fun
({Key, Old}, N) when is_integer(N) ->
case replace_row(Tid, Key, From, maps:merge(Old, Changes)) of
ok -> N + 1;
{error, _} = Err -> Err
end;
(_, Err) ->
Err
end,
0,
Matched
),
case Result of
N when is_integer(N) -> #{command => update, rows => [], num_rows => N};
{error, _} = Err -> Err
end
end);
do_run(Pool, {delete_all, Q = #kura_query{from = From}}) ->
ok = assert_supported(Q),
with_table(Pool, From, fun(Tid) ->
Types = types_of(From),
Matched = filter_objects(Tid, Q#kura_query.wheres, Types),
lists:foreach(fun({Key, _}) -> true = ets:delete(Tid, Key) end, Matched),
#{command => delete, rows => [], num_rows => length(Matched)}
end).
%%----------------------------------------------------------------------
%% Insert internals
%%----------------------------------------------------------------------
insert_one(Tid, Key, Row, undefined, Schema) ->
case ets:insert_new(Tid, {Key, Row}) of
true -> #{command => insert, rows => [Row], num_rows => 1};
false -> unique_violation(Schema)
end;
insert_one(Tid, Key, Row, {_Target, nothing}, _Schema) ->
case ets:insert_new(Tid, {Key, Row}) of
true -> #{command => insert, rows => [Row], num_rows => 1};
false -> #{command => insert, rows => [], num_rows => 0}
end;
insert_one(Tid, Key, Row, {_Target, replace_all}, _Schema) ->
New = upsert(Tid, Key, Row, fun(Old) -> maps:merge(Old, Row) end),
#{command => insert, rows => [New], num_rows => 1};
insert_one(Tid, Key, Row, {_Target, {replace, Fields}}, _Schema) ->
New = upsert(Tid, Key, Row, fun(Old) -> maps:merge(Old, maps:with(Fields, Row)) end),
#{command => insert, rows => [New], num_rows => 1};
insert_one(_Tid, _Key, _Row, Other, _Schema) ->
throw({kura_ets_unsupported, {on_conflict, Other}}).
%% The conflict target is not checked against the schema: ETS enforces
%% uniqueness only on the primary key, so every conflict clause is
%% interpreted as targeting the primary key.
upsert(Tid, Key, Row, MergeFun) ->
New =
case ets:lookup(Tid, Key) of
[] -> Row;
[{_, Old}] -> MergeFun(Old)
end,
true = ets:insert(Tid, {Key, New}),
New.
insert_many(Tid, Objects, undefined, Schema) ->
case ets:insert_new(Tid, Objects) of
true -> {ok, [Row || {_, Row} <- Objects]};
false -> unique_violation(Schema)
end;
insert_many(Tid, Objects, {_Target, nothing}, _Schema) ->
Inserted = [Row || {Key, Row} <- Objects, ets:insert_new(Tid, {Key, Row})],
{ok, Inserted};
insert_many(Tid, Objects, {_Target, replace_all}, _Schema) ->
{ok, [upsert(Tid, Key, Row, fun(Old) -> maps:merge(Old, Row) end) || {Key, Row} <- Objects]};
insert_many(Tid, Objects, {_Target, {replace, Fields}}, _Schema) ->
{ok, [
upsert(Tid, Key, Row, fun(Old) -> maps:merge(Old, maps:with(Fields, Row)) end)
|| {Key, Row} <- Objects
]};
insert_many(_Tid, _Objects, Other, _Schema) ->
throw({kura_ets_unsupported, {on_conflict, Other}}).
%% Reinsert a row under its (possibly changed) primary key.
replace_row(Tid, OldKey, Schema, New) ->
case pk_key(Schema, New) of
{ok, NewKey} when NewKey =/= OldKey ->
case ets:member(Tid, NewKey) of
true ->
unique_violation(Schema);
false ->
true = ets:delete(Tid, OldKey),
true = ets:insert(Tid, {NewKey, New}),
ok
end;
_ ->
true = ets:insert(Tid, {OldKey, New}),
ok
end.
unique_violation(Schema) ->
Table = table_of(Schema),
{error, #{
code => ~"23505",
constraint => <
>,
message => ~"duplicate key value violates unique constraint"
}}.
%%----------------------------------------------------------------------
%% Row keys and defaults
%%----------------------------------------------------------------------
row_key(Schema, Row) ->
case pk_key(Schema, Row) of
{ok, Key} -> Key;
error -> erlang:make_ref()
end.
pk_key(Schema, Row) ->
try kura_schema:key(Schema) of
[Field] -> {ok, maps:get(Field, Row, null)};
Fields -> {ok, list_to_tuple([maps:get(F, Row, null) || F <- Fields])}
catch
_:_ -> error
end.
%% Apply schema defaults and materialize missing columns as null, like
%% a SQL insert would.
fill_defaults(Schema, Data) ->
try Schema:fields() of
Fields ->
lists:foldl(
fun
(#kura_field{virtual = true}, Acc) ->
Acc;
(#kura_field{name = N, type = T, default = D}, Acc) ->
case maps:is_key(N, Acc) of
true -> Acc;
false -> Acc#{N => dump_default(T, D)}
end
end,
Data,
Fields
)
catch
_:_ -> Data
end.
dump_default(_Type, undefined) ->
null;
dump_default(Type, Default) ->
case kura_types:dump(Type, Default) of
{ok, V} -> V;
{error, _} -> Default
end.
%%----------------------------------------------------------------------
%% Table / schema resolution
%%----------------------------------------------------------------------
with_table(Pool, SchemaOrTable, Fun) ->
case kura_ets_pool:ensure_table(Pool, table_of(SchemaOrTable)) of
{ok, Tid} -> Fun(Tid);
{error, _} = Err -> Err
end.
table_of(Mod) when is_atom(Mod) ->
case code:ensure_loaded(Mod) of
{module, Mod} ->
case erlang:function_exported(Mod, table, 0) of
true -> Mod:table();
false -> atom_to_binary(Mod, utf8)
end;
_ ->
atom_to_binary(Mod, utf8)
end.
types_of(SchemaOrTable) ->
try
kura_schema:field_types(SchemaOrTable)
catch
_:_ -> #{}
end.
assert_supported(#kura_query{joins = [_ | _]}) ->
throw({kura_ets_unsupported, joins});
assert_supported(#kura_query{group_bys = [_ | _]}) ->
throw({kura_ets_unsupported, group_by});
assert_supported(#kura_query{havings = [_ | _]}) ->
throw({kura_ets_unsupported, having});
assert_supported(#kura_query{ctes = [_ | _]}) ->
throw({kura_ets_unsupported, ctes});
assert_supported(#kura_query{combinations = [_ | _]}) ->
throw({kura_ets_unsupported, combinations});
assert_supported(#kura_query{lock = Lock}) when Lock =/= undefined ->
throw({kura_ets_unsupported, lock});
assert_supported(#kura_query{prefix = Prefix}) when Prefix =/= undefined ->
throw({kura_ets_unsupported, prefix});
assert_supported(#kura_query{}) ->
ok.
%%----------------------------------------------------------------------
%% Filtering
%%----------------------------------------------------------------------
filter_rows(Tid, Wheres, Types) ->
[Row || {_, Row} <- filter_objects(Tid, Wheres, Types)].
%% Point-lookup fast path: when the top-level where conditions contain
%% an equality for every primary-key column, the storage key is fully
%% determined and the row can be fetched with ets:lookup instead of a
%% table scan. The full condition list is still evaluated against the
%% fetched row, so extra conditions (soft-delete filters, tenant
%% attributes, conflicting duplicates) keep their exact semantics.
pk_lookup(Tid, Schema, Wheres, Types) ->
try kura_schema:key(Schema) of
KeyFields ->
case eq_values(Wheres, KeyFields, Types) of
{ok, [Single]} ->
pk_fetch(Tid, Single, Wheres, Types);
{ok, Composite} ->
pk_fetch(Tid, list_to_tuple(Composite), Wheres, Types);
error ->
scan
end
catch
_:_ -> scan
end.
pk_fetch(Tid, Key, Wheres, Types) ->
{hit, [Row || {_, Row} <- ets:lookup(Tid, Key), matches(Wheres, Row, Types)]}.
%% Extract one (dumped) equality value per key column from the
%% top-level (implicitly ANDed) conditions. Any key column without an
%% equality means the key is underdetermined -> scan.
eq_values(Wheres, KeyFields, Types) ->
Eqs = lists:foldl(
fun
({F, '=', V}, Acc) when is_atom(F) ->
Acc#{F => dump_value(F, V, Types)};
({F, V}, Acc) when
is_atom(F),
F =/= 'and',
F =/= 'or',
F =/= 'not',
V =/= is_nil,
V =/= is_not_nil
->
Acc#{F => dump_value(F, V, Types)};
(_, Acc) ->
Acc
end,
#{},
Wheres
),
case lists:all(fun(F) -> maps:is_key(F, Eqs) end, KeyFields) of
true -> {ok, [maps:get(F, Eqs) || F <- KeyFields]};
false -> error
end.
filter_objects(Tid, Wheres, Types) ->
[Obj || {_, Row} = Obj <- ets:tab2list(Tid), matches(Wheres, Row, Types)].
find_by_clauses(Tid, KeyClauses, Types) ->
filter_objects(Tid, KeyClauses, Types).
matches(Conditions, Row, Types) ->
lists:all(fun(C) -> eval(C, Row, Types) end, Conditions).
eval({'and', Conditions}, Row, Types) ->
lists:all(fun(C) -> eval(C, Row, Types) end, Conditions);
eval({'or', Conditions}, Row, Types) ->
lists:any(fun(C) -> eval(C, Row, Types) end, Conditions);
eval({'not', Condition}, Row, Types) ->
not eval(Condition, Row, Types);
eval({Field, is_nil}, Row, _Types) when is_atom(Field) ->
is_null(field(Field, Row));
eval({Field, is_not_nil}, Row, _Types) when is_atom(Field) ->
not is_null(field(Field, Row));
eval({Field, '=', Value}, Row, Types) when is_atom(Field) ->
eval({Field, Value}, Row, Types);
eval({Field, '!=', Value}, Row, Types) when is_atom(Field) ->
non_null_cmp(field(Field, Row), dump_value(Field, Value, Types), fun(A, B) -> A /= B end);
eval({Field, '<', Value}, Row, Types) when is_atom(Field) ->
non_null_cmp(field(Field, Row), dump_value(Field, Value, Types), fun(A, B) -> A < B end);
eval({Field, '>', Value}, Row, Types) when is_atom(Field) ->
non_null_cmp(field(Field, Row), dump_value(Field, Value, Types), fun(A, B) -> A > B end);
eval({Field, '<=', Value}, Row, Types) when is_atom(Field) ->
non_null_cmp(field(Field, Row), dump_value(Field, Value, Types), fun(A, B) -> A =< B end);
eval({Field, '>=', Value}, Row, Types) when is_atom(Field) ->
non_null_cmp(field(Field, Row), dump_value(Field, Value, Types), fun(A, B) -> A >= B end);
eval({Field, like, Pattern}, Row, _Types) when is_atom(Field) ->
like(field(Field, Row), Pattern, []);
eval({Field, ilike, Pattern}, Row, _Types) when is_atom(Field) ->
like(field(Field, Row), Pattern, [caseless]);
eval({Field, between, {Low, High}}, Row, Types) when is_atom(Field) ->
V = field(Field, Row),
L = dump_value(Field, Low, Types),
H = dump_value(Field, High, Types),
non_null_cmp(V, L, fun(A, B) -> A >= B end) andalso
non_null_cmp(V, H, fun(A, B) -> A =< B end);
eval({Field, in, Values}, Row, Types) when is_atom(Field), is_list(Values) ->
V = field(Field, Row),
lists:any(fun(Val) -> sql_eq(V, dump_value(Field, Val, Types)) end, Values);
eval({Field, not_in, Values}, Row, Types) when is_atom(Field), is_list(Values) ->
V = field(Field, Row),
not is_null(V) andalso
not lists:any(fun(Val) -> sql_eq(V, dump_value(Field, Val, Types)) end, Values);
eval({[Col], in, Tuples}, Row, Types) when is_atom(Col), is_list(Tuples) ->
eval({Col, in, [element(1, T) || T <- Tuples]}, Row, Types);
eval({Cols, in, Tuples}, Row, Types) when is_list(Cols), is_list(Tuples) ->
RowVals = [field(C, Row) || C <- Cols],
lists:any(
fun(Tuple) ->
Vals = [dump_value(C, V, Types) || {C, V} <- lists:zip(Cols, tuple_to_list(Tuple))],
lists:all(fun({A, B}) -> sql_eq(A, B) end, lists:zip(RowVals, Vals))
end,
Tuples
);
eval({Field, Value}, Row, Types) when is_atom(Field) ->
sql_eq(field(Field, Row), dump_value(Field, Value, Types));
eval(Condition, _Row, _Types) ->
throw({kura_ets_unsupported, {condition, Condition}}).
field(Field, Row) ->
maps:get(Field, Row, undefined).
is_null(undefined) -> true;
is_null(null) -> true;
is_null(_) -> false.
%% SQL comparisons against NULL are never true.
sql_eq(A, B) ->
non_null_cmp(A, B, fun(X, Y) -> X == Y end).
non_null_cmp(A, B, Cmp) ->
not is_null(A) andalso not is_null(B) andalso Cmp(A, B).
%% Encode a comparison value the way a SQL client would encode a query
%% parameter, so it compares against the stored (dumped) form.
dump_value(Field, Value, Types) ->
case Types of
#{Field := Type} ->
case kura_types:dump(Type, Value) of
{ok, Dumped} -> Dumped;
{error, _} -> Value
end;
#{} ->
Value
end.
dump_values(Map, Types) ->
maps:map(fun(Field, Value) -> dump_value(Field, Value, Types) end, Map).
%%----------------------------------------------------------------------
%% LIKE / ILIKE
%%----------------------------------------------------------------------
like(Value, Pattern, ReOpts) when is_binary(Value), is_binary(Pattern) ->
RE = like_to_regex(Pattern),
case re:run(Value, RE, [{capture, none}, unicode | ReOpts]) of
match -> true;
_ -> false
end;
like(_Value, _Pattern, _ReOpts) ->
false.
like_to_regex(Pattern) ->
Inner = <<<<(like_char(C))/binary>> || <> <= Pattern>>,
<<"^", Inner/binary, "$">>.
like_char($%) ->
<<".*">>;
like_char($_) ->
<<".">>;
like_char(C) ->
case lists:member(C, ".^$*+?()[]{}|\\") of
true -> <<$\\, C/utf8>>;
false -> <>
end.
%%----------------------------------------------------------------------
%% Ordering / projection / distinct / paging
%%----------------------------------------------------------------------
order(Rows, []) ->
Rows;
order(Rows, Orders) ->
lists:sort(fun(A, B) -> order_cmp(Orders, A, B) end, Rows).
order_cmp([], _A, _B) ->
true;
order_cmp([{Field, Dir} | Rest], A, B) ->
Va = field(Field, A),
Vb = field(Field, B),
case {Va == Vb, Dir} of
{true, _} -> order_cmp(Rest, A, B);
{false, asc} -> Va < Vb;
{false, desc} -> Va > Vb
end.
%% DISTINCT ON (fields): keep the first row per field-value group,
%% preserving sort order. Runs before projection so the distinct
%% fields need not be selected.
distinct_on(Rows, Fields) when is_list(Fields) ->
dedupe(Rows, fun(Row) -> maps:with(Fields, Row) end);
distinct_on(Rows, _) ->
Rows.
%% Plain DISTINCT: dedupe identical projected rows.
distinct_all(Rows, true) ->
dedupe(Rows, fun(Row) -> Row end);
distinct_all(Rows, _) ->
Rows.
dedupe(Rows, KeyFun) ->
{Deduped, _} = lists:foldl(
fun(Row, {Acc, Seen}) ->
Key = KeyFun(Row),
case maps:is_key(Key, Seen) of
true -> {Acc, Seen};
false -> {[Row | Acc], Seen#{Key => true}}
end
end,
{[], #{}},
Rows
),
lists:reverse(Deduped).
project(Rows, []) ->
Rows;
project(Rows, {exprs, Exprs}) ->
Pairs = [expr_pair(E) || E <- Exprs],
[maps:from_list([{Alias, field(F, Row)} || {Alias, F} <- Pairs]) || Row <- Rows];
project(Rows, Fields) when is_list(Fields) ->
case lists:all(fun is_atom/1, Fields) of
true -> [maps:with(Fields, Row) || Row <- Rows];
false -> throw({kura_ets_unsupported, {select, Fields}})
end.
expr_pair({Alias, Field}) when is_atom(Alias), is_atom(Field) ->
{Alias, Field};
expr_pair(Expr) ->
throw({kura_ets_unsupported, {select_expr, Expr}}).
offset(Rows, undefined) -> Rows;
offset(Rows, N) when N >= length(Rows) -> [];
offset(Rows, N) -> lists:nthtail(N, Rows).
limit(Rows, undefined) -> Rows;
limit(Rows, N) -> lists:sublist(Rows, N).
%%----------------------------------------------------------------------
%% Aggregates
%%----------------------------------------------------------------------
aggregate_select(Select) when is_list(Select), Select =/= [] ->
case lists:any(fun is_agg/1, Select) of
true ->
lists:all(fun is_agg/1, Select) orelse
throw({kura_ets_unsupported, {select, Select}}),
true;
false ->
false
end;
aggregate_select(_) ->
false.
is_agg({Agg, Field}) when is_atom(Field) -> lists:member(Agg, ?AGGS);
is_agg(_) -> false.
aggregate(Select, Rows) ->
[lists:foldl(fun(Spec, Acc) -> agg(Spec, Rows, Acc) end, #{}, Select)].
agg({count, '*'}, Rows, Acc) ->
Acc#{count => length(Rows)};
agg({count, Field}, Rows, Acc) ->
Acc#{count => length(agg_values(Field, Rows))};
agg({sum, Field}, Rows, Acc) ->
Acc#{sum => sum_or_null(agg_values(Field, Rows))};
agg({avg, Field}, Rows, Acc) ->
Acc#{
avg =>
case agg_values(Field, Rows) of
[] -> null;
Vs -> lists:sum(Vs) / length(Vs)
end
};
agg({min, Field}, Rows, Acc) ->
Acc#{min => min_max_or_null(fun lists:min/1, agg_values(Field, Rows))};
agg({max, Field}, Rows, Acc) ->
Acc#{max => min_max_or_null(fun lists:max/1, agg_values(Field, Rows))}.
agg_values(Field, Rows) ->
[V || Row <- Rows, V <- [field(Field, Row)], not is_null(V)].
sum_or_null([]) -> null;
sum_or_null(Vs) -> lists:sum(Vs).
min_max_or_null(_Fun, []) -> null;
min_max_or_null(Fun, Vs) -> Fun(Vs).