defmodule ExSQL.Executor do @moduledoc """ Statement execution against an `ExSQL.Database`. Where SQLite compiles statements to VDBE bytecode and runs them in a register VM (`vdbe.c`, 190 opcodes), this first implementation walks the AST directly — the natural functional starting point, and the layer a bytecode compiler can replace later without touching parsing or storage. ## Environments Expressions evaluate inside an environment holding the current row of each table in scope: * `frames` — one frame per FROM source (table or subquery), each with the source's alias, its column order, and the current row. Joins produce envs whose frame list has one entry per joined source. * `outer` — the enclosing query's env, for correlated subqueries; column resolution falls back outward like SQLite's `resolveExprStep`. * `group` — in aggregate queries, the list of member envs of the current group; aggregate functions consume it, bare columns delegate to the first member. All functions are pure: they take a database value and return `{:ok, result, new_database}` or `{:error, %ExSQL.Error{}}`. """ alias ExSQL.AST.{ AlterTable, ColumnDef, Compound, CreateIndex, CreateTable, CreateTrigger, CreateView, Delete, DropIndex, DropTable, DropView, Insert, Pragma, Select, Update, Values, With } alias ExSQL.{Database, DateTime, Error, Json, Parser, Result, Table, Value} @aggregate_functions ~w(count sum avg total min max group_concat string_agg json_group_array json_group_object jsonb_group_array jsonb_group_object) @window_functions ~w(row_number rank dense_rank ntile lag lead first_value last_value nth_value percent_rank cume_dist) @rowid_names ~w(rowid oid _rowid_) @max_trigger_depth 1000 @sqlite_version "3.51.0" @supported_pragmas [ "analysis_limit", "application_id", "auto_vacuum", "automatic_index", "busy_timeout", "cache_size", "cache_spill", "case_sensitive_like", "cell_size_check", "checkpoint_fullfsync", "collation_list", "compile_options", "count_changes", "data_version", "database_list", "default_cache_size", "defer_foreign_keys", "empty_result_callbacks", "encoding", "foreign_key_check", "foreign_key_list", "foreign_keys", "freelist_count", "full_column_names", "fullfsync", "function_list", "hard_heap_limit", "ignore_check_constraints", "incremental_vacuum", "index_info", "index_list", "index_xinfo", "integrity_check", "journal_mode", "journal_size_limit", "locking_mode", "max_page_count", "mmap_size", "module_list", "optimize", "page_count", "page_size", "pragma_list", "query_only", "quick_check", "read_uncommitted", "recursive_triggers", "reverse_unordered_selects", "schema_version", "secure_delete", "short_column_names", "shrink_memory", "soft_heap_limit", "stats", "synchronous", "table_info", "table_list", "table_xinfo", "temp_store", "threads", "trusted_schema", "user_version", "wal_autocheckpoint", "wal_checkpoint" ] @compile_options [ "DEFAULT_CACHE_SIZE=2000", "DEFAULT_JOURNAL_SIZE_LIMIT=32768", "DEFAULT_PAGE_SIZE=4096", "DEFAULT_RECURSIVE_TRIGGERS=0", "DEFAULT_SYNCHRONOUS=2", "DEFAULT_WAL_AUTOCHECKPOINT=1000", "ENABLE_JSON1", "ENABLE_MATH_FUNCTIONS", "ENABLE_WINDOW_FUNCTIONS", "MAX_FUNCTION_ARG=127", "MAX_VARIABLE_NUMBER=32766", "THREADSAFE=0" ] # Known scalar functions and the argument counts they accept, for SQLite's # distinction between "no such function" and "wrong number of arguments". @scalar_arity %{ "abs" => 1..1, "acos" => 1..1, "acosh" => 1..1, "asin" => 1..1, "asinh" => 1..1, "atan" => 1..1, "atan2" => 2..2, "atanh" => 1..1, "char" => 0..127, "ceil" => 1..1, "ceiling" => 1..1, "changes" => 0..0, "coalesce" => 2..127, "concat" => 1..127, "concat_ws" => 2..127, "cos" => 1..1, "cosh" => 1..1, "degrees" => 1..1, "exp" => 1..1, "floor" => 1..1, "glob" => 2..2, "format" => 1..127, "hex" => 1..1, "ifnull" => 2..2, "iif" => 3..3, "instr" => 2..2, "json" => 1..1, "jsonb" => 1..1, "jsonb_array" => 0..127, "jsonb_extract" => 2..127, "jsonb_insert" => 3..127, "jsonb_object" => 0..127, "jsonb_patch" => 2..2, "jsonb_remove" => 2..127, "jsonb_replace" => 3..127, "jsonb_set" => 3..127, "json_array" => 0..127, "json_array_length" => 1..2, "json_extract" => 2..127, "json_insert" => 3..127, "json_object" => 0..127, "json_patch" => 2..2, "json_pretty" => 1..2, "json_quote" => 1..1, "json_remove" => 2..127, "json_replace" => 3..127, "json_set" => 3..127, "json_type" => 1..2, "json_valid" => 1..2, "length" => 1..1, "last_insert_rowid" => 0..0, "like" => 2..3, "ln" => 1..1, "log" => 1..2, "log10" => 1..1, "log2" => 1..1, "lower" => 1..1, "ltrim" => 1..2, "match" => 2..2, "max" => 1..127, "min" => 1..127, "mod" => 2..2, "nullif" => 2..2, "octet_length" => 1..1, "pi" => 0..0, "pow" => 2..2, "power" => 2..2, "printf" => 1..127, "quote" => 1..1, "radians" => 1..1, "random" => 0..0, "randomblob" => 1..1, "regexp" => 2..2, "replace" => 3..3, "round" => 1..2, "rtrim" => 1..2, "sign" => 1..1, "sin" => 1..1, "sinh" => 1..1, "sqlite_version" => 0..0, "sqrt" => 1..1, "substr" => 2..3, "substring" => 2..3, "tan" => 1..1, "tanh" => 1..1, "trim" => 1..2, "total_changes" => 0..0, "trunc" => 1..1, "typeof" => 1..1, "unhex" => 1..2, "unicode" => 1..1, "upper" => 1..1, "zeroblob" => 1..1, # Date/time functions (date.c) "date" => 1..127, "time" => 1..127, "timediff" => 2..2, "datetime" => 0..127, "julianday" => 1..127, "unixepoch" => 1..127, "strftime" => 2..127 } @explain_literal_scalar_functions ~w(abs acos acosh asin asinh atan atan2 atanh ceil ceiling char concat concat_ws cos cosh degrees exp floor format hex instr length ln log log10 log2 lower ltrim mod octet_length pi pow power printf quote radians replace round rtrim sign sin sinh sqrt substr substring tan tanh trim trunc typeof unicode upper zeroblob) @doc """ Parses and executes every statement in `sql`, threading the database through. Returns the results in statement order. A failing statement stops execution; effects of prior statements in the same string are kept (as with `sqlite3_exec`), so the error tuple carries the database too. """ @spec run(Database.t(), String.t(), [Value.t()] | map()) :: {:ok, [Result.t()], Database.t()} | {:error, Error.t(), Database.t()} def run(db, sql, params \\ []) do case Parser.parse(sql) do {:ok, statements} -> statements |> Enum.reduce_while({:ok, [], db}, fn stmt, {:ok, results, db} -> case bind_and_execute(db, stmt, params) do {:ok, result, db} -> {:cont, {:ok, [result | results], db}} # OR FAIL / OR ROLLBACK leave effects behind even though the # statement errors; the error carries that database state. {:error, %Error{db: %Database{} = error_db} = error} -> {:halt, {:error, %{error | db: nil}, error_db}} {:error, error} -> {:halt, {:error, error, db}} end end) |> case do {:ok, results, db} -> {:ok, Enum.reverse(results), db} error -> error end {:error, error} -> {:error, error, db} end end defp bind_and_execute(db, stmt, params) do execute(db, bind_parameters(stmt, params)) rescue e in Error -> {:error, e} end # Substitutes bound values for `{:param, index, name}` expressions across a # statement. Indexes were assigned in source order by the tokenizer; # parameters left unbound evaluate to NULL, as in the C API. defp bind_parameters(stmt, params) when params == [] or params == %{}, do: stmt defp bind_parameters(stmt, params), do: bind_walk(stmt, params) defp bind_walk({:param, index, raw}, params), do: {:literal, param_value(params, index, raw)} defp bind_walk(tuple, params) when is_tuple(tuple) do tuple |> Tuple.to_list() |> Enum.map(&bind_walk(&1, params)) |> List.to_tuple() end defp bind_walk(list, params) when is_list(list), do: Enum.map(list, &bind_walk(&1, params)) defp bind_walk(%module{} = node, params) do struct!( module, node |> Map.from_struct() |> Enum.map(fn {key, value} -> {key, bind_walk(value, params)} end) ) end defp bind_walk(%{} = map, params) do Map.new(map, fn {key, value} -> {key, bind_walk(value, params)} end) end defp bind_walk(other, _params), do: other defp param_value(params, index, _raw) when is_list(params) do validate_param_value!(Enum.at(params, index - 1)) end defp param_value(params, index, raw) when is_map(params) do bare = case raw do <> when sigil in [?:, ?@, ?$] -> name _ -> nil end atom_key = bare && safe_existing_atom(bare) value = cond do Map.has_key?(params, raw) -> Map.get(params, raw) bare != nil and Map.has_key?(params, bare) -> Map.get(params, bare) atom_key != nil and Map.has_key?(params, atom_key) -> Map.get(params, atom_key) Map.has_key?(params, index) -> Map.get(params, index) true -> nil end validate_param_value!(value) end defp safe_existing_atom(name) do String.to_existing_atom(name) rescue ArgumentError -> nil end defp validate_param_value!(value) when is_nil(value) or is_integer(value) or is_float(value) or is_binary(value), do: value defp validate_param_value!({:blob, bin} = blob) when is_binary(bin), do: blob defp validate_param_value!(other) do fail("invalid bind value: #{inspect(other)}") end @doc "Executes a single parsed statement." @spec execute(Database.t(), Parser.statement()) :: {:ok, Result.t(), Database.t()} | {:error, Error.t()} def execute(db, stmt) do if db.query_only and query_only_write_statement?(stmt) do fail("attempt to write a readonly database") end {result, db} = exec(db, stmt) {:ok, result, db} rescue e in Error -> {:error, e} end defp query_only_write_statement?(%stmt{}) when stmt in [ AlterTable, CreateIndex, CreateTable, CreateTrigger, CreateView, Delete, DropIndex, DropTable, DropView, Insert, Update ], do: true defp query_only_write_statement?(%With{query: query}), do: query_only_write_statement?(query) defp query_only_write_statement?(%Pragma{name: name, arg: arg}) do arg != nil and name in ["schema_version", "user_version", "application_id"] end defp query_only_write_statement?({:drop_trigger, _schema, _name, _if_exists}), do: true defp query_only_write_statement?({kind, _name}) when kind in [:analyze, :vacuum, :reindex], do: true defp query_only_write_statement?(_stmt), do: false defp ctas_column_names(names) do names |> Enum.with_index(1) |> Enum.reduce({[], %{}}, fn {name, index}, {acc, seen} -> base = if is_binary(name) and name != "", do: name, else: "column#{index}" {name, seen} = unique_ctas_column_name(base, seen) {[name | acc], seen} end) |> elem(0) |> Enum.reverse() end defp unique_ctas_column_name(base, seen) do key = Table.key(base) count = Map.get(seen, key, 0) name = if count == 0, do: base, else: "#{base}:#{count}" {name, Map.put(seen, key, count + 1)} end # -- CREATE TABLE ------------------------------------------------------------ defp exec(db, %Pragma{name: "database_list", arg: {:schema, schema, _arg}}) do ensure_schema_exists!(db, schema) exec(db, %Pragma{name: "database_list", arg: nil}) end defp exec(db, %Pragma{name: "database_list", arg: nil}) do {%Result{ command: :select, columns: ["seq", "name", "file"], rows: database_list_rows(db), rows_affected: 0 }, db} end defp exec(db, {:attach, filename_expr, name}) do key = attached_database_key(name) cond do key in ["main", "temp"] or attached_database?(db, key) -> fail("database #{name} is already in use") true -> filename = filename_expr |> eval(%{db: db, frames: [], outer: nil, group: nil}) |> attach_filename() attached = %{seq: next_attached_database_seq(db), name: name, file: filename} {%Result{command: :attach}, %{db | attached_databases: db.attached_databases ++ [attached]}} end end defp exec(db, {:detach, name}) do key = attached_database_key(name) cond do key in ["main", "temp"] -> fail("cannot detach database #{name}") not attached_database?(db, key) -> fail("no such database: #{name}") true -> attached = Enum.reject(db.attached_databases, fn attached -> attached_database_key(attached.name) == key end) tables = Map.reject(db.tables, fn {_table_key, table} -> table.schema != nil and attached_database_key(table.schema) == key end) views = Map.reject(db.views, fn {_view_key, view} -> view.schema != nil and attached_database_key(view.schema) == key end) triggers = Map.reject(db.triggers, fn {_trigger_key, trigger} -> trigger.schema != nil and attached_database_key(trigger.schema) == key end) {%Result{command: :detach}, db |> Map.merge(%{ attached_databases: attached, tables: tables, views: views, triggers: triggers }) |> Database.drop_schema_header(name)} end end defp exec(db, %Pragma{name: name, arg: {:schema, schema, arg}} = pragma) when name not in [ "application_id", "foreign_key_check", "foreign_key_list", "index_info", "index_list", "index_xinfo", "integrity_check", "page_count", "quick_check", "schema_version", "table_info", "table_list", "table_xinfo", "user_version" ] do ensure_schema_exists!(db, schema) exec(db, %{pragma | arg: arg}) end defp exec(db, %Pragma{name: "collation_list"}) do rows = db |> collation_list_rows() |> Enum.with_index() |> Enum.map(fn {name, seq} -> [seq, name] end) {%Result{ command: :select, columns: ["seq", "name"], rows: rows, rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "compile_options"}) do rows = Enum.map(@compile_options, &[&1]) {%Result{ command: :select, columns: ["compile_options"], rows: rows, rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "function_list"}) do {%Result{ command: :select, columns: ["name", "builtin", "type", "enc", "narg", "flags"], rows: function_list_rows(db), rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "module_list"}) do {%Result{ command: :select, columns: ["name"], rows: [["json_each"], ["json_tree"]], rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "pragma_list"}) do rows = Enum.map(@supported_pragmas, &[&1]) {%Result{ command: :select, columns: ["name"], rows: rows, rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "encoding", arg: nil}) do {%Result{ command: :select, columns: ["encoding"], rows: [["UTF-8"]], rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "encoding", arg: _arg}) do {%Result{command: :pragma}, db} end defp exec(db, %Pragma{name: "foreign_keys", arg: nil}) do {%Result{ command: :select, columns: ["foreign_keys"], rows: [[bool_int(db.foreign_keys)]], rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "foreign_keys", arg: arg}) do # As in SQLite, foreign-key enforcement may only be toggled outside a # transaction; within one the pragma is a no-op. db = if db.txn_stack == [] do %{db | foreign_keys: pragma_enabled?(arg)} else db end {%Result{command: :pragma}, db} end defp exec(db, %Pragma{name: "defer_foreign_keys", arg: nil}) do {%Result{ command: :select, columns: ["defer_foreign_keys"], rows: [[bool_int(db.defer_foreign_keys)]], rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "defer_foreign_keys", arg: arg}) do {%Result{command: :pragma}, %{db | defer_foreign_keys: pragma_enabled?(arg)}} end defp exec(db, %Pragma{name: "recursive_triggers", arg: nil}) do {%Result{ command: :select, columns: ["recursive_triggers"], rows: [[bool_int(db.recursive_triggers)]], rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "recursive_triggers", arg: arg}) do {%Result{command: :pragma}, %{db | recursive_triggers: pragma_enabled?(arg)}} end defp exec(db, %Pragma{name: "ignore_check_constraints", arg: nil}) do {%Result{ command: :select, columns: ["ignore_check_constraints"], rows: [[bool_int(db.ignore_check_constraints)]], rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "ignore_check_constraints", arg: arg}) do {%Result{command: :pragma}, %{db | ignore_check_constraints: pragma_enabled?(arg)}} end defp exec(db, %Pragma{name: "count_changes", arg: nil}) do {%Result{ command: :select, columns: ["count_changes"], rows: [[bool_int(db.count_changes)]], rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "count_changes", arg: arg}) do {%Result{command: :pragma}, %{db | count_changes: pragma_enabled?(arg)}} end defp exec(db, %Pragma{name: "read_uncommitted", arg: nil}) do {%Result{ command: :select, columns: ["read_uncommitted"], rows: [[bool_int(db.read_uncommitted)]], rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "read_uncommitted", arg: arg}) do {%Result{command: :pragma}, %{db | read_uncommitted: pragma_enabled?(arg)}} end defp exec(db, %Pragma{name: "case_sensitive_like", arg: nil}) do {%Result{ command: :select, columns: ["case_sensitive_like"], rows: [[bool_int(db.case_sensitive_like)]], rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "case_sensitive_like", arg: arg}) do {%Result{command: :pragma}, %{db | case_sensitive_like: pragma_enabled?(arg)}} end defp exec(db, %Pragma{name: "short_column_names", arg: nil}) do {%Result{ command: :select, columns: ["short_column_names"], rows: [[bool_int(db.short_column_names)]], rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "short_column_names", arg: arg}) do {%Result{command: :pragma}, %{db | short_column_names: pragma_enabled?(arg)}} end defp exec(db, %Pragma{name: "full_column_names", arg: nil}) do {%Result{ command: :select, columns: ["full_column_names"], rows: [[bool_int(db.full_column_names)]], rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "full_column_names", arg: arg}) do {%Result{command: :pragma}, %{db | full_column_names: pragma_enabled?(arg)}} end defp exec(db, %Pragma{name: "reverse_unordered_selects", arg: nil}) do {%Result{ command: :select, columns: ["reverse_unordered_selects"], rows: [[bool_int(db.reverse_unordered_selects)]], rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "reverse_unordered_selects", arg: arg}) do {%Result{command: :pragma}, %{db | reverse_unordered_selects: pragma_enabled?(arg)}} end defp exec(db, %Pragma{name: "query_only", arg: nil}) do {%Result{ command: :select, columns: ["query_only"], rows: [[bool_int(db.query_only)]], rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "query_only", arg: arg}) do {%Result{command: :pragma}, %{db | query_only: pragma_enabled?(arg)}} end defp exec(db, %Pragma{name: "empty_result_callbacks", arg: nil}) do pragma_bool_result("empty_result_callbacks", db.empty_result_callbacks, db) end defp exec(db, %Pragma{name: "empty_result_callbacks", arg: arg}) do {%Result{command: :pragma}, %{db | empty_result_callbacks: pragma_enabled?(arg)}} end defp exec(db, %Pragma{name: "automatic_index", arg: nil}) do {%Result{ command: :select, columns: ["automatic_index"], rows: [[bool_int(db.automatic_index)]], rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "automatic_index", arg: arg}) do {%Result{command: :pragma}, %{db | automatic_index: pragma_enabled?(arg)}} end defp exec(db, %Pragma{name: "auto_vacuum", arg: nil}) do pragma_integer_result("auto_vacuum", db.auto_vacuum, db) end defp exec(db, %Pragma{name: "auto_vacuum", arg: {:schema, schema, arg}}) do ensure_schema_exists!(db, schema) exec(db, %Pragma{name: "auto_vacuum", arg: arg}) end defp exec(db, %Pragma{name: "auto_vacuum", arg: arg}) do db = if db.page_size_locked do db else %{db | auto_vacuum: pragma_auto_vacuum(arg, db.auto_vacuum)} end {%Result{command: :pragma}, db} end defp exec(db, %Pragma{name: "incremental_vacuum"}) do {%Result{command: :pragma}, db} end defp exec(db, %Pragma{name: "shrink_memory"}) do {%Result{command: :pragma}, db} end defp exec(db, %Pragma{name: name, arg: {:schema, schema, arg}}) when name in [ "cache_spill", "data_version", "default_cache_size", "mmap_size", "secure_delete", "wal_autocheckpoint" ] do ensure_schema_exists!(db, schema) exec(db, %Pragma{name: name, arg: arg}) end defp exec(db, %Pragma{name: "mmap_size"}) do {%Result{command: :pragma}, db} end defp exec(db, %Pragma{name: "analysis_limit", arg: nil}) do pragma_integer_result("analysis_limit", db.analysis_limit, db) end defp exec(db, %Pragma{name: "analysis_limit", arg: arg}) do value = pragma_analysis_limit(arg, db.analysis_limit) db = %{db | analysis_limit: value} pragma_integer_result("analysis_limit", value, db) end defp exec(db, %Pragma{name: "cell_size_check", arg: nil}) do pragma_bool_result("cell_size_check", db.cell_size_check, db) end defp exec(db, %Pragma{name: "cell_size_check", arg: arg}) do {%Result{command: :pragma}, %{db | cell_size_check: pragma_enabled?(arg)}} end defp exec(db, %Pragma{name: "checkpoint_fullfsync", arg: nil}) do pragma_bool_result("checkpoint_fullfsync", db.checkpoint_fullfsync, db) end defp exec(db, %Pragma{name: "checkpoint_fullfsync", arg: arg}) do {%Result{command: :pragma}, %{db | checkpoint_fullfsync: pragma_enabled?(arg)}} end defp exec(db, %Pragma{name: "fullfsync", arg: nil}) do pragma_bool_result("fullfsync", db.fullfsync, db) end defp exec(db, %Pragma{name: "fullfsync", arg: arg}) do {%Result{command: :pragma}, %{db | fullfsync: pragma_enabled?(arg)}} end defp exec(db, %Pragma{name: "trusted_schema", arg: nil}) do pragma_bool_result("trusted_schema", db.trusted_schema, db) end defp exec(db, %Pragma{name: "trusted_schema", arg: arg}) do {%Result{command: :pragma}, %{db | trusted_schema: pragma_enabled?(arg)}} end defp exec(db, %Pragma{name: "busy_timeout", arg: nil}) do pragma_integer_result("busy_timeout", db.busy_timeout, db) end defp exec(db, %Pragma{name: "busy_timeout", arg: arg}) do value = non_negative_pragma_integer(arg) db = %{db | busy_timeout: value} pragma_integer_result("busy_timeout", value, db) end defp exec(db, %Pragma{name: "page_count", arg: {:schema, schema, nil}}) do ensure_schema_exists!(db, schema) pragma_integer_result("page_count", pragma_page_count(db, schema), db) end defp exec(db, %Pragma{name: "page_count", arg: {:schema, schema, _arg}}) do ensure_schema_exists!(db, schema) {%Result{command: :pragma}, db} end defp exec(db, %Pragma{name: "page_count", arg: nil}) do pragma_integer_result("page_count", pragma_page_count(db, nil), db) end defp exec(db, %Pragma{name: "page_count", arg: _arg}) do {%Result{command: :pragma}, db} end defp exec(db, %Pragma{name: "page_size", arg: {:schema, schema, arg}}) do ensure_schema_exists!(db, schema) exec(db, %Pragma{name: "page_size", arg: arg}) end defp exec(db, %Pragma{name: "page_size", arg: nil}) do {%Result{ command: :select, columns: ["page_size"], rows: [[db.page_size]], rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "page_size", arg: arg}) do db = if db.page_size_locked do db else case pragma_page_size(arg) do nil -> db page_size -> %{db | page_size: page_size} end end {%Result{command: :pragma}, db} end defp exec(db, %Pragma{name: "cache_size", arg: {:schema, schema, arg}}) do ensure_schema_exists!(db, schema) exec(db, %Pragma{name: "cache_size", arg: arg}) end defp exec(db, %Pragma{name: "cache_size", arg: nil}) do {%Result{ command: :select, columns: ["cache_size"], rows: [[db.cache_size]], rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "cache_size", arg: arg}) do {%Result{command: :pragma}, %{db | cache_size: pragma_cache_size(arg)}} end defp exec(db, %Pragma{name: "default_cache_size", arg: nil}) do pragma_integer_result("default_cache_size", db.default_cache_size, db) end defp exec(db, %Pragma{name: "default_cache_size", arg: arg}) do value = pragma_default_cache_size(arg) db = %{db | default_cache_size: value} pragma_integer_result("default_cache_size", value, db) end defp exec(db, %Pragma{name: "cache_spill", arg: nil}) do pragma_integer_result("cache_spill", db.cache_spill, db) end defp exec(db, %Pragma{name: "cache_spill", arg: arg}) do value = pragma_cache_spill(arg, db.cache_spill) db = %{db | cache_spill: value} pragma_integer_result("cache_spill", value, db) end defp exec(db, %Pragma{name: "max_page_count", arg: nil}) do pragma_integer_result("max_page_count", db.max_page_count, db) end defp exec(db, %Pragma{name: "max_page_count", arg: {:schema, schema, arg}}) do ensure_schema_exists!(db, schema) exec(db, %Pragma{name: "max_page_count", arg: arg}) end defp exec(db, %Pragma{name: "max_page_count", arg: arg}) do value = pragma_max_page_count(arg, db.max_page_count) db = %{db | max_page_count: value} pragma_integer_result("max_page_count", value, db) end defp exec(db, %Pragma{name: "journal_mode", arg: nil}) do journal_mode_result(db.journal_mode, db) end defp exec(db, %Pragma{name: "journal_mode", arg: {:schema, schema, arg}}) do ensure_schema_exists!(db, schema) exec(db, %Pragma{name: "journal_mode", arg: arg}) end defp exec(db, %Pragma{name: "journal_mode", arg: arg}) do mode = pragma_journal_mode(arg, db.journal_mode) db = %{db | journal_mode: mode} journal_mode_result(mode, db) end defp exec(db, %Pragma{name: "journal_size_limit", arg: nil}) do pragma_integer_result("journal_size_limit", db.journal_size_limit, db) end defp exec(db, %Pragma{name: "journal_size_limit", arg: {:schema, schema, arg}}) do ensure_schema_exists!(db, schema) exec(db, %Pragma{name: "journal_size_limit", arg: arg}) end defp exec(db, %Pragma{name: "journal_size_limit", arg: arg}) do value = pragma_journal_size_limit(arg) db = %{db | journal_size_limit: value} pragma_integer_result("journal_size_limit", value, db) end defp exec(db, %Pragma{name: "locking_mode", arg: nil}) do locking_mode_result(db.locking_mode, db) end defp exec(db, %Pragma{name: "locking_mode", arg: {:schema, schema, arg}}) do ensure_schema_exists!(db, schema) exec(db, %Pragma{name: "locking_mode", arg: arg}) end defp exec(db, %Pragma{name: "locking_mode", arg: arg}) do mode = pragma_locking_mode(arg, db.locking_mode) db = %{db | locking_mode: mode} locking_mode_result(mode, db) end defp exec(db, %Pragma{name: "synchronous", arg: {:schema, schema, arg}}) do ensure_schema_exists!(db, schema) exec(db, %Pragma{name: "synchronous", arg: arg}) end defp exec(db, %Pragma{name: "synchronous", arg: nil}) do {%Result{ command: :select, columns: ["synchronous"], rows: [[db.synchronous]], rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "synchronous", arg: arg}) do {%Result{command: :pragma}, %{db | synchronous: pragma_synchronous(arg)}} end defp exec(db, %Pragma{name: "temp_store", arg: nil}) do {%Result{ command: :select, columns: ["temp_store"], rows: [[db.temp_store]], rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "temp_store", arg: {:schema, schema, arg}}) do ensure_schema_exists!(db, schema) exec(db, %Pragma{name: "temp_store", arg: arg}) end defp exec(db, %Pragma{name: "temp_store", arg: arg}) do {%Result{command: :pragma}, %{db | temp_store: pragma_temp_store(arg)}} end defp exec(db, %Pragma{name: "soft_heap_limit", arg: nil}) do pragma_integer_result("soft_heap_limit", db.soft_heap_limit, db) end defp exec(db, %Pragma{name: "soft_heap_limit", arg: arg}) do value = non_negative_pragma_integer(arg) db = %{db | soft_heap_limit: value} pragma_integer_result("soft_heap_limit", value, db) end defp exec(db, %Pragma{name: "hard_heap_limit"}) do pragma_integer_result("hard_heap_limit", 0, db) end defp exec(db, %Pragma{name: "secure_delete", arg: nil}) do pragma_integer_result("secure_delete", db.secure_delete, db) end defp exec(db, %Pragma{name: "secure_delete", arg: arg}) do value = pragma_secure_delete(arg) db = %{db | secure_delete: value} pragma_integer_result("secure_delete", value, db) end defp exec(db, %Pragma{name: "threads", arg: nil}) do pragma_integer_result("threads", db.threads, db) end defp exec(db, %Pragma{name: "threads", arg: arg}) do value = pragma_threads(arg, db.threads) db = %{db | threads: value} pragma_integer_result("threads", value, db) end defp exec(db, %Pragma{name: "wal_autocheckpoint", arg: nil}) do pragma_integer_result("wal_autocheckpoint", db.wal_autocheckpoint, db) end defp exec(db, %Pragma{name: "wal_autocheckpoint", arg: arg}) do value = non_negative_pragma_integer(arg) db = %{db | wal_autocheckpoint: value} pragma_integer_result("wal_autocheckpoint", value, db) end defp exec(db, %Pragma{name: "wal_checkpoint"}) do {%Result{ command: :select, columns: ["busy", "log", "checkpointed"], rows: [[0, -1, -1]], rows_affected: 0, affinities: [:integer, :integer, :integer] }, db} end defp exec(db, %Pragma{name: "optimize"}) do {%Result{command: :pragma}, db} end defp exec(db, %Pragma{name: "stats"}) do {%Result{command: :pragma}, db} end defp exec(db, %Pragma{name: name, arg: {:schema, schema, nil}}) when name in ["schema_version", "user_version", "application_id"] do ensure_schema_exists!(db, schema) field = pragma_header_field(name) pragma_integer_result(name, Database.schema_header_value(db, schema, field), db) end defp exec(db, %Pragma{name: name, arg: {:schema, schema, arg}}) when name in ["schema_version", "user_version", "application_id"] do ensure_schema_exists!(db, schema) field = pragma_header_field(name) value = pragma_header_value(arg) {%Result{command: :pragma}, Database.put_schema_header_value(db, schema, field, value)} end defp exec(db, %Pragma{name: "schema_version", arg: nil}) do {%Result{ command: :select, columns: ["schema_version"], rows: [[db.schema_version]], rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "schema_version", arg: arg}) do {%Result{command: :pragma}, %{db | schema_version: pragma_header_value(arg)}} end defp exec(db, %Pragma{name: "user_version", arg: nil}) do {%Result{ command: :select, columns: ["user_version"], rows: [[db.user_version]], rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "user_version", arg: arg}) do {%Result{command: :pragma}, %{db | user_version: pragma_header_value(arg)}} end defp exec(db, %Pragma{name: "application_id", arg: nil}) do {%Result{ command: :select, columns: ["application_id"], rows: [[db.application_id]], rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "application_id", arg: arg}) do {%Result{command: :pragma}, %{db | application_id: pragma_header_value(arg)}} end defp exec(db, %Pragma{name: "data_version"}) do pragma_integer_result("data_version", 2, db) end defp exec(db, %Pragma{name: "freelist_count", arg: {:schema, schema, _arg}}) do ensure_schema_exists!(db, schema) exec(db, %Pragma{name: "freelist_count", arg: nil}) end defp exec(db, %Pragma{name: "freelist_count"}) do {%Result{ command: :select, columns: ["freelist_count"], rows: [[0]], rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "table_list", arg: arg}) do rows = db |> table_list_rows() |> filter_pragma_table_list(db, arg) {%Result{ command: :select, columns: ["schema", "name", "type", "ncol", "wr", "strict"], rows: rows, rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "integrity_check", arg: arg}) do rows = integrity_check_rows(db, arg) {%Result{ command: :select, columns: ["integrity_check"], rows: if(rows == [], do: [["ok"]], else: rows), rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "quick_check", arg: arg}) do rows = integrity_check_rows(db, arg) {%Result{ command: :select, columns: ["quick_check"], rows: if(rows == [], do: [["ok"]], else: rows), rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "table_info", arg: table_name}) do rows = case pragma_fetch_table(db, table_name) do {:ok, table} -> table_info_rows(table, false) :error -> [] end {%Result{ command: :select, columns: ["cid", "name", "type", "notnull", "dflt_value", "pk"], rows: rows, rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "table_xinfo", arg: table_name}) do rows = case pragma_fetch_table(db, table_name) do {:ok, table} -> table_info_rows(table, true) :error -> [] end {%Result{ command: :select, columns: ["cid", "name", "type", "notnull", "dflt_value", "pk", "hidden"], rows: rows, rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "foreign_key_list", arg: table_name}) do rows = case pragma_fetch_table(db, table_name) do {:ok, table} -> foreign_key_list_rows(table) :error -> [] end {%Result{ command: :select, columns: ["id", "seq", "table", "from", "to", "on_update", "on_delete", "match"], rows: rows, rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "foreign_key_check", arg: table_name}) do rows = foreign_key_check_rows(db, table_name) {%Result{ command: :select, columns: ["table", "rowid", "parent", "fkid"], rows: rows, rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "index_list", arg: table_name}) do rows = case pragma_fetch_table(db, table_name) do {:ok, table} -> index_list_rows(table) :error -> [] end {%Result{ command: :select, columns: ["seq", "name", "unique", "origin", "partial"], rows: rows, rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "index_info", arg: index_name}) do rows = case pragma_find_index_owner(db, index_name) do {table, index} -> index_info_rows(table, index) nil -> [] end {%Result{ command: :select, columns: ["seqno", "cid", "name"], rows: rows, rows_affected: 0 }, db} end defp exec(db, %Pragma{name: "index_xinfo", arg: index_name}) do rows = case pragma_find_index_owner(db, index_name) do {table, index} -> index_xinfo_rows(table, index) nil -> [] end {%Result{ command: :select, columns: ["seqno", "cid", "name", "desc", "coll", "key"], rows: rows, rows_affected: 0 }, db} end defp exec(db, %CreateTable{query: query} = stmt) when query != nil do ensure_schema_exists!(db, stmt.schema) key = Database.table_storage_key(stmt.schema, stmt.name) if stmt.if_not_exists and (Map.has_key?(db.tables, key) or Map.has_key?(db.views, key)) do {%Result{command: :create_table}, db} else result = query_result(db, query, nil) column_names = ctas_column_names(result.columns) columns = Enum.map(column_names, &%ColumnDef{name: &1, affinity: :blob}) table = result.rows |> Enum.reduce(Table.new(stmt.name, columns, schema: stmt.schema), fn row, table -> values = column_names |> Enum.map(&Table.key/1) |> Enum.zip(row) |> Map.new() case Table.insert(table, values) do {:ok, table, _rowid} -> table {:error, message} -> fail(message) :ignore -> table end end) case Database.create_table(db, table) do {:ok, db} -> {%Result{command: :create_table}, db} {:error, "there is already an index named " <> _ = message} -> fail(message) {:error, _message} when stmt.if_not_exists -> {%Result{command: :create_table}, db} {:error, message} -> fail(message) end end end defp exec(db, %CreateTable{} = stmt) do ensure_schema_exists!(db, stmt.schema) ensure_unique_names(stmt) ensure_valid_autoincrement!(stmt) ensure_without_rowid_primary_key!(stmt) ensure_valid_strict_types!(stmt) ensure_valid_check_constraints!(stmt.name, stmt.columns, stmt.constraints) {composite_keys, composite_uniques} = partition_table_constraints(stmt.constraints) table_checks = table_check_constraints(stmt.constraints) foreign_keys = table_foreign_keys(stmt.constraints) # Collect column-level CHECK constraints too column_checks = for col <- stmt.columns, col.check != nil do {col.check_name, col.check} end table = Table.new(stmt.name, stmt.columns, schema: stmt.schema, composite_keys: composite_keys, composite_uniques: composite_uniques, foreign_keys: foreign_keys, checks: column_checks ++ table_checks, without_rowid: stmt.without_rowid, strict: stmt.strict ) |> put_autoindexes(stmt.constraints) case Database.create_table(db, table) do {:ok, db} -> {%Result{command: :create_table}, db} {:error, "there is already an index named " <> _ = message} -> fail(message) {:error, _message} when stmt.if_not_exists -> {%Result{command: :create_table}, db} {:error, message} -> fail(message) end end defp exec(db, %DropTable{} = stmt) do ensure_schema_exists!(db, stmt.schema) schema = drop_object_schema(db, stmt.schema, stmt.name) db = drop_table_fk_cleanup(db, schema, stmt.name) case Database.drop_table(db, schema, stmt.name) do {:ok, db} -> {%Result{command: :drop_table}, drop_triggers_on(db, schema, stmt.name)} {:error, _} when stmt.if_exists -> {%Result{command: :drop_table}, db} {:error, message} -> fail(message) end end # -- CREATE INDEX ------------------------------------------------------------ defp exec(db, %CreateIndex{} = stmt) do ensure_schema_exists!(db, stmt.schema) table = fetch_table!(db, stmt.schema, stmt.table) index_schema = table.schema index_table_key = Database.table_storage_key(index_schema, stmt.name) if internal_sqlite_object_name?(stmt.name) do fail("object name reserved for internal use: #{stmt.name}") end # Index names are scoped to their schema and must not collide with other # indexes in that schema. if Database.index_exists?(db, index_schema, stmt.name) do if stmt.if_not_exists do {%Result{command: :create_index}, db} else fail("index #{stmt.name} already exists") end else # Index name must not collide with an existing table name if Map.has_key?(db.tables, index_table_key) or Map.has_key?(db.views, index_table_key) do fail("there is already a table named #{stmt.name}") else # Validate columns and build the member list; an index with any # expression member is enforced executor-side (it needs eval). members = Enum.map(stmt.columns, fn col -> case col do %{name: name} when name != nil -> unless Table.column(table, name) do fail("no such column: #{name}") end {:column, Table.key(name)} %{expr: expr} -> validate_index_expression!(table, expr) {:expr, expr} end end) collations = stmt.columns |> Enum.zip(members) |> Enum.map(fn {col, member} -> index_member_collation(table, col, member) end) directions = Enum.map(stmt.columns, & &1.direction) index = if Enum.any?(members, &match?({:expr, _}, &1)) do %{ name: stmt.name, columns: [], members: members, collations: collations, directions: directions, unique: stmt.unique, where: stmt.where } else %{ name: stmt.name, columns: Enum.map(members, fn {:column, key} -> key end), members: members, collations: collations, directions: directions, unique: stmt.unique, where: stmt.where } end # For UNIQUE indexes, check existing data for duplicates if stmt.unique do check_unique_index_data!(db, table, index) end updated_table = %{table | indexes: table.indexes ++ [index]} {%Result{command: :create_index}, db |> put_table(updated_table) |> Database.schema_changed(table.schema)} end end end # -- DROP INDEX ------------------------------------------------------------ defp exec(db, %DropIndex{} = stmt) do ensure_schema_exists!(db, stmt.schema) owner = if stmt.schema do drop_index_owner(db, stmt.schema, stmt.name) else drop_index_owner(db, :any, stmt.name) end case owner do nil -> if stmt.if_exists do {%Result{command: :drop_index}, db} else fail("no such index: #{stmt.name}") end {_table, %{autoindex: true}} -> fail("index associated with UNIQUE or PRIMARY KEY constraint cannot be dropped") {table, _index} -> index_key = Table.key(stmt.name) updated_table = %{ table | indexes: Enum.reject(table.indexes, &(Table.key(&1.name) == index_key)) } {%Result{command: :drop_index}, db |> put_table(updated_table) |> Database.schema_changed(table.schema)} end end # -- CREATE VIEW / DROP VIEW ------------------------------------------------ defp exec(db, %CreateView{} = stmt) do ensure_schema_exists!(db, stmt.schema) view = %{ name: stmt.name, schema: stmt.schema, columns: stmt.columns, query: qualify_view_query(stmt.query, stmt.schema) } case Database.create_view(db, view) do {:ok, db} -> {%Result{command: :create_view}, db} {:error, "there is already an index named " <> _ = message} -> fail(message) {:error, _message} when stmt.if_not_exists -> {%Result{command: :create_view}, db} {:error, message} -> fail(message) end end defp exec(db, %DropView{} = stmt) do ensure_schema_exists!(db, stmt.schema) schema = drop_object_schema(db, stmt.schema, stmt.name) case Database.drop_view(db, schema, stmt.name) do {:ok, db} -> {%Result{command: :drop_view}, drop_triggers_on(db, schema, stmt.name)} {:error, _} when stmt.if_exists -> {%Result{command: :drop_view}, db} {:error, message} -> fail(message) end end # -- CREATE TRIGGER / DROP TRIGGER --------------------------------------------- defp exec(db, %CreateTrigger{} = stmt) do ensure_trigger_schema_exists!(db, stmt.schema) target_schema = trigger_target_schema!(db, stmt) key = Database.table_storage_key(stmt.schema, stmt.name) target_key = Database.table_storage_key(target_schema, stmt.table) view? = match?({:ok, _}, Database.fetch_view(db, target_schema, stmt.table)) table? = Map.has_key?(db.tables, target_key) target_label = trigger_target_label(stmt, target_schema) cond do not view? and not table? -> fail("no such table: #{target_label}") Map.has_key?(db.triggers, key) and stmt.if_not_exists -> {%Result{command: :create_trigger}, db} Map.has_key?(db.triggers, key) -> fail("trigger #{stmt.name} already exists") stmt.timing == :instead_of and not view? -> fail("cannot create INSTEAD OF trigger on table: #{target_label}") stmt.timing != :instead_of and view? -> fail( "cannot create #{String.upcase(Atom.to_string(stmt.timing))} " <> "trigger on view: #{target_label}" ) true -> validate_trigger_definition!(stmt) trigger = %{ key: key, name: stmt.name, schema: stmt.schema, table_schema: target_schema, table_key: target_key, table_name: stmt.table, timing: stmt.timing, event: stmt.event, update_columns: stmt.update_columns, when: stmt.when, body: stmt.body, seq: map_size(db.triggers) } {%Result{command: :create_trigger}, %{db | triggers: Map.put(db.triggers, key, trigger)} |> Database.schema_changed(stmt.schema)} end end defp exec(db, {:drop_trigger, schema, name, if_exists}) do ensure_trigger_schema_exists!(db, schema) key = drop_trigger_key(db, schema, name) cond do key != nil -> trigger = Map.fetch!(db.triggers, key) {%Result{command: :drop_trigger}, %{db | triggers: Map.delete(db.triggers, key)} |> Database.schema_changed(trigger.schema)} if_exists -> {%Result{command: :drop_trigger}, db} true -> fail("no such trigger: #{name}") end end # -- WITH (CTEs) ------------------------------------------------------------ defp exec(db, %With{} = stmt) do # Extract outer LIMIT to use as a cap for recursive CTE expansion. outer_limit = extract_query_limit(stmt.query, db) db_with_ctes = resolve_ctes(db, stmt.ctes, stmt.recursive, outer_limit) case stmt.query do %Select{} = q -> {select_result(db_with_ctes, q, nil), db} %Compound{} = q -> {compound_result(db_with_ctes, q, nil), db} %Values{} = q -> {values_result(db_with_ctes, q, nil), db} %With{} = q -> # Nested WITH: chain CTEs {result, _} = exec(db_with_ctes, q) {result, db} %Insert{} = q -> {result, updated_db} = exec(db_with_ctes, q) # Write DML changes back to the real db but without CTEs {result, %{updated_db | ctes: db.ctes}} %Update{} = q -> {result, updated_db} = exec(db_with_ctes, q) {result, %{updated_db | ctes: db.ctes}} %Delete{} = q -> {result, updated_db} = exec(db_with_ctes, q) {result, %{updated_db | ctes: db.ctes}} end end # -- INSERT -------------------------------------------------------------------- defp exec(db, %Insert{} = stmt) do cond do main_schema?(stmt.schema) and Table.key(stmt.table) == "sqlite_sequence" -> exec_sqlite_sequence_insert(db, stmt) match?({:ok, _}, dml_view(db, stmt.schema, stmt.table)) -> {:ok, view} = dml_view(db, stmt.schema, stmt.table) exec_view_dml(db, view, stmt, :insert) true -> with_fk_statement_check(db, stmt, fn -> exec_insert(db, stmt) end) end end # -- UPDATE -------------------------------------------------------------------- defp exec(db, %Update{} = stmt) do cond do main_schema?(stmt.schema) and Table.key(stmt.table) == "sqlite_sequence" -> exec_sqlite_sequence_update(db, stmt) match?({:ok, _}, dml_view(db, stmt.schema, stmt.table)) -> {:ok, view} = dml_view(db, stmt.schema, stmt.table) exec_view_dml(db, view, stmt, :update) true -> with_fk_statement_check(db, stmt, fn -> exec_update(db, stmt) end) end end # -- DELETE -------------------------------------------------------------------- defp exec(db, %Delete{} = stmt) do cond do main_schema?(stmt.schema) and Table.key(stmt.table) == "sqlite_sequence" -> exec_sqlite_sequence_delete(db, stmt) match?({:ok, _}, dml_view(db, stmt.schema, stmt.table)) -> {:ok, view} = dml_view(db, stmt.schema, stmt.table) exec_view_dml(db, view, stmt, :delete) true -> with_fk_statement_check(db, stmt, fn -> exec_delete(db, stmt) end) end end # -- ALTER TABLE ----------------------------------------------------------------- defp exec(db, %AlterTable{} = stmt) do table = fetch_table!(db, stmt.schema, stmt.name) case stmt.op do {:rename_table, new_name} -> new_key = Database.table_storage_key(table.schema, new_name) if Map.has_key?(db.tables, new_key) or Map.has_key?(db.views, new_key) or Database.index_exists?(db, table.schema, new_name) do fail("there is already another table or index with this name: #{new_name}") end renamed = %{table | name: new_name} |> rename_autoindexes() db = %{ db | tables: db.tables |> Map.delete(Database.table_storage_key(table.schema, table.name)) |> Map.put(new_key, renamed) } {%Result{command: :alter_table}, Database.schema_changed(db, table.schema)} {:rename_column, old_name, new_name} -> old_key = Table.key(old_name) col = Table.column(table, old_name) unless col do fail(~s(no such column: "#{old_name}")) end new_key = Table.key(new_name) if Enum.any?(table.columns, &(Table.key(&1.name) == new_key)) do fail("error in table #{table.name} after rename: duplicate column name: #{new_name}") end new_columns = Enum.map(table.columns, fn c -> if Table.key(c.name) == old_key, do: %{c | name: new_name}, else: c end) # Update rowid_alias if needed new_rowid_alias = if table.rowid_alias == old_key, do: new_key, else: table.rowid_alias # Update composite_keys, composite_uniques, and index column lists new_composite_keys = rename_in_composites(table.composite_keys, old_key, new_key) new_composite_uniques = rename_in_composites(table.composite_uniques, old_key, new_key) new_indexes = Enum.map(table.indexes, fn index -> %{index | columns: Enum.map(index.columns, &if(&1 == old_key, do: new_key, else: &1))} end) new_autoindexes = Enum.map(table.autoindexes, fn index -> rename_index_column(index, old_key, new_key) end) # Update row maps: rename the column key in all rows new_rows = Map.new(Table.scan(table), fn {rowid, row} -> {value, rest} = Map.pop(row, old_key, nil) {rowid, Map.put(rest, new_key, value)} end) new_table = Table.narrow_all_rows(%{ table | columns: new_columns, rowid_alias: new_rowid_alias, composite_keys: new_composite_keys, composite_uniques: new_composite_uniques, indexes: new_indexes, autoindexes: new_autoindexes, rows: new_rows, frame_columns: nil, column_index: nil }) {%Result{command: :alter_table}, db |> put_table(new_table) |> Database.schema_changed()} {:add_column, col_def} -> alter_add_column(db, table, col_def) {:drop_column, col_name} -> alter_drop_column(db, table, col_name) end end # -- operational no-ops ------------------------------------------------------------ defp exec(db, {:analyze, name}) do validate_analyze_target!(db, name) {%Result{command: :analyze}, db} end defp exec(db, {:vacuum, _name}), do: {%Result{command: :vacuum}, db} defp exec(db, {:reindex, name}) do validate_reindex_target!(db, name) {%Result{command: :reindex}, db} end # -- transactions ---------------------------------------------------------------- # # On an immutable database value a transaction is just a snapshot of the # tables: BEGIN/SAVEPOINT push one, ROLLBACK restores one, COMMIT/RELEASE # discard entries. SAVEPOINT outside a transaction starts an implicit one, # as in SQLite. defp exec(db, {:begin}) do if db.txn_stack != [], do: fail("cannot start a transaction within a transaction") {%Result{command: :begin}, %{db | txn_stack: [{:begin, Database.schema_snapshot(db)}]}} end defp exec(db, {:commit}) do if db.txn_stack == [], do: fail("cannot commit - no transaction is active") check_deferred_foreign_keys!(db) {%Result{command: :commit}, %{db | txn_stack: [], defer_foreign_keys: false}} end defp exec(db, {:rollback}) do case List.last(db.txn_stack) do nil -> fail("cannot rollback - no transaction is active") {_kind, snapshot} -> db = %{db | txn_stack: [], defer_foreign_keys: false} {%Result{command: :rollback}, Database.restore_schema(db, snapshot)} end end defp exec(db, {:savepoint, name}) do stack = [{{:savepoint, Table.key(name)}, Database.schema_snapshot(db)} | db.txn_stack] {%Result{command: :savepoint}, %{db | txn_stack: stack}} end defp exec(db, {:release, name}) do case savepoint_index(db.txn_stack, name) do nil -> fail("no such savepoint: #{name}") index -> remaining = Enum.drop(db.txn_stack, index + 1) # Releasing the outermost savepoint commits the implicit transaction, # which is when deferred foreign keys are checked. db = if remaining == [] do check_deferred_foreign_keys!(db) %{db | txn_stack: remaining, defer_foreign_keys: false} else %{db | txn_stack: remaining} end {%Result{command: :release}, db} end end defp exec(db, {:rollback_to, name}) do case savepoint_index(db.txn_stack, name) do nil -> fail("no such savepoint: #{name}") index -> {_kind, snapshot} = Enum.at(db.txn_stack, index) {%Result{command: :rollback}, Database.restore_schema(%{db | txn_stack: Enum.drop(db.txn_stack, index)}, snapshot)} end end # -- SELECT -------------------------------------------------------------------- defp exec(db, %Select{} = stmt), do: {select_result(db, stmt, nil), db} defp exec(db, %Compound{} = stmt), do: {compound_result(db, stmt, nil), db} defp exec(db, %Values{} = stmt), do: {values_result(db, stmt, nil), db} defp exec(db, {:explain, stmt}) do case explain_bytecode_rows(db, stmt) do {:ok, rows} -> {%Result{ command: :select, columns: ["addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment"], rows: rows, rows_affected: 0, affinities: [:integer, :text, :integer, :integer, :integer, :text, :integer, :text] }, db} :error -> fail("unsupported EXPLAIN statement") end end defp exec(db, {:explain_query_plan, stmt}) do {%Result{ command: :select, columns: ["id", "parent", "notused", "detail"], rows: explain_query_plan_rows(db, stmt), rows_affected: 0, affinities: [:integer, :integer, :integer, :text] }, db} end defp explain_bytecode_rows(_db, %Select{ columns: columns, from: nil, where: nil, group_by: [], having: nil, windows: %{}, order_by: [], limit: limit, offset: offset, distinct: false }) do with {:ok, limit} <- explain_limit(limit, offset) do columns |> Enum.map(fn {expr, _alias} -> explain_literal_expr(expr) _other -> :error end) |> explain_literal_program(limit) end end defp explain_bytecode_rows( db, %Select{ columns: columns, from: {:table, name, alias_name}, where: where, group_by: [], having: nil, windows: %{}, order_by: [], limit: limit, offset: offset, distinct: false } ) do with table_key when not is_nil(table_key) <- relation_unqualified_table_key(db, name), %Table{} = table <- plain_table(db, table_key), {:ok, projection} <- explain_table_projection(table, name, alias_name, columns), {:ok, filter} <- explain_table_filter(table, name, alias_name, where), {:ok, limit} <- explain_limit(limit, offset) do {:ok, explain_table_scan_program(db, table, projection, filter, limit)} else _other -> :error end end defp explain_bytecode_rows(_db, %Values{rows: [row], order_by: [], limit: limit, offset: offset}) do with {:ok, limit} <- explain_limit(limit, offset) do row |> Enum.map(&explain_literal_expr/1) |> explain_literal_program(limit) end end defp explain_bytecode_rows(_db, %Values{ rows: [_, _ | _] = rows, order_by: [], limit: limit, offset: offset }) do with {:ok, limit} <- explain_limit(limit, offset) do rows |> Enum.map(&Enum.map(&1, fn expr -> explain_literal_expr(expr) end)) |> explain_values_program(limit) end end defp explain_bytecode_rows(_db, _stmt), do: :error defp explain_literal_program(items, limit) do if Enum.any?(items, &(&1 == :error)) do :error else width = length(items) setup_width = explain_limit_setup_width(limit) register_width = explain_limit_register_width(limit) offset_width = explain_limit_offset_width(limit) output_start = 1 + register_width offset_addr = 1 + setup_width literal_start_addr = offset_addr + offset_width {literal_rows, state, result_addr} = explain_literal_expression_rows( items, literal_start_addr, output_start, output_start + width ) decr_addr = result_addr + 1 halt_addr = result_addr + 1 + explain_limit_decr_width(limit) start_addr = halt_addr + 1 post_rows = explain_literal_post_rows(state, start_addr, state.next_reg) goto_addr = start_addr + length(post_rows) rows = [[0, "Init", 0, start_addr, 0, nil, 0, "Start at #{start_addr}"]] ++ explain_limit_rows(limit, halt_addr) ++ explain_limit_offset_rows(limit, offset_addr, halt_addr) ++ literal_rows ++ [ [ result_addr, "ResultRow", output_start, width, 0, nil, 0, explain_result_comment(output_start, width) ] ] ++ explain_limit_decr_rows(limit, decr_addr, halt_addr) ++ [ [halt_addr, "Halt", 0, 0, 0, nil, 0, nil] ] ++ post_rows ++ [ [goto_addr, "Goto", 0, 1, 0, nil, 0, nil] ] {:ok, rows} end end defp explain_literal_expression_rows(items, start_addr, output_start, operand_start_reg) when is_integer(operand_start_reg) do explain_literal_expression_rows( items, start_addr, output_start, explain_operand_state(operand_start_reg) ) end defp explain_literal_expression_rows(items, start_addr, output_start, state) do items |> Enum.with_index() |> Enum.reduce({[], state, start_addr}, fn {item, index}, {rows, state, addr} -> output_reg = output_start + index {item_rows, state, next_addr} = case item do {:literal, opcode} -> {[explain_literal_row(addr, opcode, output_reg)], state, addr + 1} {:cast, opcode, affinity} -> item_rows = [ explain_literal_row(addr, opcode, output_reg), [ addr + 1, "Cast", output_reg, explain_cast_affinity_code(affinity), 0, nil, 0, "affinity(r[#{output_reg}])" ] ] {item_rows, state, addr + 2} {:negate, opcode} -> {zero_reg, state} = explain_operand_register(explain_zero_opcode(), state) {operand_reg, state} = explain_operand_register(opcode, state) row = [ addr, "Subtract", operand_reg, zero_reg, output_reg, nil, 0, "r[#{output_reg}]=r[#{zero_reg}]-r[#{operand_reg}]" ] {[row], state, addr + 1} {:bitnot, opcode} -> {operand_reg, state} = explain_operand_register(opcode, state) row = [ addr, "BitNot", operand_reg, output_reg, 0, nil, 0, "r[#{output_reg}]= ~r[#{operand_reg}]" ] {[row], state, addr + 1} {:not, opcode} -> {operand_reg, state} = explain_operand_register(opcode, state) row = [ addr, "Not", operand_reg, output_reg, 0, nil, 0, "r[#{output_reg}]= !r[#{operand_reg}]" ] {[row], state, addr + 1} {:binary, opcode, left_opcode, right_opcode} -> {left_reg, state} = explain_operand_register(left_opcode, state) {right_reg, state} = explain_operand_register(right_opcode, state) row = [ addr, opcode, right_reg, left_reg, output_reg, nil, 0, explain_binary_comment(opcode, output_reg, left_reg, right_reg) ] {[row], state, addr + 1} {:compare, opcode, left_opcode, right_opcode, collation_p4} -> {left_reg, state} = explain_operand_register(left_opcode, state) {right_reg, state} = explain_operand_register(right_opcode, state) item_rows = [ [addr, "Integer", 1, output_reg, 0, nil, 0, "r[#{output_reg}]=1"], [ addr + 1, opcode, right_reg, addr + 3, left_reg, collation_p4, 64, explain_compare_comment(opcode, left_reg, right_reg, addr + 3) ], [ addr + 2, "ZeroOrNull", left_reg, output_reg, right_reg, nil, 0, "r[#{output_reg}] = 0 OR NULL" ] ] {item_rows, state, addr + 3} {:is_compare, opcode, left_opcode, right_opcode, collation_p4} -> {left_reg, state} = explain_operand_register(left_opcode, state) {right_reg, state} = explain_operand_register(right_opcode, state) item_rows = [ [addr, "Integer", 1, output_reg, 0, nil, 0, "r[#{output_reg}]=1"], [ addr + 1, opcode, right_reg, addr + 3, left_reg, collation_p4, 192, explain_compare_comment(opcode, left_reg, right_reg, addr + 3) ], [addr + 2, "Integer", 0, output_reg, 0, nil, 0, "r[#{output_reg}]=0"] ] {item_rows, state, addr + 3} {:null_test, opcode, operand_opcode} -> {operand_reg, state} = explain_operand_register(operand_opcode, state) item_rows = [ [addr, "Integer", 1, output_reg, 0, nil, 0, "r[#{output_reg}]=1"], [ addr + 1, opcode, operand_reg, addr + 3, 0, nil, 0, explain_null_test_comment(opcode, operand_reg, addr + 3) ], [addr + 2, "Integer", 0, output_reg, 0, nil, 0, "r[#{output_reg}]=0"] ] {item_rows, state, addr + 3} {:in_list, value_opcode, list_opcodes} -> {{value_reg, bitand_reg, list_regs}, state} = explain_in_registers(value_opcode, list_opcodes, state) list_width = length(list_regs) eq_start_addr = addr + 4 match_addr = addr + list_width + 6 addimm_addr = addr + list_width + 7 next_addr = addr + list_width + 8 eq_rows = list_regs |> Enum.with_index() |> Enum.map(fn {list_reg, eq_index} -> [ eq_start_addr + eq_index, "Eq", value_reg, match_addr, list_reg, nil, 0, explain_compare_comment("Eq", list_reg, value_reg, match_addr) ] end) item_rows = [ [addr, "Null", 0, output_reg, 0, nil, 0, "r[#{output_reg}]=NULL"], [addr + 1, "Noop", 0, 0, 0, nil, 0, "begin IN expr"], explain_literal_row(addr + 2, value_opcode, value_reg), [ addr + 3, "BitAnd", value_reg, value_reg, bitand_reg, nil, 0, "r[#{bitand_reg}]=r[#{value_reg}]&r[#{value_reg}]" ] ] ++ eq_rows ++ [ [ addr + list_width + 4, "IsNull", bitand_reg, next_addr, 0, nil, 0, explain_null_test_comment("IsNull", bitand_reg, next_addr) ], [addr + list_width + 5, "Goto", 0, addimm_addr, 0, nil, 0, "end IN expr"], [match_addr, "Integer", 1, output_reg, 0, nil, 0, "r[#{output_reg}]=1"], [ addimm_addr, "AddImm", output_reg, 0, 0, nil, 0, "r[#{output_reg}]=r[#{output_reg}]+0" ] ] {item_rows, state, next_addr} {:not_in_list, value_opcode, list_opcodes} -> {in_reg, state} = explain_post_in_register(value_opcode, list_opcodes, state) row = [ addr, "Not", in_reg, output_reg, 0, nil, 0, "r[#{output_reg}]= !r[#{in_reg}]" ] {[row], state, addr + 1} {:case, nil, branches, else_opcode} -> {item_rows, state, next_addr} = explain_searched_case_rows(addr, output_reg, branches, else_opcode, state) {item_rows, state, next_addr} {:case, operand_opcode, branches, else_opcode} -> {item_rows, state, next_addr} = explain_simple_case_rows( addr, output_reg, operand_opcode, branches, else_opcode, state ) {item_rows, state, next_addr} {:literal_function, name, arg_opcodes, negated} -> {item_rows, state, next_addr} = explain_literal_function_rows(addr, output_reg, name, arg_opcodes, negated, state) {item_rows, state, next_addr} {:coalesce_function, arg_opcodes} -> {item_rows, state, next_addr} = explain_literal_coalesce_rows(addr, output_reg, arg_opcodes, state) {item_rows, state, next_addr} {:nullif_function, left_opcode, right_opcode, collation_p4} -> {item_rows, state, next_addr} = explain_literal_nullif_rows( addr, output_reg, left_opcode, right_opcode, collation_p4, state ) {item_rows, state, next_addr} {:collated_function, name, arg_opcodes, collation_p4} -> {item_rows, state, next_addr} = explain_literal_collated_function_rows( addr, output_reg, name, arg_opcodes, collation_p4, state ) {item_rows, state, next_addr} {:iif_function, condition_opcode, true_opcode, false_opcode} -> {item_rows, state, next_addr} = explain_literal_iif_rows( addr, output_reg, condition_opcode, true_opcode, false_opcode, state ) {item_rows, state, next_addr} {:between, value_opcode, low_opcode, high_opcode} -> {{value_reg, lower_temp, upper_temp, low_reg, high_reg}, state} = explain_between_registers(value_opcode, low_opcode, high_opcode, state) item_rows = [ [addr, "Integer", 1, lower_temp, 0, nil, 0, "r[#{lower_temp}]=1"], [ addr + 1, "Ge", low_reg, addr + 3, value_reg, nil, 64, explain_compare_comment("Ge", value_reg, low_reg, addr + 3) ], [ addr + 2, "ZeroOrNull", value_reg, lower_temp, low_reg, nil, 0, "r[#{lower_temp}] = 0 OR NULL" ], [addr + 3, "Integer", 1, upper_temp, 0, nil, 0, "r[#{upper_temp}]=1"], [ addr + 4, "Le", high_reg, addr + 6, value_reg, nil, 64, explain_compare_comment("Le", value_reg, high_reg, addr + 6) ], [ addr + 5, "ZeroOrNull", value_reg, upper_temp, high_reg, nil, 0, "r[#{upper_temp}] = 0 OR NULL" ], [ addr + 6, "And", upper_temp, lower_temp, output_reg, nil, 0, "r[#{output_reg}]=(r[#{upper_temp}] && r[#{lower_temp}])" ] ] {item_rows, state, addr + 7} {:not_between, value_opcode, low_opcode, high_opcode} -> {between_reg, state} = explain_post_between_register(value_opcode, low_opcode, high_opcode, state) row = [ addr, "Not", between_reg, output_reg, 0, nil, 0, "r[#{output_reg}]= !r[#{between_reg}]" ] {[row], state, addr + 1} end {rows ++ item_rows, state, next_addr} end) end defp explain_literal_post_rows(%{operands: [], post_exprs: []}, _addr, _scratch_start), do: [] defp explain_literal_post_rows(state, addr, scratch_start) do {post_expression_rows, next_addr} = state.post_exprs |> Enum.with_index() |> Enum.reduce({[], addr}, fn {expr, index}, {rows, row_addr} -> expr_rows = explain_post_expression_rows(expr, row_addr, scratch_start, index) {rows ++ expr_rows, row_addr + length(expr_rows)} end) operand_rows = state.operands |> Enum.with_index() |> Enum.map(fn {{opcode, reg}, index} -> explain_literal_row(next_addr + index, opcode, reg) end) post_expression_rows ++ operand_rows end defp explain_post_expression_rows( {:in_list, value_opcode, list_opcodes, result_reg}, addr, scratch_start, index ) do value_reg = scratch_start + index bitand_reg = value_reg + 1 list_reg = value_reg + 2 list_width = length(list_opcodes) match_addr = addr + 2 * list_width + 6 addimm_addr = addr + 2 * list_width + 7 next_addr = addr + 2 * list_width + 8 list_rows = list_opcodes |> Enum.with_index() |> Enum.flat_map(fn {list_opcode, list_index} -> literal_addr = addr + 4 + 2 * list_index eq_addr = literal_addr + 1 [ explain_literal_row(literal_addr, list_opcode, list_reg), [ eq_addr, "Eq", value_reg, match_addr, list_reg, nil, 0, explain_compare_comment("Eq", list_reg, value_reg, match_addr) ] ] end) [ [addr, "Null", 0, result_reg, 0, nil, 0, "r[#{result_reg}]=NULL"], [addr + 1, "Noop", 0, 0, 0, nil, 0, "begin IN expr"], explain_literal_row(addr + 2, value_opcode, value_reg), [ addr + 3, "BitAnd", value_reg, value_reg, bitand_reg, nil, 0, "r[#{bitand_reg}]=r[#{value_reg}]&r[#{value_reg}]" ] ] ++ list_rows ++ [ [ addr + 2 * list_width + 4, "IsNull", bitand_reg, next_addr, 0, nil, 0, explain_null_test_comment("IsNull", bitand_reg, next_addr) ], [addr + 2 * list_width + 5, "Goto", 0, addimm_addr, 0, nil, 0, "end IN expr"], [match_addr, "Integer", 1, result_reg, 0, nil, 0, "r[#{result_reg}]=1"], [ addimm_addr, "AddImm", result_reg, 0, 0, nil, 0, "r[#{result_reg}]=r[#{result_reg}]+0" ] ] end defp explain_post_expression_rows( {:between, value_opcode, low_opcode, high_opcode, result_reg}, addr, scratch_start, index ) do value_reg = scratch_start high_reg = scratch_start + 3 {low_reg, lower_temp, upper_temp} = if rem(index, 2) == 0 do {scratch_start + 2, scratch_start + 1, scratch_start + 2} else {scratch_start + 1, scratch_start + 2, scratch_start + 1} end [ explain_literal_row(addr, value_opcode, value_reg), explain_literal_row(addr + 1, low_opcode, low_reg), [addr + 2, "Integer", 1, lower_temp, 0, nil, 0, "r[#{lower_temp}]=1"], [ addr + 3, "Ge", low_reg, addr + 5, value_reg, nil, 64, explain_compare_comment("Ge", value_reg, low_reg, addr + 5) ], [ addr + 4, "ZeroOrNull", value_reg, lower_temp, low_reg, nil, 0, "r[#{lower_temp}] = 0 OR NULL" ], explain_literal_row(addr + 5, high_opcode, high_reg), [addr + 6, "Integer", 1, upper_temp, 0, nil, 0, "r[#{upper_temp}]=1"], [ addr + 7, "Le", high_reg, addr + 9, value_reg, nil, 64, explain_compare_comment("Le", value_reg, high_reg, addr + 9) ], [ addr + 8, "ZeroOrNull", value_reg, upper_temp, high_reg, nil, 0, "r[#{upper_temp}] = 0 OR NULL" ], [ addr + 9, "And", upper_temp, lower_temp, result_reg, nil, 0, "r[#{result_reg}]=(r[#{upper_temp}] && r[#{lower_temp}])" ] ] end defp explain_post_in_register(value_opcode, list_opcodes, state) do result_reg = state.next_reg state = %{ state | next_reg: result_reg + 1, post_exprs: state.post_exprs ++ [{:in_list, value_opcode, list_opcodes, result_reg}] } {result_reg, state} end defp explain_post_between_register(value_opcode, low_opcode, high_opcode, state) do result_reg = state.next_reg state = %{ state | next_reg: result_reg + 1, post_exprs: state.post_exprs ++ [{:between, value_opcode, low_opcode, high_opcode, result_reg}] } {result_reg, state} end defp explain_operand_state(next_reg) do %{ operands: [], operand_regs: %{}, next_reg: next_reg, between_temps: nil, between_count: 0, between_scratch_reserved: false, in_value_reg: nil, iif_condition_reg: nil, function_output_regs: MapSet.new(), post_exprs: [] } end defp explain_operand_register(opcode, state) do case state.operand_regs do %{^opcode => reg} -> {reg, state} _other -> reg = state.next_reg state = %{ state | operands: state.operands ++ [{opcode, reg}], operand_regs: Map.put(state.operand_regs, opcode, reg), next_reg: reg + 1 } {reg, state} end end defp explain_in_registers(_value_opcode, list_opcodes, %{in_value_reg: nil} = state) do value_reg = state.next_reg bitand_reg = value_reg + 1 state = %{state | next_reg: bitand_reg + 1, in_value_reg: bitand_reg} {list_regs, state} = explain_operand_registers(list_opcodes, state) {{value_reg, bitand_reg, list_regs}, state} end defp explain_in_registers(_value_opcode, list_opcodes, state) do value_reg = state.in_value_reg bitand_reg = state.next_reg state = %{state | next_reg: bitand_reg + 1, in_value_reg: bitand_reg} {list_regs, state} = explain_operand_registers(list_opcodes, state) {{value_reg, bitand_reg, list_regs}, state} end defp explain_operand_registers(opcodes, state) do Enum.map_reduce(opcodes, state, fn opcode, state -> explain_operand_register(opcode, state) end) end defp explain_case_operand_registers(operand_opcode, when_opcodes, state) do {operand_reg, state} = explain_operand_register(operand_opcode, state) state = %{state | next_reg: state.next_reg + 1} {when_regs, state} = explain_operand_registers(when_opcodes, state) {{operand_reg, when_regs}, state} end defp explain_function_registers(arg_opcodes, state) do result_reg = state.next_reg arg_start_reg = result_reg + 1 arg_regs = Enum.to_list(arg_start_reg..(arg_start_reg + length(arg_opcodes) - 1)//1) state = %{state | next_reg: result_reg + length(arg_opcodes) + 1} {{result_reg, arg_start_reg, arg_regs}, state} end defp explain_between_registers(value_opcode, low_opcode, high_opcode, state) do {value_reg, state} = explain_operand_register(value_opcode, state) {lower_base, upper_base, state} = explain_between_temp_registers(state) {lower_temp, upper_temp} = if rem(state.between_count, 2) == 0 do {lower_base, upper_base} else {upper_base, lower_base} end {low_reg, state} = explain_operand_register(low_opcode, state) state = explain_reserve_between_scratch(state) {high_reg, state} = explain_operand_register(high_opcode, state) state = %{state | between_count: state.between_count + 1} {{value_reg, lower_temp, upper_temp, low_reg, high_reg}, state} end defp explain_between_temp_registers(%{between_temps: {lower_reg, upper_reg}} = state), do: {lower_reg, upper_reg, state} defp explain_between_temp_registers(state) do lower_reg = state.next_reg upper_reg = lower_reg + 1 state = %{state | between_temps: {lower_reg, upper_reg}, next_reg: upper_reg + 1} {lower_reg, upper_reg, state} end defp explain_reserve_between_scratch(%{between_scratch_reserved: true} = state), do: state defp explain_reserve_between_scratch(state), do: %{state | next_reg: state.next_reg + 1, between_scratch_reserved: true} defp explain_searched_case_rows(addr, output_reg, branches, else_opcode, state) do {branch_specs, state} = Enum.map_reduce(branches, state, fn {when_opcode, then_opcode}, state -> case explain_case_truth_opcode(when_opcode) do true -> {{true, then_opcode}, state} false -> {{false, then_opcode}, state} :dynamic -> {when_reg, state} = explain_operand_register(when_opcode, state) {{:dynamic, when_reg, then_opcode}, state} end end) end_addr = addr + Enum.reduce(branch_specs, 1, fn spec, width -> width + explain_case_branch_width(spec) end) {branch_rows, else_addr} = Enum.reduce(branch_specs, {[], addr}, fn spec, {rows, branch_addr} -> next_addr = branch_addr + explain_case_branch_width(spec) {rows ++ explain_searched_case_branch_rows(spec, branch_addr, next_addr, end_addr, output_reg), next_addr} end) else_rows = [explain_case_else_row(else_addr, else_opcode, output_reg)] {branch_rows ++ else_rows, state, end_addr} end defp explain_simple_case_rows(addr, output_reg, operand_opcode, branches, else_opcode, state) do when_opcodes = Enum.map(branches, &elem(&1, 0)) {{operand_reg, when_regs}, state} = explain_case_operand_registers(operand_opcode, when_opcodes, state) branch_specs = branches |> Enum.map(&elem(&1, 1)) |> Enum.zip(when_regs) end_addr = addr + 1 + length(branch_specs) * 3 {branch_rows, else_addr} = Enum.reduce(branch_specs, {[], addr}, fn {then_opcode, when_reg}, {rows, branch_addr} -> next_addr = branch_addr + 3 branch_rows = [ [ branch_addr, "Ne", when_reg, next_addr, operand_reg, nil, 80, explain_compare_comment("Ne", operand_reg, when_reg, next_addr) ], explain_literal_row(branch_addr + 1, then_opcode, output_reg), [branch_addr + 2, "Goto", 0, end_addr, 0, nil, 0, nil] ] {rows ++ branch_rows, next_addr} end) else_rows = [explain_case_else_row(else_addr, else_opcode, output_reg)] {branch_rows ++ else_rows, state, end_addr} end defp explain_case_truth_opcode({"Integer", value, nil}) when value != 0, do: true defp explain_case_truth_opcode({"Integer", 0, nil}), do: false defp explain_case_truth_opcode(_opcode), do: :dynamic defp explain_case_branch_width({true, _then_opcode}), do: 2 defp explain_case_branch_width({false, _then_opcode}), do: 3 defp explain_case_branch_width({:dynamic, _when_reg, _then_opcode}), do: 3 defp explain_searched_case_branch_rows( {true, then_opcode}, addr, _next_addr, end_addr, output_reg ) do [ explain_literal_row(addr, then_opcode, output_reg), [addr + 1, "Goto", 0, end_addr, 0, nil, 0, nil] ] end defp explain_searched_case_branch_rows( {false, then_opcode}, addr, next_addr, end_addr, output_reg ) do [ [addr, "Goto", 0, next_addr, 0, nil, 0, nil], explain_literal_row(addr + 1, then_opcode, output_reg), [addr + 2, "Goto", 0, end_addr, 0, nil, 0, nil] ] end defp explain_searched_case_branch_rows( {:dynamic, when_reg, then_opcode}, addr, next_addr, end_addr, output_reg ) do [ [addr, "IfNot", when_reg, next_addr, 1, nil, 0, nil], explain_literal_row(addr + 1, then_opcode, output_reg), [addr + 2, "Goto", 0, end_addr, 0, nil, 0, nil] ] end defp explain_case_else_row(addr, nil, output_reg), do: [addr, "Null", 0, output_reg, 0, nil, 0, "r[#{output_reg}]=NULL"] defp explain_case_else_row(addr, opcode, output_reg), do: explain_literal_row(addr, opcode, output_reg) defp explain_literal_function_rows(addr, output_reg, name, arg_opcodes, negated, state) do {{result_reg, arg_start_reg, arg_regs}, state} = explain_function_registers(arg_opcodes, state) arg_count = length(arg_opcodes) function_addr = addr + arg_count + 1 output_addr = function_addr + 1 next_addr = output_addr + 1 arg_rows = arg_opcodes |> Enum.zip(arg_regs) |> Enum.with_index() |> Enum.map(fn {{opcode, reg}, index} -> explain_literal_row(addr + 1 + index, opcode, reg) end) {output_row, state} = explain_function_output_row(output_addr, result_reg, output_reg, negated, state) rows = [ [addr, "Once", 0, output_addr, 0, nil, 0, nil] ] ++ arg_rows ++ [ [ function_addr, "Function", explain_function_const_mask(arg_count), explain_function_start_reg(arg_start_reg, arg_count), result_reg, explain_function_p4(name, arg_count), 0, explain_function_comment(result_reg, arg_start_reg, arg_count) ], output_row ] {rows, state, next_addr} end defp explain_literal_coalesce_rows(addr, output_reg, arg_opcodes, state) do result_reg = state.next_reg state = %{state | next_reg: result_reg + 1} arg_count = length(arg_opcodes) output_addr = addr + 2 * arg_count arg_rows = arg_opcodes |> Enum.with_index() |> Enum.flat_map(fn {opcode, index} -> literal_addr = addr + 1 + 2 * index if index == arg_count - 1 do [explain_literal_row(literal_addr, opcode, result_reg)] else [ explain_literal_row(literal_addr, opcode, result_reg), [ literal_addr + 1, "NotNull", result_reg, output_addr, 0, nil, 0, explain_null_test_comment("NotNull", result_reg, output_addr) ] ] end end) {output_row, state} = explain_computed_output_row(output_addr, result_reg, output_reg, state) rows = [[addr, "Once", 0, output_addr, 0, nil, 0, nil]] ++ arg_rows ++ [output_row] {rows, state, output_addr + 1} end defp explain_literal_nullif_rows( addr, output_reg, left_opcode, right_opcode, collation_p4, state ) do arg_opcodes = [left_opcode, right_opcode] {{result_reg, arg_start_reg, arg_regs}, state} = explain_function_registers(arg_opcodes, state) function_addr = addr + 4 output_addr = function_addr + 1 arg_rows = arg_opcodes |> Enum.zip(arg_regs) |> Enum.with_index() |> Enum.map(fn {{opcode, reg}, index} -> explain_literal_row(addr + 1 + index, opcode, reg) end) {output_row, state} = explain_computed_output_row(output_addr, result_reg, output_reg, state) rows = [ [addr, "Once", 0, output_addr, 0, nil, 0, nil] ] ++ arg_rows ++ [ [addr + 3, "CollSeq", 0, 0, 0, collation_p4, 0, nil], [ function_addr, "Function", explain_function_const_mask(2), arg_start_reg, result_reg, "nullif(2)", 0, explain_function_comment(result_reg, arg_start_reg, 2) ], output_row ] {rows, state, output_addr + 1} end defp explain_literal_collated_function_rows( addr, output_reg, name, arg_opcodes, collation_p4, state ) do {{result_reg, arg_start_reg, arg_regs}, state} = explain_function_registers(arg_opcodes, state) arg_count = length(arg_opcodes) collseq_addr = addr + arg_count + 1 function_addr = collseq_addr + 1 output_addr = function_addr + 1 arg_rows = arg_opcodes |> Enum.zip(arg_regs) |> Enum.with_index() |> Enum.map(fn {{opcode, reg}, index} -> explain_literal_row(addr + 1 + index, opcode, reg) end) {output_row, state} = explain_computed_output_row(output_addr, result_reg, output_reg, state) rows = [ [addr, "Once", 0, output_addr, 0, nil, 0, nil] ] ++ arg_rows ++ [ [collseq_addr, "CollSeq", 0, 0, 0, collation_p4, 0, nil], [ function_addr, "Function", explain_function_const_mask(arg_count), arg_start_reg, result_reg, explain_function_p4(name, arg_count), 0, explain_function_comment(result_reg, arg_start_reg, arg_count) ], output_row ] {rows, state, output_addr + 1} end defp explain_literal_iif_rows( addr, output_reg, condition_opcode, true_opcode, false_opcode, state ) do {{result_reg, condition_reg}, state} = explain_iif_registers(condition_opcode, state) {rows, output_addr} = case {explain_case_truth_opcode(condition_opcode), condition_reg} do {true, nil} -> output_addr = addr + 4 {[ [addr, "Once", 0, output_addr, 0, nil, 0, nil], explain_literal_row(addr + 1, true_opcode, result_reg), [addr + 2, "Goto", 0, output_addr, 0, nil, 0, nil], explain_literal_row(addr + 3, false_opcode, result_reg) ], output_addr} {false, nil} -> false_addr = addr + 4 output_addr = addr + 5 {[ [addr, "Once", 0, output_addr, 0, nil, 0, nil], [addr + 1, "Goto", 0, false_addr, 0, nil, 0, nil], explain_literal_row(addr + 2, true_opcode, result_reg), [addr + 3, "Goto", 0, output_addr, 0, nil, 0, nil], explain_literal_row(false_addr, false_opcode, result_reg) ], output_addr} {:dynamic, condition_reg} -> false_addr = addr + 5 output_addr = addr + 6 {[ [addr, "Once", 0, output_addr, 0, nil, 0, nil], explain_literal_row(addr + 1, condition_opcode, condition_reg), [addr + 2, "IfNot", condition_reg, false_addr, 1, nil, 0, nil], explain_literal_row(addr + 3, true_opcode, result_reg), [addr + 4, "Goto", 0, output_addr, 0, nil, 0, nil], explain_literal_row(false_addr, false_opcode, result_reg) ], output_addr} end {output_row, state} = explain_computed_output_row(output_addr, result_reg, output_reg, state) {rows ++ [output_row], state, output_addr + 1} end defp explain_function_output_row(addr, result_reg, output_reg, true, state) do row = [ addr, "Not", result_reg, output_reg, 0, nil, 0, "r[#{output_reg}]= !r[#{result_reg}]" ] {row, state} end defp explain_function_output_row(addr, result_reg, output_reg, false, state), do: explain_computed_output_row(addr, result_reg, output_reg, state) defp explain_computed_output_row(addr, result_reg, output_reg, state) do opcode = if MapSet.member?(state.function_output_regs, output_reg), do: "SCopy", else: "Copy" row = [ addr, opcode, result_reg, output_reg, 0, nil, 0, "r[#{output_reg}]=r[#{result_reg}]" ] state = %{state | function_output_regs: MapSet.put(state.function_output_regs, output_reg)} {row, state} end defp explain_function_const_mask(arg_count), do: Integer.pow(2, arg_count) - 1 defp explain_function_start_reg(_arg_start_reg, 0), do: 0 defp explain_function_start_reg(arg_start_reg, _arg_count), do: arg_start_reg defp explain_function_p4(name, _arg_count) when name in ["char", "format", "printf"], do: "#{name}(-1)" defp explain_function_p4("concat", _arg_count), do: "concat(-3)" defp explain_function_p4("concat_ws", _arg_count), do: "concat_ws(-4)" defp explain_function_p4(name, _arg_count) when name in ["min", "max"], do: "#{name}(-3)" defp explain_function_p4(name, arg_count), do: "#{name}(#{arg_count})" defp explain_function_comment(result_reg, _arg_start_reg, 0), do: "r[#{result_reg}]=func()" defp explain_function_comment(result_reg, arg_start_reg, 1), do: "r[#{result_reg}]=func(r[#{arg_start_reg}])" defp explain_function_comment(result_reg, arg_start_reg, arg_count), do: "r[#{result_reg}]=func(r[#{arg_start_reg}..#{arg_start_reg + arg_count - 1}])" defp explain_iif_registers(condition_opcode, state) do result_reg = state.next_reg state = %{state | next_reg: result_reg + 1} case explain_case_truth_opcode(condition_opcode) do :dynamic -> explain_iif_condition_register(result_reg, state) _constant -> {{result_reg, nil}, state} end end defp explain_iif_condition_register(result_reg, %{iif_condition_reg: reg} = state) when not is_nil(reg), do: {{result_reg, reg}, state} defp explain_iif_condition_register(result_reg, state) do condition_reg = state.next_reg state = %{state | next_reg: condition_reg + 1, iif_condition_reg: condition_reg} {{result_reg, condition_reg}, state} end defp explain_literal_row(addr, {opcode, p1, p4}, reg), do: [addr, opcode, p1, reg, 0, p4, 0, explain_literal_comment(opcode, p1, p4, reg)] defp explain_values_program(rows, limit) do cond do Enum.any?(rows, &Enum.any?(&1, fn item -> item == :error end)) -> :error rows |> Enum.map(&length/1) |> Enum.uniq() |> length() != 1 -> :error true -> width = rows |> hd() |> length() row_count = length(rows) limit_register_width = explain_limit_register_width(limit) limit_setup_width = explain_limit_setup_width(limit) coroutine_reg = limit_register_width + 1 producer_reg_start = coroutine_reg + 3 operand_count = explain_expression_operand_count(rows) base_output_start = producer_reg_start + width * 2 output_start = if operand_count == 0 do base_output_start else base_output_start + operand_count end {producer_rows, state, producer_end_addr} = rows |> Enum.with_index() |> Enum.reduce( {[], explain_operand_state(base_output_start), limit_setup_width + 2}, fn {row, _row_index}, {acc, state, row_start_addr} -> {literal_rows, state, yield_row_addr} = explain_literal_expression_rows(row, row_start_addr, producer_reg_start, state) producer_rows = acc ++ literal_rows ++ [[yield_row_addr, "Yield", coroutine_reg, 0, 0, nil, 0, nil]] {producer_rows, state, yield_row_addr + 1} end ) consumer_start_addr = producer_end_addr + 1 yield_addr = consumer_start_addr + 1 copy_start_addr = yield_addr + 1 + explain_limit_offset_width(limit) result_addr = copy_start_addr + width decr_addr = result_addr + 1 loop_addr = result_addr + 1 + explain_limit_decr_width(limit) halt_addr = loop_addr + 1 start_addr = halt_addr + 1 post_rows = explain_literal_post_rows(state, start_addr, output_start + width) goto_addr = start_addr + length(post_rows) copy_rows = 0..(width - 1) |> Enum.map(fn column_index -> src_reg = producer_reg_start + column_index dest_reg = output_start + column_index [ copy_start_addr + column_index, "Copy", src_reg, dest_reg, 0, nil, 2, "r[#{dest_reg}]=r[#{src_reg}]" ] end) {:ok, [[0, "Init", 0, start_addr, 0, nil, 0, "Start at #{start_addr}"]] ++ explain_limit_rows(limit, halt_addr) ++ [ [ limit_setup_width + 1, "InitCoroutine", coroutine_reg, consumer_start_addr, 2, nil, 0, nil ] ] ++ producer_rows ++ [ [producer_end_addr, "EndCoroutine", coroutine_reg, 0, 0, nil, 0, nil], [consumer_start_addr, "InitCoroutine", coroutine_reg, 0, 2, nil, 0, nil], [ yield_addr, "Yield", coroutine_reg, halt_addr, 0, nil, 0, "next row of #{row_count}-ROW VALUES CLAUSE" ] ] ++ explain_limit_offset_rows(limit, yield_addr + 1, loop_addr) ++ copy_rows ++ [ [ result_addr, "ResultRow", output_start, width, 0, nil, 0, explain_result_comment(output_start, width) ] ] ++ explain_limit_decr_rows(limit, decr_addr, halt_addr) ++ [ [loop_addr, "Goto", 0, yield_addr, 0, nil, 0, nil], [halt_addr, "Halt", 0, 0, 0, nil, 0, nil] ] ++ post_rows ++ [ [goto_addr, "Goto", 0, 1, 0, nil, 0, nil] ]} end end defp explain_expression_operand_count(rows) do rows |> List.flatten() |> Enum.reduce(explain_operand_state(0), fn item, state -> {_regs, state} = explain_expression_registers(item, state) state end) |> Map.fetch!(:next_reg) end defp explain_expression_registers({:negate, opcode}, state) do {zero_reg, state} = explain_operand_register(explain_zero_opcode(), state) {operand_reg, state} = explain_operand_register(opcode, state) {[zero_reg, operand_reg], state} end defp explain_expression_registers({:bitnot, opcode}, state) do {operand_reg, state} = explain_operand_register(opcode, state) {[operand_reg], state} end defp explain_expression_registers({:not, opcode}, state) do {operand_reg, state} = explain_operand_register(opcode, state) {[operand_reg], state} end defp explain_expression_registers({:in_list, value_opcode, list_opcodes}, state) do {{value_reg, bitand_reg, list_regs}, state} = explain_in_registers(value_opcode, list_opcodes, state) {[value_reg, bitand_reg | list_regs], state} end defp explain_expression_registers({:not_in_list, value_opcode, list_opcodes}, state) do {in_reg, state} = explain_post_in_register(value_opcode, list_opcodes, state) {[in_reg], state} end defp explain_expression_registers({:case, nil, branches, _else_opcode}, state) do Enum.map_reduce(branches, state, fn {when_opcode, _then_opcode}, state -> case explain_case_truth_opcode(when_opcode) do :dynamic -> explain_operand_register(when_opcode, state) _constant -> {nil, state} end end) end defp explain_expression_registers({:case, operand_opcode, branches, _else_opcode}, state) do when_opcodes = Enum.map(branches, &elem(&1, 0)) {{operand_reg, when_regs}, state} = explain_case_operand_registers(operand_opcode, when_opcodes, state) {[operand_reg | when_regs], state} end defp explain_expression_registers({:literal_function, _name, arg_opcodes, _negated}, state) do {{result_reg, _arg_start_reg, arg_regs}, state} = explain_function_registers(arg_opcodes, state) {[result_reg | arg_regs], state} end defp explain_expression_registers({:coalesce_function, _arg_opcodes}, state) do result_reg = state.next_reg {[result_reg], %{state | next_reg: result_reg + 1}} end defp explain_expression_registers( {:nullif_function, left_opcode, right_opcode, _collation_p4}, state ) do {{result_reg, _arg_start_reg, arg_regs}, state} = explain_function_registers([left_opcode, right_opcode], state) {[result_reg | arg_regs], state} end defp explain_expression_registers( {:collated_function, _name, arg_opcodes, _collation_p4}, state ) do {{result_reg, _arg_start_reg, arg_regs}, state} = explain_function_registers(arg_opcodes, state) {[result_reg | arg_regs], state} end defp explain_expression_registers( {:iif_function, condition_opcode, _true_opcode, _false_opcode}, state ) do {{result_reg, condition_reg}, state} = explain_iif_registers(condition_opcode, state) regs = if is_nil(condition_reg), do: [result_reg], else: [result_reg, condition_reg] {regs, state} end defp explain_expression_registers({:binary, _opcode, left_opcode, right_opcode}, state) do {left_reg, state} = explain_operand_register(left_opcode, state) {right_reg, state} = explain_operand_register(right_opcode, state) {[left_reg, right_reg], state} end defp explain_expression_registers( {tag, _opcode, left_opcode, right_opcode, _collation_p4}, state ) when tag in [:compare, :is_compare] do {left_reg, state} = explain_operand_register(left_opcode, state) {right_reg, state} = explain_operand_register(right_opcode, state) {[left_reg, right_reg], state} end defp explain_expression_registers({:null_test, _opcode, operand_opcode}, state) do {operand_reg, state} = explain_operand_register(operand_opcode, state) {[operand_reg], state} end defp explain_expression_registers({:between, value_opcode, low_opcode, high_opcode}, state) do {{value_reg, lower_temp, upper_temp, low_reg, high_reg}, state} = explain_between_registers(value_opcode, low_opcode, high_opcode, state) {[value_reg, lower_temp, upper_temp, low_reg, high_reg], state} end defp explain_expression_registers({:not_between, value_opcode, low_opcode, high_opcode}, state) do {between_reg, state} = explain_post_between_register(value_opcode, low_opcode, high_opcode, state) {[between_reg], state} end defp explain_expression_registers(_item, state), do: {[], state} defp explain_zero_opcode, do: {"Integer", 0, nil} defp explain_binary_comment("Add", output_reg, left_reg, right_reg), do: "r[#{output_reg}]=r[#{right_reg}]+r[#{left_reg}]" defp explain_binary_comment("Subtract", output_reg, left_reg, right_reg), do: "r[#{output_reg}]=r[#{left_reg}]-r[#{right_reg}]" defp explain_binary_comment("Multiply", output_reg, left_reg, right_reg), do: "r[#{output_reg}]=r[#{right_reg}]*r[#{left_reg}]" defp explain_binary_comment("Divide", output_reg, left_reg, right_reg), do: "r[#{output_reg}]=r[#{left_reg}]/r[#{right_reg}]" defp explain_binary_comment("Remainder", output_reg, left_reg, right_reg), do: "r[#{output_reg}]=r[#{left_reg}]%r[#{right_reg}]" defp explain_binary_comment("Concat", output_reg, left_reg, right_reg), do: "r[#{output_reg}]=r[#{left_reg}]+r[#{right_reg}]" defp explain_binary_comment("BitAnd", output_reg, left_reg, right_reg), do: "r[#{output_reg}]=r[#{right_reg}]&r[#{left_reg}]" defp explain_binary_comment("BitOr", output_reg, left_reg, right_reg), do: "r[#{output_reg}]=r[#{right_reg}]|r[#{left_reg}]" defp explain_binary_comment("ShiftLeft", output_reg, left_reg, right_reg), do: "r[#{output_reg}]=r[#{left_reg}]<>r[#{right_reg}]" defp explain_binary_comment("And", output_reg, left_reg, right_reg), do: "r[#{output_reg}]=(r[#{right_reg}] && r[#{left_reg}])" defp explain_binary_comment("Or", output_reg, left_reg, right_reg), do: "r[#{output_reg}]=(r[#{right_reg}] || r[#{left_reg}])" defp explain_compare_comment("Eq", left_reg, right_reg, next_addr), do: "if r[#{left_reg}]==r[#{right_reg}] goto #{next_addr}" defp explain_compare_comment("Ne", left_reg, right_reg, next_addr), do: "if r[#{left_reg}]!=r[#{right_reg}] goto #{next_addr}" defp explain_compare_comment("Lt", left_reg, right_reg, next_addr), do: "if r[#{left_reg}]r[#{right_reg}] goto #{next_addr}" defp explain_compare_comment("Ge", left_reg, right_reg, next_addr), do: "if r[#{left_reg}]>=r[#{right_reg}] goto #{next_addr}" defp explain_null_test_comment("IsNull", operand_reg, next_addr), do: "if r[#{operand_reg}]==NULL goto #{next_addr}" defp explain_null_test_comment("NotNull", operand_reg, next_addr), do: "if r[#{operand_reg}]!=NULL goto #{next_addr}" defp explain_literal_expr({:collate, expr, _name}), do: explain_literal_expr(expr) defp explain_literal_expr({:cast, expr, affinity}) do case explain_literal_opcode(expr) do :error -> :error opcode -> {:cast, opcode, affinity} end end defp explain_literal_expr({:negate, {:literal, value}} = expr) when is_integer(value) or is_float(value) do {:literal, explain_literal_opcode(expr)} end defp explain_literal_expr({:negate, {:literal, value}}) do case explain_literal_opcode({:literal, value}) do :error -> :error opcode -> {:negate, opcode} end end defp explain_literal_expr({:bitnot, expr}) do case explain_literal_opcode(expr) do :error -> :error opcode -> {:bitnot, opcode} end end defp explain_literal_expr({:not, expr}) do case explain_literal_opcode(expr) do :error -> :error opcode -> {:not, opcode} end end defp explain_literal_expr({:binary, :and, left, right}) do with left_opcode when left_opcode != :error <- explain_literal_opcode(left), right_opcode when right_opcode != :error <- explain_literal_opcode(right) do if explain_zero_integer_opcode?(left_opcode) or explain_zero_integer_opcode?(right_opcode) do {:literal, explain_zero_opcode()} else {:binary, "And", left_opcode, right_opcode} end else _other -> :error end end defp explain_literal_expr({:binary, :or, left, right}) do with left_opcode when left_opcode != :error <- explain_literal_opcode(left), right_opcode when right_opcode != :error <- explain_literal_opcode(right) do {:binary, "Or", left_opcode, right_opcode} else _other -> :error end end defp explain_literal_expr({:in, _expr, [], false}), do: {:literal, explain_zero_opcode()} defp explain_literal_expr({:in, _expr, [], true}), do: {:literal, {"Integer", 1, nil}} defp explain_literal_expr({:in, expr, list, false}) when is_list(list) do with value_opcode when value_opcode != :error <- explain_literal_opcode(expr), list_opcodes <- Enum.map(list, &explain_literal_opcode/1), false <- Enum.any?(list_opcodes, &(&1 == :error)) do {:in_list, value_opcode, list_opcodes} else _other -> :error end end defp explain_literal_expr({:in, expr, list, true}) when is_list(list) do with value_opcode when value_opcode != :error <- explain_literal_opcode(expr), list_opcodes <- Enum.map(list, &explain_literal_opcode/1), false <- Enum.any?(list_opcodes, &(&1 == :error)) do {:not_in_list, value_opcode, list_opcodes} else _other -> :error end end defp explain_literal_expr({:case, operand, branches, else_expr}) do with operand_opcode <- explain_case_operand_opcode(operand), false <- operand_opcode == :error, branch_opcodes <- explain_case_branch_opcodes(branches), false <- branch_opcodes == :error, else_opcode <- explain_case_else_opcode(else_expr), false <- else_opcode == :error do {:case, operand_opcode, branch_opcodes, else_opcode} else _other -> :error end end defp explain_literal_expr({:like, expr, pattern, nil, negated}) do with pattern_opcode when pattern_opcode != :error <- explain_literal_opcode(pattern), expr_opcode when expr_opcode != :error <- explain_literal_opcode(expr) do {:literal_function, "like", [pattern_opcode, expr_opcode], negated} else _other -> :error end end defp explain_literal_expr({:like, expr, pattern, escape, negated}) do with pattern_opcode when pattern_opcode != :error <- explain_literal_opcode(pattern), expr_opcode when expr_opcode != :error <- explain_literal_opcode(expr), escape_opcode when escape_opcode != :error <- explain_literal_opcode(escape) do {:literal_function, "like", [pattern_opcode, expr_opcode, escape_opcode], negated} else _other -> :error end end defp explain_literal_expr({:glob, expr, pattern, negated}) do with pattern_opcode when pattern_opcode != :error <- explain_literal_opcode(pattern), expr_opcode when expr_opcode != :error <- explain_literal_opcode(expr) do {:literal_function, "glob", [pattern_opcode, expr_opcode], negated} else _other -> :error end end defp explain_literal_expr({:regexp, expr, pattern, negated}) do with pattern_opcode when pattern_opcode != :error <- explain_literal_opcode(pattern), expr_opcode when expr_opcode != :error <- explain_literal_opcode(expr) do {:literal_function, "regexp", [pattern_opcode, expr_opcode], negated} else _other -> :error end end defp explain_literal_expr({:function, name, args}) when name in ["coalesce", "ifnull"] and is_list(args) do with {:ok, arity_range} <- Map.fetch(@scalar_arity, name), true <- length(args) in arity_range, arg_opcodes <- Enum.map(args, &explain_literal_opcode/1), false <- Enum.any?(arg_opcodes, &(&1 == :error)) do {:coalesce_function, arg_opcodes} else _other -> :error end end defp explain_literal_expr({:function, "nullif", [left, right]}) do with left_opcode when left_opcode != :error <- explain_literal_opcode(left), right_opcode when right_opcode != :error <- explain_literal_opcode(right) do {:nullif_function, left_opcode, right_opcode, explain_nullif_collation_p4(left, right)} else _other -> :error end end defp explain_literal_expr({:function, name, [_, _ | _] = args}) when name in ["min", "max"] do with true <- length(args) in 2..127, arg_opcodes <- Enum.map(args, &explain_literal_opcode/1), false <- Enum.any?(arg_opcodes, &(&1 == :error)) do {:collated_function, name, arg_opcodes, explain_collated_function_p4(args)} else _other -> :error end end defp explain_literal_expr({:function, "iif", [condition, true_expr, false_expr]}) do with condition_opcode when condition_opcode != :error <- explain_literal_opcode(condition), true_opcode when true_opcode != :error <- explain_literal_opcode(true_expr), false_opcode when false_opcode != :error <- explain_literal_opcode(false_expr) do {:iif_function, condition_opcode, true_opcode, false_opcode} else _other -> :error end end defp explain_literal_expr({:function, name, args}) when is_list(args) do with true <- name in @explain_literal_scalar_functions, {:ok, arity_range} <- Map.fetch(@scalar_arity, name), true <- length(args) in arity_range, arg_opcodes <- Enum.map(args, &explain_literal_opcode/1), false <- Enum.any?(arg_opcodes, &(&1 == :error)) do {:literal_function, name, arg_opcodes, false} else _other -> :error end end defp explain_literal_expr({:binary, op, left, right}) when op in [:add, :sub, :mul, :div, :mod, :concat, :bitand, :bitor, :shl, :shr] do with {:ok, opcode} <- explain_binary_opcode(op), left_opcode when left_opcode != :error <- explain_literal_opcode(left), right_opcode when right_opcode != :error <- explain_literal_opcode(right) do {:binary, opcode, left_opcode, right_opcode} else _other -> :error end end defp explain_literal_expr({:binary, op, left, right}) when op in [:eq, :ne, :lt, :le, :gt, :ge] do with {:ok, opcode} <- explain_compare_opcode(op), left_opcode when left_opcode != :error <- explain_literal_opcode(left), right_opcode when right_opcode != :error <- explain_literal_opcode(right) do {:compare, opcode, left_opcode, right_opcode, explain_compare_collation_p4(left, right)} else _other -> :error end end defp explain_literal_expr({:is, left, {:literal, nil}}) do case explain_literal_opcode(left) do :error -> :error opcode -> {:null_test, "IsNull", opcode} end end defp explain_literal_expr({:is_not, left, {:literal, nil}}) do case explain_literal_opcode(left) do :error -> :error opcode -> {:null_test, "NotNull", opcode} end end defp explain_literal_expr({:is, left, right}) do with left_opcode when left_opcode != :error <- explain_literal_opcode(left), right_opcode when right_opcode != :error <- explain_literal_opcode(right) do {:is_compare, "Eq", left_opcode, right_opcode, explain_compare_collation_p4(left, right)} else _other -> :error end end defp explain_literal_expr({:is_not, left, right}) do with left_opcode when left_opcode != :error <- explain_literal_opcode(left), right_opcode when right_opcode != :error <- explain_literal_opcode(right) do {:is_compare, "Ne", left_opcode, right_opcode, explain_compare_collation_p4(left, right)} else _other -> :error end end defp explain_literal_expr({:between, expr, low, high, false}) do with value_opcode when value_opcode != :error <- explain_literal_opcode(expr), low_opcode when low_opcode != :error <- explain_literal_opcode(low), high_opcode when high_opcode != :error <- explain_literal_opcode(high) do {:between, value_opcode, low_opcode, high_opcode} else _other -> :error end end defp explain_literal_expr({:between, expr, low, high, true}) do with value_opcode when value_opcode != :error <- explain_literal_opcode(expr), low_opcode when low_opcode != :error <- explain_literal_opcode(low), high_opcode when high_opcode != :error <- explain_literal_opcode(high) do {:not_between, value_opcode, low_opcode, high_opcode} else _other -> :error end end defp explain_literal_expr(expr) do case explain_literal_opcode(expr) do :error -> :error opcode -> {:literal, opcode} end end defp explain_case_operand_opcode(nil), do: nil defp explain_case_operand_opcode(expr), do: explain_literal_opcode(expr) defp explain_case_branch_opcodes(branches) do branch_opcodes = Enum.map(branches, fn {when_expr, then_expr} -> {explain_literal_opcode(when_expr), explain_literal_opcode(then_expr)} end) if Enum.any?(branch_opcodes, fn {when_opcode, then_opcode} -> when_opcode == :error or then_opcode == :error end) do :error else branch_opcodes end end defp explain_case_else_opcode(nil), do: nil defp explain_case_else_opcode(expr), do: explain_literal_opcode(expr) defp explain_compare_collation_p4(left, right) do case explain_explicit_collation_name(left) || explain_explicit_collation_name(right) do nil -> nil name -> "#{String.upcase(to_string(name))}-8" end end defp explain_nullif_collation_p4(left, right), do: explain_compare_collation_p4(left, right) || "BINARY-8" defp explain_collated_function_p4(args) do args |> Enum.find_value(&explain_explicit_collation_name/1) |> case do nil -> "BINARY-8" name -> "#{String.upcase(to_string(name))}-8" end end defp explain_explicit_collation_name({:collate, _expr, name}), do: name defp explain_explicit_collation_name(_expr), do: nil defp explain_binary_opcode(:add), do: {:ok, "Add"} defp explain_binary_opcode(:sub), do: {:ok, "Subtract"} defp explain_binary_opcode(:mul), do: {:ok, "Multiply"} defp explain_binary_opcode(:div), do: {:ok, "Divide"} defp explain_binary_opcode(:mod), do: {:ok, "Remainder"} defp explain_binary_opcode(:concat), do: {:ok, "Concat"} defp explain_binary_opcode(:bitand), do: {:ok, "BitAnd"} defp explain_binary_opcode(:bitor), do: {:ok, "BitOr"} defp explain_binary_opcode(:shl), do: {:ok, "ShiftLeft"} defp explain_binary_opcode(:shr), do: {:ok, "ShiftRight"} defp explain_compare_opcode(:eq), do: {:ok, "Eq"} defp explain_compare_opcode(:ne), do: {:ok, "Ne"} defp explain_compare_opcode(:lt), do: {:ok, "Lt"} defp explain_compare_opcode(:le), do: {:ok, "Le"} defp explain_compare_opcode(:gt), do: {:ok, "Gt"} defp explain_compare_opcode(:ge), do: {:ok, "Ge"} defp explain_literal_opcode({:literal, nil}), do: {"Null", 0, nil} defp explain_literal_opcode({:literal, value}) when is_integer(value), do: explain_integer_literal_opcode(value) defp explain_literal_opcode({:literal, value}) when is_boolean(value), do: {"Integer", bool_int(value), nil} # EXPLAIN renders a Real literal at full (round-trippable) precision — unlike # value display / `CAST(x AS TEXT)`, which uses SQLite's `%.15g`. defp explain_literal_opcode({:literal, value}) when is_float(value), do: {"Real", 0, Float.to_string(value)} defp explain_literal_opcode({:literal, {:blob, value}}) when is_binary(value), do: {"Blob", byte_size(value), value} defp explain_literal_opcode({:literal, value}) when is_binary(value), do: {"String8", 0, value} defp explain_literal_opcode({:collate, expr, _name}), do: explain_literal_opcode(expr) defp explain_literal_opcode({:negate, {:literal, value}}) when is_integer(value), do: explain_integer_literal_opcode(-value) defp explain_literal_opcode({:negate, {:literal, value}}) when is_float(value), do: {"Real", 0, Float.to_string(-value)} defp explain_literal_opcode(_expr), do: :error defp explain_zero_integer_opcode?({"Integer", 0, nil}), do: true defp explain_zero_integer_opcode?(_opcode), do: false defp explain_cast_affinity_code(:blob), do: 65 defp explain_cast_affinity_code(:text), do: 66 defp explain_cast_affinity_code(:numeric), do: 67 defp explain_cast_affinity_code(:integer), do: 68 defp explain_cast_affinity_code(:real), do: 69 defp explain_integer_literal_opcode(value) when value >= -2_147_483_647 and value <= 2_147_483_647, do: {"Integer", value, nil} defp explain_integer_literal_opcode(value), do: {"Int64", 0, Value.to_text(value)} defp explain_literal_comment("Null", _p1, _p4, reg), do: "r[#{reg}]=NULL" defp explain_literal_comment("Integer", value, _p4, reg), do: "r[#{reg}]=#{value}" defp explain_literal_comment("Int64", _p1, value, reg), do: "r[#{reg}]=#{value}" defp explain_literal_comment("Real", _p1, value, reg), do: "r[#{reg}]=#{value}" defp explain_literal_comment("String8", _p1, value, reg), do: "r[#{reg}]='#{value}'" defp explain_literal_comment("Blob", size, value, reg), do: "r[#{reg}]=#{value} (len=#{size})" defp explain_limit(nil, nil), do: {:ok, nil} defp explain_limit(limit, nil) do case explain_limit_integer(limit) do {:ok, limit} -> {:ok, {:limit, limit}} :error -> :error end end defp explain_limit(limit_expr, offset_expr) do with {:ok, limit} <- explain_limit_integer(limit_expr), {:ok, offset} <- explain_limit_integer(offset_expr) do {:ok, {:limit_offset, limit, offset}} else _other -> :error end end defp explain_limit_integer({:literal, value}) when is_integer(value), do: {:ok, value} defp explain_limit_integer({:negate, {:literal, value}}) when is_integer(value), do: {:ok, -value} defp explain_limit_integer(_expr), do: :error defp explain_table_projection(table, name, alias_name, columns) do columns |> Enum.reduce_while({:ok, []}, fn :star, {:ok, acc} -> {:cont, {:ok, acc ++ explain_all_table_columns(table)}} {:qualified_star, qualifier}, {:ok, acc} -> if explain_table_qualifier?(qualifier, table, name, alias_name) do {:cont, {:ok, acc ++ explain_all_table_columns(table)}} else {:halt, :error} end {{:column, qualifier, column_name}, _alias}, {:ok, acc} -> if qualifier == nil or explain_table_qualifier?(qualifier, table, name, alias_name) do case explain_table_column(table, column_name) do nil -> {:halt, :error} column -> {:cont, {:ok, acc ++ [column]}} end else {:halt, :error} end _other, _acc -> {:halt, :error} end) end defp explain_all_table_columns(table) do table.columns |> Enum.with_index() |> Enum.map(fn {column, index} -> if Table.key(column.name) == table.rowid_alias do {:rowid, column.name} else {:column, index} end end) end defp explain_table_column(table, column_name) do key = Table.key(column_name) case Enum.find_index(table.columns, &(Table.key(&1.name) == key)) do nil when key in @rowid_names and not table.without_rowid -> {:rowid, table.name} nil -> nil _index when key == table.rowid_alias -> {:rowid, table.name} index -> {:column, index} end end defp explain_table_qualifier?(qualifier, table, name, alias_name) do key = Table.key(qualifier) key in [Table.key(alias_name || name), Table.key(table.name)] end defp explain_table_filter(_table, _name, _alias_name, nil), do: {:ok, nil} defp explain_table_filter(table, name, alias_name, {:binary, :and, _left, _right} = where) do where |> where_conjuncts() |> Enum.reduce_while({:ok, []}, fn term, {:ok, acc} -> case explain_table_filter_term(table, name, alias_name, term) do {:ok, filter} -> {:cont, {:ok, [filter | acc]}} :error -> {:halt, :error} end end) |> case do {:ok, filters} -> explain_combine_filters(Enum.reverse(filters)) :error -> :error end end defp explain_table_filter(table, name, alias_name, where), do: explain_table_filter_term(table, name, alias_name, where) defp explain_table_filter_term(table, name, alias_name, {:binary, op, left, right}) when op in [:eq, :ne, :lt, :le, :gt, :ge] do cond do op == :eq and explain_filter_rowid?(table, name, alias_name, left) -> explain_rowid_filter(right) op == :eq and explain_filter_rowid?(table, name, alias_name, right) -> explain_rowid_filter(left) op in [:lt, :le, :gt, :ge] and explain_filter_rowid?(table, name, alias_name, left) -> explain_rowid_range_filter(op, right) op in [:lt, :le, :gt, :ge] and explain_filter_rowid?(table, name, alias_name, right) -> explain_rowid_range_filter(explain_flip_comparison_op(op), left) column = explain_filter_column(table, name, alias_name, left) -> explain_comparison_filter(column, op, right) column = explain_filter_column(table, name, alias_name, right) -> explain_comparison_filter(column, explain_flip_comparison_op(op), left) true -> :error end end defp explain_table_filter_term(table, name, alias_name, {:binary, :or, _left, _right} = where) do where |> where_disjuncts() |> explain_rowid_or_filter(table, name, alias_name) end defp explain_table_filter_term( table, name, alias_name, {:between, expr, low, high, false} ) do if explain_filter_rowid?(table, name, alias_name, expr) do with {:ok, lower} <- explain_rowid_range_filter(:ge, low), {:ok, upper} <- explain_rowid_range_filter(:le, high) do case explain_merge_rowid_ranges([lower, upper]) do :error -> :error range -> {:ok, range} end end else :error end end defp explain_table_filter_term(table, name, alias_name, {:in, expr, list, false}) when is_list(list) do if explain_filter_rowid?(table, name, alias_name, expr) do explain_rowid_in_filter(list) else :error end end defp explain_table_filter_term(table, name, alias_name, {:is, expr, {:literal, nil}}) do case explain_filter_column(table, name, alias_name, expr) do nil -> :error column -> {:ok, {:null, column, "NotNull"}} end end defp explain_table_filter_term(table, name, alias_name, {:is_not, expr, {:literal, nil}}) do case explain_filter_column(table, name, alias_name, expr) do nil -> :error column -> {:ok, {:null, column, "IsNull"}} end end defp explain_table_filter_term(_table, _name, _alias_name, _where), do: :error defp explain_combine_filters(filters) do {rowid_filters, residual_filters} = Enum.split_with(filters, fn {:rowid_eq, _literal_opcode} -> true {:rowid_range, _lower, _upper} -> true {:rowid_in, _literal_opcodes} -> true _filter -> false end) case explain_combine_rowid_filters(rowid_filters) do {:ok, nil} -> case residual_filters do [] -> {:ok, nil} [filter] -> {:ok, filter} filters -> {:ok, {:filters, filters}} end {:ok, {:rowid_eq, literal_opcode}} -> case residual_filters do [] -> {:ok, {:rowid_eq, literal_opcode}} filters -> {:ok, {:rowid_eq, literal_opcode, filters}} end {:ok, {:rowid_range, lower, upper}} -> case residual_filters do [] -> {:ok, {:rowid_range, lower, upper}} filters -> {:ok, {:rowid_range, lower, upper, filters}} end {:ok, {:rowid_in, literal_opcodes}} -> case residual_filters do [] -> {:ok, {:rowid_in, literal_opcodes}} filters -> {:ok, {:rowid_in, literal_opcodes, filters}} end :error -> :error end end defp explain_combine_rowid_filters([]), do: {:ok, nil} defp explain_combine_rowid_filters([{:rowid_eq, literal_opcode}]), do: {:ok, {:rowid_eq, literal_opcode}} defp explain_combine_rowid_filters([{:rowid_in, literal_opcodes}]), do: {:ok, {:rowid_in, literal_opcodes}} defp explain_combine_rowid_filters(filters) do if Enum.any?( filters, &(match?({:rowid_eq, _literal_opcode}, &1) or match?({:rowid_in, _literal_opcodes}, &1)) ) do :error else case explain_merge_rowid_ranges(filters) do :error -> :error {:rowid_range, nil, nil} -> :error range -> {:ok, range} end end end defp explain_merge_rowid_ranges(filters) do Enum.reduce_while(filters, {:rowid_range, nil, nil}, fn {:rowid_range, next_lower, next_upper}, {:rowid_range, lower, upper} -> cond do next_lower != nil and lower != nil -> {:halt, :error} next_upper != nil and upper != nil -> {:halt, :error} true -> {:cont, {:rowid_range, next_lower || lower, next_upper || upper}} end end) end defp explain_rowid_filter(literal) do case explain_literal_opcode(literal) do :error -> :error opcode -> {:ok, {:rowid_eq, opcode}} end end defp explain_rowid_in_filter([]), do: :error defp explain_rowid_in_filter(list) do list |> Enum.reduce_while([], fn expr, acc -> case explain_literal_opcode(expr) do :error -> {:halt, :error} opcode -> {:cont, [opcode | acc]} end end) |> case do :error -> :error literal_opcodes -> {:ok, {:rowid_in, Enum.reverse(literal_opcodes)}} end end defp explain_rowid_or_filter(disjuncts, table, name, alias_name) do if multiple_terms?(disjuncts) do disjuncts |> Enum.reduce_while([], fn disjunct, acc -> case explain_rowid_or_disjunct(table, name, alias_name, disjunct) do {:ok, literal_opcodes} -> {:cont, acc ++ literal_opcodes} :error -> {:halt, :error} end end) |> case do :error -> :error [] -> :error literal_opcodes -> {:ok, {:rowid_in, literal_opcodes}} end else :error end end defp explain_rowid_or_disjunct(table, name, alias_name, {:binary, :eq, left, right}) do cond do explain_filter_rowid?(table, name, alias_name, left) -> explain_rowid_or_literal(right) explain_filter_rowid?(table, name, alias_name, right) -> explain_rowid_or_literal(left) true -> :error end end defp explain_rowid_or_disjunct(table, name, alias_name, {:in, expr, list, false}) when is_list(list) do if explain_filter_rowid?(table, name, alias_name, expr) do list |> Enum.reduce_while([], fn item, acc -> case explain_rowid_or_literal(item) do {:ok, [literal_opcode]} -> {:cont, acc ++ [literal_opcode]} :error -> {:halt, :error} end end) |> case do :error -> :error [] -> :error literal_opcodes -> {:ok, literal_opcodes} end else :error end end defp explain_rowid_or_disjunct(_table, _name, _alias_name, _disjunct), do: :error defp explain_rowid_or_literal(expr) do case explain_literal_opcode(expr) do :error -> :error literal_opcode -> {:ok, [literal_opcode]} end end defp explain_rowid_range_filter(op, literal) do case explain_literal_opcode(literal) do :error -> :error opcode -> {:ok, explain_rowid_range_bound(op, opcode)} end end defp explain_rowid_range_bound(:gt, literal_opcode), do: {:rowid_range, {"SeekGT", literal_opcode}, nil} defp explain_rowid_range_bound(:ge, literal_opcode), do: {:rowid_range, {"SeekGE", literal_opcode}, nil} defp explain_rowid_range_bound(:lt, literal_opcode), do: {:rowid_range, nil, {"Ge", literal_opcode}} defp explain_rowid_range_bound(:le, literal_opcode), do: {:rowid_range, nil, {"Gt", literal_opcode}} defp explain_comparison_filter(column, op, literal) do case explain_literal_opcode(literal) do :error -> :error opcode -> {:ok, {:compare, column, explain_jump_opcode(op), opcode}} end end defp explain_filter_column(table, name, alias_name, {:collate, expr, _collation}), do: explain_filter_column(table, name, alias_name, expr) defp explain_filter_column(table, name, alias_name, {:column, qualifier, column_name}) do if qualifier == nil or explain_table_qualifier?(qualifier, table, name, alias_name) do case explain_table_column(table, column_name) do {:column, _index} = column -> column _other -> nil end end end defp explain_filter_column(_table, _name, _alias_name, _expr), do: nil defp explain_filter_rowid?(table, name, alias_name, {:collate, expr, _collation}), do: explain_filter_rowid?(table, name, alias_name, expr) defp explain_filter_rowid?(table, name, alias_name, {:column, qualifier, column_name}) do (qualifier == nil or explain_table_qualifier?(qualifier, table, name, alias_name)) and match?({:rowid, _display}, explain_table_column(table, column_name)) end defp explain_filter_rowid?(_table, _name, _alias_name, _expr), do: false defp explain_flip_comparison_op(:lt), do: :gt defp explain_flip_comparison_op(:le), do: :ge defp explain_flip_comparison_op(:gt), do: :lt defp explain_flip_comparison_op(:ge), do: :le defp explain_flip_comparison_op(op), do: op defp explain_jump_opcode(:eq), do: "Ne" defp explain_jump_opcode(:ne), do: "Eq" defp explain_jump_opcode(:lt), do: "Ge" defp explain_jump_opcode(:le), do: "Gt" defp explain_jump_opcode(:gt), do: "Le" defp explain_jump_opcode(:ge), do: "Lt" defp explain_table_scan_program(db, table, projection, filter, limit) do case filter do {:rowid_eq, literal_opcode} -> explain_table_rowid_seek_program(db, table, projection, literal_opcode, [], limit) {:rowid_eq, literal_opcode, residual_filters} -> explain_table_rowid_seek_program( db, table, projection, literal_opcode, residual_filters, limit ) {:rowid_range, lower, upper} -> explain_table_rowid_range_program(db, table, projection, lower, upper, [], limit) {:rowid_range, lower, upper, residual_filters} -> explain_table_rowid_range_program( db, table, projection, lower, upper, residual_filters, limit ) {:rowid_in, literal_opcodes} -> explain_table_rowid_in_program(db, table, projection, literal_opcodes, [], limit) {:rowid_in, literal_opcodes, residual_filters} -> explain_table_rowid_in_program( db, table, projection, literal_opcodes, residual_filters, limit ) _other -> explain_table_scan_loop_program(db, table, projection, filter, limit) end end defp explain_table_scan_loop_program(db, table, projection, filter, limit) do width = length(projection) filter_conditions = explain_filter_conditions(filter) filter_width = explain_filter_width(filter_conditions) const_width = explain_filter_const_width(filter_conditions) setup_width = explain_limit_setup_width(limit) register_width = explain_limit_register_width(limit) offset_width = explain_limit_offset_width(limit) open_addr = 1 + setup_width rewind_addr = open_addr + 1 filter_start_addr = rewind_addr + 1 filter_column_reg = 1 + register_width filter_const_start_reg = filter_column_reg + explain_filter_column_register_width(filter_conditions) output_start = filter_const_start_reg + const_width offset_addr = filter_start_addr + filter_width projection_addr = offset_addr + offset_width result_addr = projection_addr + width decr_addr = result_addr + 1 next_addr = result_addr + 1 + explain_limit_decr_width(limit) halt_addr = result_addr + 2 + explain_limit_decr_width(limit) transaction_addr = halt_addr + 1 start_addr = transaction_addr rootpage = explain_table_rootpage(db, table) open_p4 = explain_table_open_p4(projection ++ explain_filter_projection(filter)) filter_rows = explain_filter_rows( table, filter_conditions, filter_start_addr, filter_column_reg, filter_const_start_reg, next_addr ) projection_rows = projection |> Enum.with_index() |> Enum.map(fn {column, index} -> explain_table_projection_row( table, column, output_start + index, projection_addr + index, 0 ) end) const_rows = explain_filter_const_rows(filter_conditions, transaction_addr + 1, filter_const_start_reg) [[0, "Init", 0, start_addr, 0, nil, 0, "Start at #{start_addr}"]] ++ explain_limit_rows(limit, halt_addr) ++ [ [ open_addr, "OpenRead", 0, rootpage, 0, open_p4, 0, "root=#{rootpage} iDb=0; #{table.name}" ] ] ++ [[rewind_addr, "Rewind", 0, halt_addr, 0, nil, 0, nil]] ++ filter_rows ++ explain_limit_offset_rows(limit, offset_addr, next_addr) ++ projection_rows ++ [ [ result_addr, "ResultRow", output_start, width, 0, nil, 0, explain_result_comment(output_start, width) ] ] ++ explain_limit_decr_rows(limit, decr_addr, halt_addr) ++ [ [next_addr, "Next", 0, filter_start_addr, 0, nil, 1, nil], [halt_addr, "Halt", 0, 0, 0, nil, 0, nil], [transaction_addr, "Transaction", 0, 0, db.schema_version, "0", 1, "usesStmtJournal=0"] ] ++ const_rows ++ [[transaction_addr + const_width + 1, "Goto", 0, 1, 0, nil, 0, nil]] end defp explain_filter_width(conditions), do: length(conditions) * 2 defp explain_filter_column_register_width([]), do: 0 defp explain_filter_column_register_width(_conditions), do: 1 defp explain_filter_register_width([]), do: 0 defp explain_filter_register_width(conditions), do: 1 + explain_filter_const_width(conditions) defp explain_filter_const_width(conditions), do: Enum.count(conditions, &match?({:compare, _column, _jump_opcode, _literal_opcode}, &1)) defp explain_filter_projection({:rowid_eq, _literal_opcode}), do: [] defp explain_filter_projection({:rowid_eq, _literal_opcode, filters}), do: explain_filters_projection(filters) defp explain_filter_projection({:rowid_in, _literal_opcodes}), do: [] defp explain_filter_projection({:rowid_in, _literal_opcodes, filters}), do: explain_filters_projection(filters) defp explain_filter_projection({:compare, column, _jump_opcode, _literal_opcode}), do: [column] defp explain_filter_projection({:null, column, _jump_opcode}), do: [column] defp explain_filter_projection({:filters, filters}), do: explain_filters_projection(filters) defp explain_filter_projection(nil), do: [] defp explain_filters_projection(filters), do: Enum.flat_map(filters, &explain_filter_projection/1) defp explain_filter_conditions(nil), do: [] defp explain_filter_conditions({:rowid_eq, _literal_opcode}), do: [] defp explain_filter_conditions({:rowid_eq, _literal_opcode, filters}), do: filters defp explain_filter_conditions({:rowid_in, _literal_opcodes}), do: [] defp explain_filter_conditions({:rowid_in, _literal_opcodes, filters}), do: filters defp explain_filter_conditions({:filters, filters}), do: filters defp explain_filter_conditions({:compare, _column, _jump_opcode, _literal_opcode} = filter), do: [filter] defp explain_filter_conditions({:null, _column, _jump_opcode} = filter), do: [filter] defp explain_filter_rows(table, conditions, start_addr, column_reg, const_start_reg, next_addr) do {rows, _next_const_reg} = conditions |> Enum.with_index() |> Enum.reduce({[], const_start_reg}, fn {condition, index}, {acc, const_reg} -> addr = start_addr + index * 2 {condition_rows, next_const_reg} = explain_filter_condition_rows(table, condition, addr, column_reg, const_reg, next_addr) {acc ++ condition_rows, next_const_reg} end) rows end defp explain_filter_condition_rows( table, {:compare, column, jump_opcode, _literal_opcode}, addr, column_reg, const_reg, next_addr ) do rows = [ explain_table_projection_row(table, column, column_reg, addr, 0), [ addr + 1, jump_opcode, const_reg, next_addr, column_reg, "BINARY-8", 81, explain_jump_comment(jump_opcode, column_reg, const_reg, next_addr) ] ] {rows, const_reg + 1} end defp explain_filter_condition_rows( table, {:null, column, jump_opcode}, addr, column_reg, const_reg, next_addr ) do rows = [ explain_table_projection_row(table, column, column_reg, addr, 128), [ addr + 1, jump_opcode, column_reg, next_addr, 0, nil, 0, explain_null_jump_comment(jump_opcode, column_reg, next_addr) ] ] {rows, const_reg} end defp explain_filter_const_rows(conditions, addr, const_start_reg) do {rows, _next_addr, _next_const_reg} = Enum.reduce(conditions, {[], addr, const_start_reg}, fn {:compare, _column, _jump_opcode, literal_opcode}, {acc, next_addr, const_reg} -> {opcode, p1, p4} = literal_opcode row = [ next_addr, opcode, p1, const_reg, 0, p4, 0, explain_literal_comment(opcode, p1, p4, const_reg) ] {acc ++ [row], next_addr + 1, const_reg + 1} {:null, _column, _jump_opcode}, acc -> acc end) rows end defp explain_table_rowid_seek_program( db, table, projection, literal_opcode, residual_filters, limit ) do width = length(projection) setup_width = explain_limit_setup_width(limit) register_width = explain_limit_register_width(limit) offset_width = explain_limit_offset_width(limit) filter_width = explain_filter_width(residual_filters) filter_register_width = explain_filter_register_width(residual_filters) const_width = explain_filter_const_width(residual_filters) key_reg = 1 + register_width filter_column_reg = key_reg + 1 filter_const_start_reg = filter_column_reg + explain_filter_column_register_width(residual_filters) output_start = key_reg + 1 + filter_register_width open_addr = 1 + setup_width literal_addr = open_addr + 1 seek_addr = literal_addr + 1 offset_addr = seek_addr + 1 + filter_width projection_addr = offset_addr + offset_width result_addr = projection_addr + width decr_addr = result_addr + 1 halt_addr = result_addr + 1 + explain_limit_decr_width(limit) transaction_addr = halt_addr + 1 rootpage = explain_table_rootpage(db, table) open_p4 = explain_table_open_p4(projection ++ explain_filters_projection(residual_filters)) {literal_op, literal_p1, literal_p4} = literal_opcode filter_rows = explain_filter_rows( table, residual_filters, seek_addr + 1, filter_column_reg, filter_const_start_reg, halt_addr ) projection_rows = projection |> Enum.with_index() |> Enum.map(fn {column, index} -> explain_table_projection_row( table, column, output_start + index, projection_addr + index, 0 ) end) [ [0, "Init", 0, transaction_addr, 0, nil, 0, "Start at #{transaction_addr}"] ] ++ explain_limit_rows(limit, halt_addr) ++ [ [ open_addr, "OpenRead", 0, rootpage, 0, open_p4, 0, "root=#{rootpage} iDb=0; #{table.name}" ], [ literal_addr, literal_op, literal_p1, key_reg, 0, literal_p4, 0, explain_literal_comment(literal_op, literal_p1, literal_p4, key_reg) ], [seek_addr, "SeekRowid", 0, halt_addr, key_reg, nil, 0, "intkey=r[#{key_reg}]"] ] ++ filter_rows ++ explain_limit_offset_rows(limit, offset_addr, halt_addr) ++ projection_rows ++ [ [ result_addr, "ResultRow", output_start, width, 0, nil, 0, explain_result_comment(output_start, width) ] ] ++ explain_limit_decr_rows(limit, decr_addr, halt_addr) ++ [ [halt_addr, "Halt", 0, 0, 0, nil, 0, nil], [transaction_addr, "Transaction", 0, 0, db.schema_version, "0", 1, "usesStmtJournal=0"] ] ++ explain_filter_const_rows(residual_filters, transaction_addr + 1, filter_const_start_reg) ++ [ [transaction_addr + const_width + 1, "Goto", 0, 1, 0, nil, 0, nil] ] end defp explain_table_rowid_in_program( db, table, projection, literal_opcodes, residual_filters, limit ) do width = length(projection) key_count = length(literal_opcodes) setup_width = explain_limit_setup_width(limit) register_width = explain_limit_register_width(limit) offset_width = explain_limit_offset_width(limit) filter_width = explain_filter_width(residual_filters) residual_const_width = explain_filter_const_width(residual_filters) key_start_reg = 1 + register_width filter_column_reg = key_start_reg + key_count filter_const_start_reg = filter_column_reg + explain_filter_column_register_width(residual_filters) output_start = filter_const_start_reg + residual_const_width open_addr = 1 + setup_width seek_start_addr = open_addr + 1 candidate_width = 1 + filter_width + offset_width + width + 1 + explain_limit_decr_width(limit) + 1 halt_addr = seek_start_addr + key_count * candidate_width transaction_addr = halt_addr + 1 rootpage = explain_table_rootpage(db, table) open_p4 = explain_table_open_p4(projection ++ explain_filters_projection(residual_filters)) candidate_rows = literal_opcodes |> Enum.with_index() |> Enum.flat_map(fn {_literal_opcode, index} -> candidate_addr = seek_start_addr + index * candidate_width next_candidate_addr = candidate_addr + candidate_width key_reg = key_start_reg + index offset_addr = candidate_addr + 1 + filter_width projection_addr = offset_addr + offset_width result_addr = projection_addr + width decr_addr = result_addr + 1 loop_addr = result_addr + 1 + explain_limit_decr_width(limit) filter_rows = explain_filter_rows( table, residual_filters, candidate_addr + 1, filter_column_reg, filter_const_start_reg, next_candidate_addr ) projection_rows = projection |> Enum.with_index() |> Enum.map(fn {column, projection_index} -> explain_table_projection_row( table, column, output_start + projection_index, projection_addr + projection_index, 0 ) end) [ [ candidate_addr, "SeekRowid", 0, next_candidate_addr, key_reg, nil, 0, "intkey=r[#{key_reg}]" ] ] ++ filter_rows ++ explain_limit_offset_rows(limit, offset_addr, next_candidate_addr) ++ projection_rows ++ [ [ result_addr, "ResultRow", output_start, width, 0, nil, 0, explain_result_comment(output_start, width) ] ] ++ explain_limit_decr_rows(limit, decr_addr, halt_addr) ++ [[loop_addr, "Goto", 0, next_candidate_addr, 0, nil, 0, nil]] end) literal_rows = literal_opcodes |> Enum.with_index() |> Enum.map(fn {literal_opcode, index} -> {literal_op, literal_p1, literal_p4} = literal_opcode reg = key_start_reg + index [ transaction_addr + 1 + index, literal_op, literal_p1, reg, 0, literal_p4, 0, explain_literal_comment(literal_op, literal_p1, literal_p4, reg) ] end) const_start_addr = transaction_addr + 1 + key_count const_width = key_count + residual_const_width [ [0, "Init", 0, transaction_addr, 0, nil, 0, "Start at #{transaction_addr}"] ] ++ explain_limit_rows(limit, halt_addr) ++ [ [ open_addr, "OpenRead", 0, rootpage, 0, open_p4, 0, "root=#{rootpage} iDb=0; #{table.name}" ] ] ++ candidate_rows ++ [ [halt_addr, "Halt", 0, 0, 0, nil, 0, nil], [transaction_addr, "Transaction", 0, 0, db.schema_version, "0", 1, "usesStmtJournal=0"] ] ++ literal_rows ++ explain_filter_const_rows(residual_filters, const_start_addr, filter_const_start_reg) ++ [ [transaction_addr + const_width + 1, "Goto", 0, 1, 0, nil, 0, nil] ] end defp explain_table_rowid_range_program( db, table, projection, lower, upper, residual_filters, limit ) do width = length(projection) residual_conditions = explain_filter_conditions({:filters, residual_filters}) filter_width = explain_filter_width(residual_conditions) residual_const_width = explain_filter_const_width(residual_conditions) setup_width = explain_limit_setup_width(limit) register_width = explain_limit_register_width(limit) offset_width = explain_limit_offset_width(limit) {lower_reg, next_reg} = explain_optional_register(lower, 1 + register_width) {upper_reg, next_reg} = explain_optional_register(upper, next_reg) {upper_rowid_reg, next_reg} = explain_optional_register(upper, next_reg) {filter_column_reg, next_reg} = explain_optional_register(residual_conditions, next_reg) filter_const_start_reg = next_reg output_start = filter_const_start_reg + residual_const_width open_addr = 1 + setup_width start_addr = open_addr + 1 upper_start_addr = start_addr + 1 upper_width = if upper, do: 3, else: 0 loop_body_addr = upper_start_addr + upper_width offset_addr = loop_body_addr + filter_width projection_addr = offset_addr + offset_width result_addr = projection_addr + width decr_addr = result_addr + 1 next_addr = result_addr + 1 + explain_limit_decr_width(limit) halt_addr = result_addr + 2 + explain_limit_decr_width(limit) transaction_addr = halt_addr + 1 rootpage = explain_table_rootpage(db, table) open_p4 = explain_table_open_p4(projection ++ explain_filters_projection(residual_filters)) start_row = case lower do {seek_opcode, _literal_opcode} -> [start_addr, seek_opcode, 0, halt_addr, lower_reg, nil, 0, "key=r[#{lower_reg}]; pk"] nil -> [start_addr, "Rewind", 0, halt_addr, 0, nil, 0, nil] end {upper_rows, next_target_addr} = explain_rowid_upper_bound_rows( upper, upper_start_addr, upper_reg, upper_rowid_reg, halt_addr ) filter_rows = explain_filter_rows( table, residual_conditions, loop_body_addr, filter_column_reg, filter_const_start_reg, next_addr ) projection_rows = projection |> Enum.with_index() |> Enum.map(fn {column, index} -> explain_table_projection_row( table, column, output_start + index, projection_addr + index, 0 ) end) post_transaction_rows = [] |> explain_rowid_lower_const_row(lower, transaction_addr + 1, lower_reg) |> then(fn {rows, addr} -> rows ++ explain_filter_const_rows(residual_conditions, addr, filter_const_start_reg) end) const_width = if(lower, do: 1, else: 0) + residual_const_width [ [0, "Init", 0, transaction_addr, 0, nil, 0, "Start at #{transaction_addr}"] ] ++ explain_limit_rows(limit, halt_addr) ++ [ [ open_addr, "OpenRead", 0, rootpage, 0, open_p4, 0, "root=#{rootpage} iDb=0; #{table.name}" ], start_row ] ++ upper_rows ++ filter_rows ++ explain_limit_offset_rows(limit, offset_addr, next_addr) ++ projection_rows ++ [ [ result_addr, "ResultRow", output_start, width, 0, nil, 0, explain_result_comment(output_start, width) ] ] ++ explain_limit_decr_rows(limit, decr_addr, halt_addr) ++ [ [next_addr, "Next", 0, next_target_addr, 0, nil, 0, nil], [halt_addr, "Halt", 0, 0, 0, nil, 0, nil], [transaction_addr, "Transaction", 0, 0, db.schema_version, "0", 1, "usesStmtJournal=0"] ] ++ post_transaction_rows ++ [[transaction_addr + const_width + 1, "Goto", 0, 1, 0, nil, 0, nil]] end defp explain_optional_register(nil, next_reg), do: {nil, next_reg} defp explain_optional_register([], next_reg), do: {nil, next_reg} defp explain_optional_register(_value, next_reg), do: {next_reg, next_reg + 1} defp explain_limit_setup_width(nil), do: 0 defp explain_limit_setup_width({:limit, 0}), do: 2 defp explain_limit_setup_width({:limit, _limit}), do: 1 defp explain_limit_setup_width({:limit_offset, 0, _offset}), do: 5 defp explain_limit_setup_width({:limit_offset, _limit, _offset}), do: 4 defp explain_limit_register_width(nil), do: 0 defp explain_limit_register_width({:limit, _limit}), do: 1 defp explain_limit_register_width({:limit_offset, _limit, _offset}), do: 3 defp explain_limit_decr_width(nil), do: 0 defp explain_limit_decr_width(_limit), do: 1 defp explain_limit_offset_width({:limit_offset, _limit, _offset}), do: 1 defp explain_limit_offset_width(_limit), do: 0 defp explain_limit_rows(nil, _halt_addr), do: [] defp explain_limit_rows({:limit, 0}, halt_addr), do: [ [1, "Integer", 0, 1, 0, nil, 0, "r[1]=0; LIMIT counter"], [2, "Goto", 0, halt_addr, 0, nil, 0, nil] ] defp explain_limit_rows({:limit, value}, _halt_addr), do: [[1, "Integer", value, 1, 0, nil, 0, "r[1]=#{value}; LIMIT counter"]] defp explain_limit_rows({:limit_offset, 0, offset}, halt_addr) do [ [1, "Integer", 0, 1, 0, nil, 0, "r[1]=0; LIMIT counter"], [2, "Goto", 0, halt_addr, 0, nil, 0, nil], [3, "Integer", offset, 2, 0, nil, 0, "r[2]=#{offset}"], [4, "MustBeInt", 2, 0, 0, nil, 0, "OFFSET counter"], [ 5, "OffsetLimit", 1, 3, 2, nil, 0, "if r[1]>0 then r[3]=r[1]+max(0,r[2]) else r[3]=(-1); LIMIT+OFFSET" ] ] end defp explain_limit_rows({:limit_offset, limit, offset}, _halt_addr) do [ [1, "Integer", limit, 1, 0, nil, 0, "r[1]=#{limit}; LIMIT counter"], [2, "Integer", offset, 2, 0, nil, 0, "r[2]=#{offset}"], [3, "MustBeInt", 2, 0, 0, nil, 0, "OFFSET counter"], [ 4, "OffsetLimit", 1, 3, 2, nil, 0, "if r[1]>0 then r[3]=r[1]+max(0,r[2]) else r[3]=(-1); LIMIT+OFFSET" ] ] end defp explain_limit_offset_rows(nil, _addr, _jump_addr), do: [] defp explain_limit_offset_rows({:limit, _limit}, _addr, _jump_addr), do: [] defp explain_limit_offset_rows({:limit_offset, _limit, _offset}, addr, jump_addr), do: [ [ addr, "IfPos", 2, jump_addr, 1, nil, 0, "if r[2]>0 then r[2]-=1, goto #{jump_addr}; OFFSET" ] ] defp explain_limit_decr_rows(nil, _addr, _halt_addr), do: [] defp explain_limit_decr_rows(_limit, addr, halt_addr), do: [[addr, "DecrJumpZero", 1, halt_addr, 0, nil, 0, "if (--r[1])==0 goto #{halt_addr}"]] defp explain_rowid_upper_bound_rows(nil, addr, _upper_reg, _rowid_reg, _halt_addr), do: {[], addr} defp explain_rowid_upper_bound_rows( {jump_opcode, literal_opcode}, addr, upper_reg, rowid_reg, halt_addr ) do {literal_op, literal_p1, literal_p4} = literal_opcode rows = [ [ addr, literal_op, literal_p1, upper_reg, 0, literal_p4, 0, explain_literal_comment(literal_op, literal_p1, literal_p4, upper_reg) ], [addr + 1, "Rowid", 0, rowid_reg, 0, nil, 0, "r[#{rowid_reg}]= rowid of 0"], [ addr + 2, jump_opcode, upper_reg, halt_addr, rowid_reg, nil, 83, explain_jump_comment(jump_opcode, rowid_reg, upper_reg, halt_addr) ] ] {rows, addr + 1} end defp explain_rowid_lower_const_row(rows, nil, addr, _lower_reg), do: {rows, addr} defp explain_rowid_lower_const_row(rows, {_seek_opcode, literal_opcode}, addr, lower_reg) do {literal_op, literal_p1, literal_p4} = literal_opcode row = [ addr, literal_op, literal_p1, lower_reg, 0, literal_p4, 0, explain_literal_comment(literal_op, literal_p1, literal_p4, lower_reg) ] {rows ++ [row], addr + 1} end defp explain_jump_comment("Eq", left_reg, right_reg, next_addr), do: "if r[#{left_reg}]==r[#{right_reg}] goto #{next_addr}" defp explain_jump_comment("Ne", left_reg, right_reg, next_addr), do: "if r[#{left_reg}]!=r[#{right_reg}] goto #{next_addr}" defp explain_jump_comment("Lt", left_reg, right_reg, next_addr), do: "if r[#{left_reg}]r[#{right_reg}] goto #{next_addr}" defp explain_jump_comment("Ge", left_reg, right_reg, next_addr), do: "if r[#{left_reg}]>=r[#{right_reg}] goto #{next_addr}" defp explain_null_jump_comment("IsNull", reg, next_addr), do: "if r[#{reg}]==NULL goto #{next_addr}" defp explain_null_jump_comment("NotNull", reg, next_addr), do: "if r[#{reg}]!=NULL goto #{next_addr}" defp explain_table_projection_row(table, {:rowid, _display}, reg, addr, p5), do: [addr, "Rowid", 0, reg, 0, nil, p5, "r[#{reg}]=#{table.name}.rowid"] defp explain_table_projection_row(_table, {:column, index}, reg, addr, p5), do: [ addr, "Column", 0, index, reg, nil, p5, "r[#{reg}]= cursor 0 column #{index}" ] defp explain_result_comment(start, 1), do: "output=r[#{start}]" defp explain_result_comment(start, width), do: "output=r[#{start}..#{start + width - 1}]" defp explain_table_open_p4(projection) do projection |> Enum.flat_map(fn {:column, index} -> [index] {:rowid, _display} -> [] end) |> case do [] -> 0 indices -> Enum.max(indices) + 1 end end defp explain_table_rootpage(db, table) do table_key = Database.table_storage_key(table.schema, table.name) db.tables |> Map.values() |> Enum.filter( &(Database.table_storage_key(&1.schema, "") == Database.table_storage_key(table.schema, "")) ) |> Enum.sort_by(&Table.key(&1.name)) |> Enum.find_index(&(Database.table_storage_key(&1.schema, &1.name) == table_key)) |> case do nil -> 0 index -> index + 1 end end defp internal_sqlite_object_name?(name), do: name |> Table.key() |> String.starts_with?("sqlite_") defp drop_index_owner(db, :any, name), do: ordered_index_owner(db, name) defp drop_index_owner(db, schema, name), do: Database.find_index_owner(db, schema, name) || find_autoindex_owner(db, schema, name) defp exec_insert(db, %Insert{} = stmt) do # Materialize index entries up front so the reduce can keep them current # row-by-row (incremental add + O(1) unique-conflict lookup) instead of a # full rebuild per row. table = db |> fetch_table!(stmt.schema, stmt.table) |> then(&ensure_index_entries(db, &1)) targets = insert_targets(table, stmt) rows = insert_rows(db, stmt, table, targets) on_conflict = stmt.or_conflict || :abort old_rows = table.rows before_triggers = triggers_for(db, stmt, :before, :insert) after_triggers = triggers_for(db, stmt, :after, :insert) {db, table, count, last_insert_rowid, returning_rows, fk_pairs, upserted_rowids} = Enum.reduce( rows, {db, table, 0, nil, [], [], MapSet.new()}, fn row, acc -> insert_row_step(row, acc, stmt, targets, on_conflict, before_triggers, after_triggers) end ) db = db |> insert_put_table( table, count, last_insert_rowid, upserted_rowids, fk_pairs, before_triggers, after_triggers ) |> apply_replace_deleted_actions(table, old_rows, upserted_rowids) |> apply_fk_update_actions(table, Enum.reverse(fk_pairs)) |> Database.record_changes(count, last_insert_rowid) {dml_result(db, table, stmt.returning, Enum.reverse(returning_rows), :insert, count), db} end # The insert reduce keeps index entries current row-by-row (insert adds an # entry; an upsert UPDATE rebuilds), so a plain insert needs no full rebuild # here — making a bulk load of an indexed table O(n) rather than O(n²). When # post-insert work mutates rows the per-row maintenance did not see (REPLACE/FK # cascade deletions), or entries were never materialized, fall back to the full # rebuild so the stored entries are correct. defp insert_put_table(db, table, _count, _rowid, upserted_rowids, fk_pairs, _before, _after) do if MapSet.size(upserted_rowids) == 0 and fk_pairs == [] and indexes_have_entries?(table) do Database.put_table(db, table) else put_table(db, table) end end defp insert_row_step(row, acc, stmt, targets, on_conflict, before_triggers, after_triggers) do {db, table, count, last_insert_rowid, returning_rows, fk_pairs, upserted_rowids} = acc {values, explicit_rowid} = insert_values(targets, row) # Build a candidate row map for CHECK validation candidate = table |> build_candidate_row(values, db) |> with_explicit_rowid(table, explicit_rowid) |> apply_generated_columns(db, table, explicit_rowid) check_strict_types!(table, candidate) {trigger_status, db, table} = if before_triggers == [] do {:ok, db, table} else db = put_table(db, table) {status, db} = fire_triggers( db, before_triggers, table, nil, trigger_row(before_insert_trigger_rowid(table, explicit_rowid), candidate) ) {status, db, refetch_table(db, table)} end if trigger_status == :ignored do {db, table, count, last_insert_rowid, returning_rows, fk_pairs, upserted_rowids} else values = candidate check_env = table_env(db, table, nil, candidate) case check_violations(db, table, candidate, check_env, on_conflict) do :ok -> before_update_triggers = triggers_for(db, stmt, :before, :update) after_update_triggers = triggers_for(db, stmt, :after, :update) case insert_or_upsert( db, table, values, explicit_rowid, stmt, on_conflict, before_update_triggers, after_update_triggers ) do {:inserted, db, table, rowid} -> stored_row = Table.fetch_row!(table, rowid) returning_rows = [{rowid, stored_row} | returning_rows] {db, table} = fire_after_row_triggers( db, table, after_triggers, nil, trigger_row(rowid, stored_row) ) last_insert_rowid = last_insert_rowid_for_table(table, rowid, last_insert_rowid) {db, table, count + 1, last_insert_rowid, returning_rows, fk_pairs, upserted_rowids} {:updated, db, table, rowid, {old_rowid, old_row, returning_row}} -> returning_rows = [{rowid, returning_row} | returning_rows] {db, table, count + 1, last_insert_rowid, returning_rows, [{old_row, returning_row} | fk_pairs], upserted_rowids |> MapSet.put(rowid) |> MapSet.put(old_rowid)} :ignore -> {db, table, count, last_insert_rowid, returning_rows, fk_pairs, upserted_rowids} {:error, message} -> conflict_fail!(db, table, on_conflict, message) end :ignore -> {db, table, count, last_insert_rowid, returning_rows, fk_pairs, upserted_rowids} {:error, message} -> conflict_fail!(db, table, on_conflict, message) end end end defp fire_after_row_triggers(db, table, [], _old_row, _new_row), do: {db, table} defp fire_after_row_triggers(db, table, triggers, old_row, new_row) do db = put_table(db, table) {_status, db} = fire_triggers(db, triggers, table, old_row, new_row) {db, refetch_table(db, table)} end defp trigger_row(rowid, row), do: {:trigger_row, rowid, row} defp before_insert_trigger_rowid(%{without_rowid: true}, _explicit_rowid), do: nil defp before_insert_trigger_rowid(_table, rowid) when is_integer(rowid), do: rowid defp before_insert_trigger_rowid(_table, _rowid), do: -1 defp last_insert_rowid_for_table(%{without_rowid: true}, _rowid, previous), do: previous defp last_insert_rowid_for_table(_table, rowid, _previous), do: rowid # DML against a view runs its INSTEAD OF triggers, once per affected row; # without a matching trigger the view is read-only. defp exec_view_dml(db, view, stmt, event) do triggers = triggers_for(db, view.name, :instead_of, event) if triggers == [] do fail("cannot modify #{view.name} because it is a view") end pseudo = view_pseudo_table(db, view) {db, count, returning_rows} = case event do :insert -> targets = insert_targets(pseudo, stmt) rows = insert_rows(db, stmt, pseudo, targets) Enum.reduce(rows, {db, 0, []}, fn row, {db, count, returning_rows} -> {values, _explicit_rowid} = insert_values(targets, row) candidate = build_candidate_row(pseudo, values, db) {_status, db} = fire_triggers(db, triggers, pseudo, nil, candidate) {db, count + 1, [{count + 1, candidate} | returning_rows]} end) :update -> changed_keys = Enum.map(stmt.assignments, fn {name, _expr} -> Table.key(name) end) Enum.each(stmt.assignments, fn {name, _expr} -> Table.column(pseudo, name) || fail("no such column: #{name}") end) db |> update_target_rows(pseudo, stmt) |> Enum.reduce({db, 0, []}, fn {rowid, row, env}, {db, count, returning_rows} -> new_row = Enum.reduce(stmt.assignments, row, fn {name, expr}, acc -> Map.put(acc, Table.key(name), eval(expr, env)) end) {_status, db} = fire_triggers(db, triggers, pseudo, row, new_row, changed_keys) {db, count + 1, [{rowid, new_row} | returning_rows]} end) :delete -> db |> dml_target_rows(pseudo, stmt) |> Enum.reduce({db, 0, []}, fn {rowid, row}, {db, count, returning_rows} -> {_status, db} = fire_triggers(db, triggers, pseudo, row, nil) {db, count + 1, [{rowid, row} | returning_rows]} end) end db = Database.record_changes(db, count) {dml_result(db, pseudo, stmt.returning, Enum.reverse(returning_rows), event, count), db} end # A view materialized as a throwaway table value, for INSTEAD OF trigger # row iteration and OLD./NEW. construction. defp view_pseudo_table(db, view) do result = query_result(db, view.query, nil) names = view.columns || result.columns columns = Enum.map(names, &%ColumnDef{name: &1, affinity: :blob}) # Rows are stored positionally; `row` is already in `names`/`columns` order. rows = result.rows |> Enum.with_index(1) |> Map.new(fn {row, index} -> {index, List.to_tuple(row)} end) %Table{name: view.name, columns: columns, rows: rows, next_rowid: map_size(rows) + 1} end defp exec_update(db, %Update{} = stmt) do check_window_placement!(stmt.where) table = fetch_table!(db, stmt.schema, stmt.table) assignments = Enum.map(stmt.assignments, &update_assignment(table, &1)) on_conflict = stmt.or_conflict || :abort old_rows = table.rows target_rows = update_target_rows(db, table, stmt) target_rowids = MapSet.new(target_rows, fn {rowid, _row, _env} -> rowid end) changed_keys = Enum.map(assignments, &update_assignment_key/1) before_triggers = triggers_for(db, stmt, :before, :update) after_triggers = triggers_for(db, stmt, :after, :update) # Maintain index entries incrementally (remove the old row's entries, add the # new row's) on the common path instead of rebuilding every index from a full # scan per statement — that `put_table`/`refresh_index_entries` was O(n) per # UPDATE, i.e. O(n²) over a table. REPLACE (deletes conflicting rows mid-loop) # and trigger-bearing updates (re-fetch/refresh mid-loop) fall back to the # full rebuild, where the incremental bookkeeping is too fragile to be worth it. incremental? = on_conflict != :replace and before_triggers == [] and after_triggers == [] table = if incremental?, do: ensure_index_entries(db, table), else: table {db, table, count, returning_rows, fk_pairs} = target_rows |> Enum.reduce({db, table, 0, [], []}, fn {rowid, row, env}, {db, table, count, returning_rows, fk_pairs} -> {new_row, explicit_rowid} = updated_row_and_rowid(table, assignments, row, env) new_row = apply_generated_columns(new_row, db, table, update_rowid_value(explicit_rowid, rowid)) check_strict_types!(table, new_row) {trigger_status, db, table} = if before_triggers == [] do {:ok, db, table} else db = put_table(db, table) {status, db} = fire_triggers( db, before_triggers, table, trigger_row(rowid, row), trigger_row(rowid, new_row), changed_keys ) {status, db, refetch_table(db, table)} end skip_row = {db, table, count, returning_rows, fk_pairs} cond do trigger_status == :ignored -> skip_row # A BEFORE trigger may have deleted the row out from under us. not Map.has_key?(table.rows, rowid) -> skip_row true -> new_row = if before_triggers == [] do new_row else table |> Table.fetch_row!(rowid) |> rebase_updated_row(table, assignments, new_row) |> apply_generated_columns( db, table, update_conflict_rowid(table, rowid, new_row, explicit_rowid) ) end check_strict_types!(table, new_row) check_env = table_env(db, table, rowid, new_row) case check_violations(db, table, new_row, check_env, on_conflict) do :ok -> conflict_rowid = update_conflict_rowid(table, rowid, new_row, explicit_rowid) case resolve_unique_indexes_for_dml( db, table, conflict_rowid, new_row, on_conflict, excluding_rowids: [rowid] ) do {:ok, db, table} -> opts = [on_conflict: stmt.or_conflict] opts = update_rowid_opts(opts, explicit_rowid) case Table.update_row(table, rowid, new_row, opts) do {:ok, table} -> new_rowid = updated_rowid(table, rowid, new_row, explicit_rowid) stored_row = Table.fetch_row!(table, new_rowid) # Incrementally retarget this row's index entries: drop the # old row's, add the new row's. Keeps entries current so the # end-of-statement commit skips the full rebuild. table = if incremental? do table = remove_index_entries(db, table, [{rowid, row}]) add_index_entries(db, table, [new_rowid]) else table end {db, table} = if after_triggers == [] do {db, table} else db = put_table(db, table) {_status, db} = fire_triggers( db, after_triggers, table, trigger_row(rowid, row), trigger_row(new_rowid, stored_row), changed_keys ) {db, refetch_table(db, table)} end returning_rows = [{new_rowid, stored_row} | returning_rows] {db, table, count + 1, returning_rows, [{row, stored_row} | fk_pairs]} :ignore -> skip_row {:error, message} -> conflict_fail!(db, table, on_conflict, message) end :ignore -> skip_row {:error, message} -> conflict_fail!(db, table, on_conflict, message) end :ignore -> skip_row {:error, message} -> conflict_fail!(db, table, on_conflict, message) end end end) # Incremental path kept entries current → store without the full refresh; # otherwise `put_table` rebuilds them. db = if incremental?, do: Database.put_table(db, table), else: put_table(db, table) db = db |> apply_replace_deleted_actions(table, old_rows, target_rowids) |> apply_fk_update_actions(table, Enum.reverse(fk_pairs)) |> Database.record_changes(count) {dml_result(db, table, stmt.returning, Enum.reverse(returning_rows), :update, count), db} end defp exec_delete(db, %Delete{} = stmt) do check_window_placement!(stmt.where) table = fetch_table!(db, stmt.schema, stmt.table) before_triggers = triggers_for(db, stmt, :before, :delete) after_triggers = triggers_for(db, stmt, :after, :delete) returning_rows = dml_target_rows(db, table, stmt) {db, table, deleted_rows, deleted_pairs} = if before_triggers == [] and after_triggers == [] do rowids = Enum.map(returning_rows, &elem(&1, 0)) # `deleted_pairs` (the deleted `{rowid, row}`s) lets the index entries be # maintained incrementally below, instead of rebuilt by a full scan. {db, Table.delete_rows(table, rowids), Enum.map(returning_rows, &elem(&1, 1)), returning_rows} else {db, table, deleted} = Enum.reduce(returning_rows, {db, table, []}, fn {rowid, row}, {db, table, deleted} -> # A trigger fired for an earlier row may have deleted this one. if Map.has_key?(table.rows, rowid) do db = put_table(db, table) {status, db} = fire_triggers(db, before_triggers, table, trigger_row(rowid, row), nil) table = refetch_table(db, table) if status == :ignored or not Map.has_key?(table.rows, rowid) do {db, table, deleted} else table = Table.delete_rows(table, [rowid]) {db, table} = fire_after_row_triggers(db, table, after_triggers, trigger_row(rowid, row), nil) {db, table, [row | deleted]} end else {db, table, deleted} end end) # Triggers can delete arbitrary rows mid-loop, so fall back to a rebuild. {db, table, Enum.reverse(deleted), nil} end count = length(deleted_rows) db_after_put = if deleted_pairs do Database.put_table(db, remove_index_entries(db, table, deleted_pairs)) else put_table(db, table) end db = db_after_put |> apply_fk_delete_actions(table, deleted_rows) |> Database.record_changes(count) {dml_result(db, table, stmt.returning, returning_rows, :delete, count), db} end defp exec_sqlite_sequence_update(db, %Update{} = stmt) do ensure_sqlite_sequence_exists!(db) sequence_table = sqlite_sequence_table(db) assignments = Enum.map(stmt.assignments, fn {name, expr} -> column = Table.column(sequence_table, name) || fail("no such column: #{name}") if Table.key(column.name) == "name" do fail("cannot UPDATE sqlite_sequence.name") end {column, expr} end) {db, count, returning_rows} = db |> update_target_rows(sequence_table, stmt) |> Enum.reduce({db, 0, []}, fn {rowid, row, env}, {db, count, returning_rows} -> new_row = Enum.reduce(assignments, row, fn {column, expr}, new_row -> value = expr |> eval(env) |> Value.apply_affinity(column.affinity) unless is_integer(value) do fail("datatype mismatch") end Map.put(new_row, Table.key(column.name), value) end) db = put_sqlite_sequence(db, Map.fetch!(row, "name"), Map.fetch!(new_row, "seq"), true) {db, count + 1, [{rowid, new_row} | returning_rows]} end) result_table = sqlite_sequence_table(db) db = Database.record_changes(db, count) {dml_result(db, result_table, stmt.returning, Enum.reverse(returning_rows), :update, count), db} end defp exec_sqlite_sequence_delete(db, %Delete{} = stmt) do ensure_sqlite_sequence_exists!(db) sequence_table = sqlite_sequence_table(db) returning_rows = dml_target_rows(db, sequence_table, stmt) db = Enum.reduce(returning_rows, db, fn {_rowid, row}, db -> put_sqlite_sequence(db, Map.fetch!(row, "name"), 0, false) end) count = length(returning_rows) db = Database.record_changes(db, count) {dml_result(db, sequence_table, stmt.returning, returning_rows, :delete, count), db} end defp exec_sqlite_sequence_insert(db, %Insert{} = stmt) do ensure_sqlite_sequence_exists!(db) sequence_table = sqlite_sequence_table(db) targets = insert_targets(sequence_table, stmt) rows = insert_rows(db, stmt, sequence_table, targets) {db, count, returning_rows} = rows |> Enum.reduce({db, 0, []}, fn row, {db, count, returning_rows} -> {values, _explicit_rowid} = insert_values(targets, row) new_row = sequence_table |> build_candidate_row(values, db) |> Map.update!("name", &Value.apply_affinity(&1, :text)) |> Map.update!("seq", &Value.apply_affinity(&1, :integer)) unless is_integer(Map.fetch!(new_row, "seq")) do fail("datatype mismatch") end db = put_sqlite_sequence(db, Map.fetch!(new_row, "name"), Map.fetch!(new_row, "seq"), true) {db, count + 1, [{count + 1, new_row} | returning_rows]} end) result_table = sqlite_sequence_table(db) db = Database.record_changes(db, count) {dml_result(db, result_table, stmt.returning, Enum.reverse(returning_rows), :insert, count), db} end defp table_info_rows(table, include_hidden) do pk_positions = primary_key_positions(table) table.columns |> Enum.reject(&(not include_hidden and &1.generated)) |> Enum.with_index() |> Enum.map(fn {column, index} -> key = Table.key(column.name) row = [ index, column.name, column.declared_type || "", if(column.not_null, do: 1, else: 0), pragma_default(column.default), Map.get(pk_positions, key, 0) ] if include_hidden, do: row ++ [generated_hidden(column)], else: row end) end defp generated_hidden(%{generated: {:virtual, _}}), do: 2 defp generated_hidden(%{generated: {:stored, _}}), do: 3 defp generated_hidden(_column), do: 0 defp foreign_key_list_rows(table) do table |> foreign_key_specs() |> Enum.with_index() |> Enum.flat_map(fn {spec, id} -> Enum.with_index(spec.child_keys) |> Enum.map(fn {child_key, seq} -> [ id, seq, spec.parent_table, display_column_name(table, child_key), Enum.at(spec.parent_keys, seq), fk_action_name(spec.on_update), fk_action_name(spec.on_delete), "NONE" ] end) end) end defp foreign_key_check_rows(db, nil) do db.tables |> Map.values() |> Enum.filter(&main_schema?(&1.schema)) |> Enum.sort_by(&Table.key(&1.name)) |> Enum.flat_map(&foreign_key_check_table_rows(db, &1)) end defp foreign_key_check_rows(db, {:schema, schema, nil}) do ensure_schema_exists!(db, schema) db.tables |> Map.values() |> Enum.filter(&(Table.key(&1.schema || "main") == Table.key(schema))) |> Enum.sort_by(&Table.key(&1.name)) |> Enum.flat_map(&foreign_key_check_table_rows(db, &1)) end defp foreign_key_check_rows(db, {:schema, schema, table_name}) do ensure_schema_exists!(db, schema) case Map.fetch(db.tables, Database.table_storage_key(schema, table_name)) do {:ok, table} -> foreign_key_check_table_rows(db, table) :error -> fail("no such table: #{table_name}") end end defp foreign_key_check_rows(db, table_name) do case pragma_fetch_table(db, table_name) do {:ok, table} -> foreign_key_check_table_rows(db, table) :error -> fail("no such table: #{table_name}") end end defp foreign_key_check_table_rows(db, child_table) do child_table |> foreign_key_specs() |> Enum.with_index() |> Enum.flat_map(fn {spec, id} -> {parent_table, parent_columns} = foreign_key_check_parent(db, child_table, spec) child_table |> Table.scan() |> Enum.flat_map(fn {rowid, row} -> child_values = Enum.map(spec.child_keys, &Map.get(row, &1)) cond do Enum.any?(child_values, &is_nil/1) -> [] parent_table == nil -> [[child_table.name, rowid, spec.parent_table, id]] parent_row_exists?(parent_table, parent_columns, child_values) -> [] true -> [[child_table.name, rowid, parent_table.name, id]] end end) end) end defp foreign_key_check_parent(db, child_table, spec) do case fetch_fk_parent_table(db, child_table, spec.parent_table) do {:ok, parent_table} -> {parent_table, referenced_columns!(child_table, spec, parent_table)} {:error, _message} -> {nil, []} end end defp primary_key_positions(%{composite_keys: [{_name, keys} | _]}) do keys |> Enum.with_index(1) |> Map.new() end defp primary_key_positions(table) do table.columns |> Enum.filter(& &1.primary_key) |> Enum.map(&Table.key(&1.name)) |> Enum.with_index(1) |> Map.new() end defp pragma_default(nil), do: nil defp pragma_default({:literal, nil}), do: "NULL" defp pragma_default({:literal, value}) when is_binary(value), do: "'#{String.replace(value, "'", "''")}'" defp pragma_default({:literal, value}), do: Value.to_text(value) defp pragma_default({:negate, {:literal, value}}), do: "-" <> Value.to_text(value) defp pragma_default({:column, nil, word}), do: word # An expression default (function call, etc.): render it parenthesized, the # form SQLite stores in `sqlite_master`/`PRAGMA table_info` and re-parses. defp pragma_default(expr), do: "(#{expr_name(expr)})" defp index_list_rows(table) do (table.indexes ++ Enum.reverse(table.autoindexes)) |> Enum.with_index() |> Enum.map(fn {index, seq} -> [ seq, index.name, if(index.unique, do: 1, else: 0), Map.get(index, :origin, "c"), if(index.where, do: 1, else: 0) ] end) end defp put_autoindexes(table, constraints) do %{table | autoindexes: autoindexes_for_table(table, constraints)} end defp autoindexes_for_table(table, constraints) do inline_specs = table.columns |> Enum.flat_map(fn column -> key = Table.key(column.name) cond do column.primary_key and key != table.rowid_alias -> [{:pk, [key]}] column.unique -> [{:u, [key]}] true -> [] end end) table_specs = Enum.flat_map(constraints, fn {:primary_key, _name, keys} -> if keys == [table.rowid_alias], do: [], else: [{:pk, keys}] {:unique, _name, keys} -> [{:u, keys}] _constraint -> [] end) table.name |> autoindex_names(inline_specs ++ table_specs) |> Enum.map(fn {name, origin, keys} -> autoindex(table, name, origin, keys) end) end defp autoindex_names(table_name, specs) do specs |> Enum.with_index(1) |> Enum.map(fn {{origin, keys}, seq} -> {"sqlite_autoindex_#{table_name}_#{seq}", origin, keys} end) end defp autoindex(table, name, origin, keys) do %{ name: name, columns: keys, members: Enum.map(keys, &{:column, &1}), collations: Enum.map(keys, &column_collation_name(table, &1)), directions: List.duplicate(:asc, length(keys)), unique: true, where: nil, origin: Atom.to_string(origin), autoindex: true } end defp rename_autoindexes(table) do %{table | autoindexes: autoindexes_with_table_name(table.autoindexes, table.name)} end defp autoindexes_with_table_name(autoindexes, table_name) do autoindexes |> Enum.with_index(1) |> Enum.map(fn {index, seq} -> %{index | name: "sqlite_autoindex_#{table_name}_#{seq}"} end) end defp rename_index_column(index, old_key, new_key) do columns = Enum.map(index.columns, &if(&1 == old_key, do: new_key, else: &1)) members = Enum.map(index_members(index), fn {:column, ^old_key} -> {:column, new_key} member -> member end) %{index | columns: columns, members: members} end defp drop_from_autoindexes(autoindexes, col_key) do autoindexes |> Enum.map(fn index -> columns = Enum.reject(index.columns, &(&1 == col_key)) members = Enum.reject(index_members(index), &(&1 == {:column, col_key})) %{index | columns: columns, members: members} end) |> Enum.reject(&(&1.columns == [])) end defp index_info_rows(table, index) do index |> index_members() |> Enum.with_index() |> Enum.map(fn {member, seqno} -> index_info_row(table, member, seqno) end) end defp index_xinfo_rows(table, index) do member_rows = index |> index_members() |> Enum.with_index() |> Enum.map(fn {member, seqno} -> [seqno, cid, name] = index_info_row(table, member, seqno) direction = Enum.at(Map.get(index, :directions, []), seqno, :asc) collation = Enum.at(Map.get(index, :collations, []), seqno) || :binary [seqno, cid, name, if(direction == :desc, do: 1, else: 0), pragma_collation(collation), 1] end) member_rows ++ [[length(member_rows), -1, nil, 0, "BINARY", 0]] end defp index_info_row(table, {:column, column_key}, seqno) do [seqno, column_position(table, column_key), display_column_name(table, column_key)] end defp index_info_row(_table, {:expr, _expr}, seqno) do [seqno, -2, nil] end defp column_position(table, column_key) do Enum.find_index(table.columns, &(Table.key(&1.name) == column_key)) || -1 end defp pragma_collation(:binary), do: "BINARY" defp pragma_collation(collation), do: collation |> to_string() |> String.upcase() defp display_column_name(table, column_key) do case Enum.find(table.columns, &(Table.key(&1.name) == column_key)) do nil -> column_key column -> column.name end end defp collation_list_rows(db) do custom = db.collations |> Map.values() |> Enum.map(&pragma_collation(&1.name)) |> Enum.reject(&(&1 in ["BINARY", "NOCASE", "RTRIM"])) |> Enum.sort() ["BINARY", "NOCASE", "RTRIM"] ++ custom end defp function_list_rows(db) do scalar_rows = @scalar_arity |> Enum.map(fn {name, arity} -> function_list_row(name, 1, "s", function_narg(arity)) end) aggregate_rows = @aggregate_functions |> Enum.flat_map(fn name -> Enum.map(aggregate_nargs(name), &function_list_row(name, 1, "w", &1)) end) window_rows = @window_functions |> Enum.flat_map(fn name -> Enum.map(window_nargs(name), &function_list_row(name, 1, "w", &1)) end) custom_scalar_rows = db.scalar_functions |> Map.values() |> Enum.map(&function_list_row(&1.name, 0, "s", &1.arity)) custom_aggregate_rows = db.aggregate_functions |> Map.values() |> Enum.map(&function_list_row(&1.name, 0, "w", &1.arity)) (scalar_rows ++ aggregate_rows ++ window_rows ++ custom_scalar_rows ++ custom_aggregate_rows) |> Enum.sort_by(fn [name, builtin, type, _enc, narg, _flags] -> {name, builtin, type, narg} end) end defp function_list_row(name, builtin, type, narg), do: [name, builtin, type, "utf8", narg, 0] defp function_narg(%Range{first: first, last: last}) when first == last, do: first defp function_narg(%Range{}), do: -1 defp aggregate_nargs("count"), do: [0, 1] defp aggregate_nargs(name) when name in ["group_concat", "string_agg"], do: [1, 2] defp aggregate_nargs(name) when name in ["json_group_object", "jsonb_group_object"], do: [2] defp aggregate_nargs(_name), do: [1] defp window_nargs(name) when name in ["row_number", "rank", "dense_rank", "percent_rank", "cume_dist"], do: [0] defp window_nargs("ntile"), do: [1] defp window_nargs(name) when name in ["lag", "lead"], do: [1, 2, 3] defp window_nargs(name) when name in ["first_value", "last_value"], do: [1] defp window_nargs("nth_value"), do: [2] defp pragma_page_count(db, schema) do tables = db.tables |> Map.values() |> Enum.filter(&(Table.key(&1.schema || "main") == Table.key(schema || "main"))) if tables == [] do 0 else table_pages = length(tables) index_pages = tables |> Enum.map(&length(&1.indexes)) |> Enum.sum() max(2, 1 + table_pages + index_pages) end end defp database_list_rows(db) do [[0, "main", ""]] ++ Enum.map(db.attached_databases, fn attached -> [attached.seq, attached.name, attached.file] end) end defp attached_database_key(name), do: String.downcase(name) defp attached_database?(db, key) do Enum.any?(db.attached_databases, fn attached -> attached_database_key(attached.name) == key end) end defp next_attached_database_seq(db) do used = MapSet.new(Enum.map(db.attached_databases, & &1.seq)) Enum.find(2..125, &(not MapSet.member?(used, &1))) end defp ensure_schema_exists!(_db, schema) when schema in [nil, "main", "temp"], do: :ok defp ensure_schema_exists!(db, schema) do unless attached_database?(db, attached_database_key(schema)) do fail("unknown database #{schema}") end end # When resolving a `schema.table` reference in FROM, SQLite reports an unknown # schema as a missing table (`no such table: schema.table`), not "unknown # database" — that wording is reserved for DDL/ATTACH-level operations. defp ensure_table_schema!(_db, schema, _name) when schema in [nil, "main", "temp"], do: :ok defp ensure_table_schema!(db, schema, name) do unless attached_database?(db, attached_database_key(schema)) do fail("no such table: #{schema}.#{name}") end end defp main_schema?(schema), do: schema in [nil, "main"] defp temp_schema?(schema), do: schema == "temp" defp trigger_schema(schema) when schema in [nil, "main"], do: nil defp trigger_schema(schema), do: schema defp ensure_trigger_schema_exists!(_db, "temp"), do: :ok defp ensure_trigger_schema_exists!(db, schema), do: ensure_schema_exists!(db, schema) defp trigger_target_schema!(db, %CreateTrigger{} = stmt) do schema = trigger_schema(stmt.schema) cond do temp_schema?(stmt.schema) and stmt.table_schema == nil -> case trigger_target_lookup_schema(db, stmt.table) do {:ok, schema} -> schema :error -> nil end temp_schema?(stmt.schema) -> ensure_schema_exists!(db, stmt.table_schema) stmt.table_schema stmt.table_schema != nil and trigger_schema(stmt.table_schema) != schema -> fail("trigger #{stmt.name} cannot reference objects in database #{stmt.table_schema}") true -> ensure_schema_exists!(db, schema) schema end end defp trigger_target_lookup_schema(db, table_name) do Enum.find_value(table_lookup_order(db), :error, fn schema -> key = Database.table_storage_key(schema, table_name) if Map.has_key?(db.tables, key) or Map.has_key?(db.views, key) do {:ok, schema} else nil end end) end defp trigger_target_label(%CreateTrigger{schema: "temp", table_schema: nil, table: table}, nil), do: table defp trigger_target_label(%CreateTrigger{table: table}, schema), do: "#{schema || "main"}.#{table}" defp attach_filename(":memory:"), do: "" defp attach_filename(nil), do: "" defp attach_filename({:blob, blob}), do: blob defp attach_filename(value), do: Value.to_text(value) defp qualify_view_query(query, nil), do: qualify_view_query(query, "main") defp qualify_view_query(%Select{} = query, schema) do %{query | from: qualify_view_source(query.from, schema)} end defp qualify_view_query(%Compound{} = query, schema) do %{ query | left: qualify_view_query(query.left, schema), right: qualify_view_query(query.right, schema) } end defp qualify_view_query(query, _schema), do: query defp qualify_view_source(nil, _schema), do: nil defp qualify_view_source({:table, {:schema, _source_schema, _name}, _alias} = source, _schema), do: source defp qualify_view_source({:table, name, alias_name}, schema), do: {:table, {:schema, schema, name}, alias_name} defp qualify_view_source({:subquery, query, alias_name}, schema), do: {:subquery, qualify_view_query(query, schema), alias_name} defp qualify_view_source({:join, type, left, right, constraint}, schema), do: {:join, type, qualify_view_source(left, schema), qualify_view_source(right, schema), constraint} defp qualify_trigger_statement(stmt, "temp"), do: stmt defp qualify_trigger_statement(stmt, nil), do: qualify_trigger_statement(stmt, "main") defp qualify_trigger_statement(%Insert{schema: nil, source: {:select, query}} = stmt, schema), do: %{stmt | schema: schema, source: {:select, qualify_view_query(query, schema)}} defp qualify_trigger_statement(%Insert{source: {:select, query}} = stmt, schema), do: %{stmt | source: {:select, qualify_view_query(query, schema)}} defp qualify_trigger_statement(%Insert{schema: nil} = stmt, schema), do: %{stmt | schema: schema} defp qualify_trigger_statement(%Update{schema: nil, from: from} = stmt, schema), do: %{stmt | schema: schema, from: qualify_view_source(from, schema)} defp qualify_trigger_statement(%Update{from: from} = stmt, schema), do: %{stmt | from: qualify_view_source(from, schema)} defp qualify_trigger_statement(%Delete{schema: nil} = stmt, schema), do: %{stmt | schema: schema} defp qualify_trigger_statement(%Select{} = stmt, schema), do: qualify_view_query(stmt, schema) defp qualify_trigger_statement(%Compound{} = stmt, schema), do: qualify_view_query(stmt, schema) defp qualify_trigger_statement(stmt, _schema), do: stmt defp integrity_check_rows(%{ignore_check_constraints: true}, _arg), do: [] defp integrity_check_rows(db, {:schema, schema, arg}) do ensure_schema_exists!(db, schema) integrity_check_rows(db, arg, schema) end defp integrity_check_rows(db, arg), do: integrity_check_rows(db, arg, :all) defp integrity_check_rows(db, nil, scope), do: integrity_check_scope_rows(db, scope, nil) defp integrity_check_rows(db, arg, scope) when is_integer(arg), do: integrity_check_scope_rows(db, scope, integrity_check_limit(arg)) defp integrity_check_rows(db, table_name, scope) when is_binary(table_name) do case integrity_check_target(db, scope, table_name) do {:table, table} -> integrity_check_table_rows(db, table) :view -> [] :error -> fail("no such table: #{table_name}") end end defp integrity_check_scope_rows(db, scope, limit) do db.tables |> Map.values() |> Enum.filter(&integrity_check_scope_match?(&1.schema, scope)) |> Enum.sort_by(fn table -> {Table.key(table.schema || "main"), Table.key(table.name)} end) |> Enum.flat_map(&integrity_check_table_rows(db, &1)) |> limit_integrity_check_rows(limit) end defp integrity_check_table_rows(db, table) do table |> Table.scan() |> Enum.flat_map(fn {rowid, row} -> env = table_env(db, table, rowid, row) if Enum.any?(table.checks, fn {_name, expr} -> truth(expr, env) == false end) do [["CHECK constraint failed in #{table.name}"]] else [] end end) end defp integrity_check_target(db, :all, table_name) do Enum.find_value(table_lookup_order(db), :error, fn schema -> integrity_check_target(db, schema, table_name, :soft) end) end defp integrity_check_target(db, schema, table_name), do: integrity_check_target(db, schema, table_name, :strict) defp integrity_check_target(db, schema, table_name, mode) do key = Database.table_storage_key(schema, table_name) cond do Map.has_key?(db.tables, key) -> {:table, Map.fetch!(db.tables, key)} Map.has_key?(db.views, key) -> :view mode == :soft -> nil true -> :error end end defp integrity_check_scope_match?(_schema, :all), do: true defp integrity_check_scope_match?(schema, wanted), do: schema_matches?(schema, wanted) defp integrity_check_limit(0), do: nil defp integrity_check_limit(limit) when limit < 0, do: 1 defp integrity_check_limit(limit), do: limit defp limit_integrity_check_rows(rows, nil), do: rows defp limit_integrity_check_rows(rows, limit), do: Enum.take(rows, limit) defp schema_matches?(schema, wanted), do: Table.key(schema || "main") == Table.key(wanted) defp table_list_rows(db) do table_rows = db.tables |> Map.values() |> Enum.sort_by(&table_list_sort_key/1) |> Enum.map(fn table -> [ table.schema || "main", table.name, "table", length(table.columns), bool_int(table.without_rowid), bool_int(table.strict) ] end) sequence_rows = if sqlite_sequence_exists?(db) do [["main", "sqlite_sequence", "table", 2, 0, 0]] else [] end view_rows = db.views |> Map.values() |> Enum.sort_by(&table_list_sort_key/1) |> Enum.map(fn view -> [view.schema || "main", view.name, "view", view_column_count(db, view), 0, 0] end) table_rows ++ sequence_rows ++ view_rows ++ schema_table_list_rows(db) end defp schema_table_list_rows(db) do attached_rows = Enum.map(db.attached_databases, fn attached -> [attached.name, "sqlite_schema", "table", 5, 0, 0] end) [ ["main", "sqlite_schema", "table", 5, 0, 0], ["temp", "sqlite_temp_schema", "table", 5, 0, 0] ] ++ attached_rows end defp table_list_sort_key(table), do: {schema_sort_rank(table.schema), Table.key(table.name)} defp schema_sort_rank(schema) when schema in [nil, "main"], do: {0, "main"} defp schema_sort_rank("temp"), do: {1, "temp"} defp schema_sort_rank(schema), do: {2, Table.key(schema)} defp pragma_fetch_table(db, {:schema, schema, name}) do ensure_schema_exists!(db, schema) case Map.fetch(db.tables, Database.table_storage_key(schema, name)) do {:ok, table} -> {:ok, table} :error -> :error end end defp pragma_fetch_table(db, name) do Enum.find_value(table_lookup_order(db), :error, fn schema -> case Map.fetch(db.tables, Database.table_storage_key(schema, name)) do {:ok, table} -> {:ok, table} :error -> nil end end) end defp pragma_find_index_owner(db, {:schema, schema, name}) do ensure_schema_exists!(db, schema) Database.find_index_owner(db, schema, name) || find_autoindex_owner(db, schema, name) end defp pragma_find_index_owner(db, name), do: ordered_index_owner(db, name) defp find_autoindex_owner(db, schema, index_name) do key = Table.key(index_name) Enum.find_value(db.tables, fn {_table_key, table} -> if index_schema_matches?(table.schema, schema) do case Enum.find(table.autoindexes, &(Table.key(&1.name) == key)) do nil -> nil index -> {table, index} end else nil end end) end defp index_schema_matches?(_table_schema, :any), do: true defp index_schema_matches?(nil, nil), do: true defp index_schema_matches?(nil, "main"), do: true defp index_schema_matches?("main", nil), do: true defp index_schema_matches?(table_schema, schema), do: Table.key(table_schema || "main") == Table.key(schema || "main") defp validate_analyze_target!(_db, nil), do: :ok defp validate_analyze_target!(db, name) do {schema, target} = operational_target(name) ensure_schema_exists!(db, schema) with :error <- operational_table_lookup(db, schema, target), nil <- operational_index_lookup(db, schema, target) do fail("no such table: #{target}") else _ -> :ok end end defp validate_reindex_target!(_db, nil), do: :ok defp validate_reindex_target!(db, name) do {schema, target} = operational_target(name) ensure_schema_exists!(db, schema) with :error <- operational_table_lookup(db, schema, target), nil <- operational_index_lookup(db, schema, target) do fail("unable to identify the object to be reindexed") else _ -> :ok end end defp operational_target(name) do case String.split(name, ".", parts: 2) do [target] -> {nil, target} [schema, target] -> {schema, target} end end defp operational_table_lookup(db, nil, name), do: pragma_fetch_table(db, name) defp operational_table_lookup(db, schema, name) do case Map.fetch(db.tables, Database.table_storage_key(schema, name)) do {:ok, table} -> {:ok, table} :error -> :error end end defp operational_index_lookup(db, nil, name), do: ordered_index_owner(db, name) defp operational_index_lookup(db, schema, name), do: Database.find_index_owner(db, schema, name) || find_autoindex_owner(db, schema, name) defp table_lookup_order(db), do: ["temp", nil] ++ Enum.map(db.attached_databases, & &1.name) defp ordered_index_owner(db, name) do Enum.find_value(table_lookup_order(db), fn schema -> Database.find_index_owner(db, schema, name) || find_autoindex_owner(db, schema, name) end) end defp drop_object_schema(_db, schema, _name) when schema != nil, do: schema defp drop_object_schema(db, nil, name) do case Enum.find_value(table_lookup_order(db), fn schema -> key = Database.table_storage_key(schema, name) if Map.has_key?(db.tables, key) or Map.has_key?(db.views, key) do {:ok, schema} end end) do {:ok, schema} -> schema nil -> nil end end defp filter_pragma_table_list(rows, _db, nil), do: rows defp filter_pragma_table_list(rows, db, {:schema, schema, name}) do ensure_schema_exists!(db, schema) rows |> Enum.filter(fn [row_schema, _row_name, _type, _ncol, _wr, _strict] -> row_schema == schema end) |> filter_pragma_table_list(db, name) end defp filter_pragma_table_list(rows, _db, name) do key = Table.key(name) Enum.filter(rows, fn [_schema, row_name, _type, _ncol, _wr, _strict] -> Table.key(row_name) == key end) end defp view_column_count(_db, %{columns: columns}) when is_list(columns), do: length(columns) defp view_column_count(db, view) do db |> query_result(view.query, nil) |> then(&length(&1.columns)) end defp bool_int(true), do: 1 defp bool_int(false), do: 0 defp pragma_enabled?(value) when is_integer(value), do: value != 0 defp pragma_enabled?(value) when is_binary(value) do value |> String.downcase() |> then(&(&1 in ["1", "on", "true", "yes"])) end defp pragma_enabled?({:literal, value}), do: pragma_enabled?(value) defp pragma_enabled?(_value), do: false defp pragma_auto_vacuum(value, current) when is_integer(value) do case value do 1 -> 1 2 -> 2 0 -> if(current == 0, do: 0, else: current) _other -> current end end defp pragma_auto_vacuum(value, current) when is_binary(value) do case String.downcase(value) do "full" -> 1 "incremental" -> 2 "none" -> if(current == 0, do: 0, else: current) "off" -> if(current == 0, do: 0, else: current) "false" -> if(current == 0, do: 0, else: current) "0" -> if(current == 0, do: 0, else: current) "1" -> 1 "2" -> 2 _other -> current end end defp pragma_auto_vacuum({:literal, value}, current), do: pragma_auto_vacuum(value, current) defp pragma_auto_vacuum(_value, current), do: current defp pragma_header_field("schema_version"), do: :schema_version defp pragma_header_field("user_version"), do: :user_version defp pragma_header_field("application_id"), do: :application_id defp pragma_header_value(value) when is_integer(value) and value in -2_147_483_648..2_147_483_647, do: value defp pragma_header_value(_value), do: 0 @valid_page_sizes MapSet.new([512, 1024, 2048, 4096, 8192, 16_384, 32_768, 65_536]) defp pragma_page_size(value) when is_integer(value) do if MapSet.member?(@valid_page_sizes, value), do: value end defp pragma_page_size(_value), do: nil defp pragma_cache_size(value) when is_integer(value), do: value defp pragma_cache_size(_value), do: 0 defp pragma_default_cache_size(value) when is_integer(value), do: abs(value) defp pragma_default_cache_size(_value), do: 2_000 defp pragma_cache_spill(value, _current) when is_integer(value) and value <= 0, do: 0 defp pragma_cache_spill(value, current) when is_integer(value), do: max(value, min(current, 2_000)) defp pragma_cache_spill(value, _current) when is_binary(value) do case String.downcase(value) do "off" -> 0 "false" -> 0 "no" -> 0 "0" -> 0 "on" -> 2_000 "true" -> 2_000 "yes" -> 2_000 "1" -> 2_000 _other -> 2_000 end end defp pragma_cache_spill({:literal, value}, current), do: pragma_cache_spill(value, current) defp pragma_cache_spill(_value, current), do: current defp pragma_max_page_count(value, _current) when is_integer(value) and value > 0, do: value defp pragma_max_page_count(_value, current), do: current defp pragma_journal_mode(value, current) when is_binary(value) do case String.downcase(value) do "memory" -> "memory" "off" -> "off" "wal" -> current mode when mode in ["delete", "truncate", "persist"] -> current _other -> current end end defp pragma_journal_mode(_value, current), do: current defp pragma_journal_size_limit(value) when is_integer(value) and value >= -1, do: value defp pragma_journal_size_limit(_value), do: 0 defp journal_mode_result(mode, db) do {%Result{ command: :select, columns: ["journal_mode"], rows: [[mode]], rows_affected: 0, affinities: [:text] }, db} end defp pragma_locking_mode(value, current) when is_binary(value) do case String.downcase(value) do mode when mode in ["normal", "exclusive"] -> mode _other -> current end end defp pragma_locking_mode(_value, current), do: current defp locking_mode_result(mode, db) do {%Result{ command: :select, columns: ["locking_mode"], rows: [[mode]], rows_affected: 0, affinities: [:text] }, %{db | locking_mode: mode}} end defp pragma_synchronous(value) when is_integer(value) and value in 0..3, do: value defp pragma_synchronous(value) when is_binary(value) do case String.downcase(value) do "off" -> 0 "normal" -> 1 "full" -> 2 "extra" -> 3 _other -> 0 end end defp pragma_synchronous(_value), do: 0 defp pragma_temp_store(value) when is_integer(value) and value in 0..2, do: value defp pragma_temp_store(value) when is_binary(value) do case String.downcase(value) do "default" -> 0 "file" -> 1 "memory" -> 2 _other -> 0 end end defp pragma_temp_store(_value), do: 0 defp non_negative_pragma_integer(value) when is_integer(value) and value >= 0, do: value defp non_negative_pragma_integer(_value), do: 0 defp pragma_analysis_limit(value, _current) when is_integer(value) and value >= 0, do: value defp pragma_analysis_limit(_value, current), do: current defp pragma_threads(value, _current) when is_integer(value) and value >= 0, do: min(value, 8) defp pragma_threads(_value, current), do: current defp pragma_secure_delete(value) when is_integer(value) do cond do value == 0 -> 0 value > 0 -> 1 true -> 0 end end defp pragma_secure_delete(value) when is_binary(value) do case String.downcase(value) do "fast" -> 2 "on" -> 1 "true" -> 1 "yes" -> 1 "1" -> 1 _other -> 0 end end defp pragma_secure_delete({:literal, value}), do: pragma_secure_delete(value) defp pragma_secure_delete(_value), do: 0 defp pragma_bool_result(name, value, db) do {%Result{ command: :select, columns: [name], rows: [[bool_int(value)]], rows_affected: 0, affinities: [:integer] }, db} end defp pragma_integer_result(name, value, db) do {%Result{ command: :select, columns: [name], rows: [[value]], rows_affected: 0, affinities: [:integer] }, db} end defp explain_query_plan_rows(db, %With{query: query}), do: explain_query_plan_rows(db, query) defp explain_query_plan_rows(_db, %Values{}), do: [[1, 0, 0, "SCAN CONSTANT ROW"]] defp explain_query_plan_rows(db, %Compound{} = stmt) do {left_rows, next_id} = db |> explain_query_plan_rows(stmt.left) |> renumber_query_plan_rows(3, 2) op_id = next_id {right_rows, _next_id} = db |> explain_query_plan_rows(stmt.right) |> renumber_query_plan_rows(op_id + 1, op_id) [[1, 0, 0, "COMPOUND QUERY"], [2, 1, 0, "LEFT-MOST SUBQUERY"]] ++ left_rows ++ [[op_id, 1, 0, compound_query_plan_detail(stmt.op)]] ++ right_rows end defp explain_query_plan_rows(_db, %Select{from: nil}), do: [[1, 0, 0, "SCAN CONSTANT ROW"]] defp explain_query_plan_rows(db, %Select{from: from, where: where}) do {rows, _next_id} = explain_from_rows(db, from, where, 2) rows end defp explain_from_rows(db, {:table, name, alias_name}, where, id) do {[[id, 0, 0, explain_table_detail(db, name, alias_name, where)]], id + 1} end defp explain_from_rows(_db, {:table_function, name, _args, alias_name}, _where, id) do {[[id, 0, 0, "SCAN #{alias_name || name} VIRTUAL TABLE INDEX 1:"]], id + 1} end defp explain_from_rows(db, {:subquery, query, alias_name}, _where, id) do {rows, next_id} = db |> explain_query_plan_rows(query) |> renumber_query_plan_rows(id, 0) rows = Enum.map(rows, fn [row_id, parent, notused, detail] -> [row_id, parent, notused, subquery_detail(detail, alias_name)] end) {rows, next_id} end defp explain_from_rows( db, {:join, type, left, {:table, right_name, right_alias}, constraint}, where, id ) do case explain_join_right_table_detail( db, type, left, right_name, right_alias, constraint, where ) do nil -> case explain_join_left_table_detail( db, type, left, right_name, right_alias, constraint, where ) do {_left_name, _left_alias, left_detail} -> right_detail = explain_table_detail(db, right_name, right_alias, nil) {[[id, 0, 0, right_detail], [id + 1, 0, 0, left_detail]], id + 2} nil -> {left_rows, next_id} = explain_from_rows(db, left, nil, id) right_detail = explain_table_detail(db, right_name, right_alias, nil) {left_rows ++ [[next_id, 0, 0, right_detail]], next_id + 1} end right_detail -> {left_rows, next_id} = explain_from_rows(db, left, nil, id) {left_rows ++ [[next_id, 0, 0, right_detail]], next_id + 1} end end defp explain_from_rows(db, {:join, _type, left, right, _constraint}, _where, id) do {left_rows, next_id} = explain_from_rows(db, left, nil, id) {right_rows, next_id} = explain_from_rows(db, right, nil, next_id) {left_rows ++ right_rows, next_id} end defp explain_table_detail(db, name, alias_name, where) do display = table_source_display(name, alias_name) case plain_table(db, table_source_key(name)) do nil -> "SCAN #{display}" table -> access_path_detail(table, display, table_access_path(db, table, where)) end end defp explain_join_right_table_detail(db, type, left, right_name, right_alias, constraint, where) do with join_kind when join_kind in [:inner, :left, :right, :full] <- indexed_join_kind(type), %Table{} = table <- plain_table(db, table_source_key(right_name)), ltmpls when ltmpls != [] <- explain_templates(db, left) do table = ensure_index_entries(db, table) rtmpl = table_frame(table, right_alias) using = using_columns(type, constraint, ltmpls, rtmpl) right_qualifier = table_source_qualifier(right_name, right_alias) right_display = table_source_display(right_name, right_alias) lookup_terms = join_lookup_terms(join_kind, constraint, where) ++ using_lookup_terms(using, right_qualifier) case join_rowid_lookup_plan(table, right_qualifier, ltmpls, lookup_terms) do {:ok, {:eq, expr}} -> access_path_detail(table, right_display, {:rowid_eq, expr}) {:ok, {:in, exprs}} -> access_path_detail(table, right_display, {:rowid_in, exprs}) {:ok, {:range, bounds}} -> access_path_detail(table, right_display, {:rowid_range, bounds}) :error -> join_index_right_table_detail(table, right_name, right_alias, ltmpls, lookup_terms) end else _ -> nil end end defp join_index_right_table_detail(table, right_name, right_alias, ltmpls, lookup_terms) do right_display = table_source_display(right_name, right_alias) case join_index_lookup_plan(table, right_name, right_alias, ltmpls, lookup_terms) do {:ok, index, {:eq, prefix}} -> access_path_detail(table, right_display, {:index_member_eq, index, prefix}) {:ok, index, {:in, prefix, member, exprs}} -> access_path_detail( table, right_display, {:index_member_in, index, prefix, member, exprs} ) {:ok, index, {:range, prefix, range_member, bounds}} -> access_path_detail( table, right_display, {:index_member_range, index, prefix, range_member, bounds} ) :error -> nil end end defp explain_join_left_table_detail( db, type, {:table, left_name, left_alias}, right_name, right_alias, constraint, where ) do with :inner <- indexed_join_kind(type), %Table{} = table <- plain_table(db, table_source_key(left_name)), %Table{} = right_table <- plain_table(db, table_source_key(right_name)) do table = ensure_index_entries(db, table) ltmpl = table_frame(table, left_alias) rtmpl = table_frame(right_table, right_alias) using = using_columns(type, constraint, [ltmpl], rtmpl) left_qualifier = table_source_qualifier(left_name, left_alias) left_display = table_source_display(left_name, left_alias) rtmpls = [%{rtmpl | hidden: MapSet.union(rtmpl.hidden, MapSet.new(using))}] lookup_terms = join_lookup_terms(:inner, constraint, where) ++ using_lookup_terms(using, left_qualifier) case join_rowid_lookup_plan(table, left_qualifier, rtmpls, lookup_terms) do {:ok, {:eq, expr}} -> {left_name, left_alias, access_path_detail(table, left_display, {:rowid_eq, expr})} {:ok, {:in, exprs}} -> {left_name, left_alias, access_path_detail(table, left_display, {:rowid_in, exprs})} {:ok, {:range, bounds}} -> {left_name, left_alias, access_path_detail(table, left_display, {:rowid_range, bounds})} :error -> case join_index_lookup_plan(table, left_name, left_alias, rtmpls, lookup_terms) do {:ok, index, {:eq, prefix}} -> {left_name, left_alias, access_path_detail( table, left_display, {:index_member_eq, index, prefix} )} {:ok, index, {:in, prefix, member, exprs}} -> {left_name, left_alias, access_path_detail( table, left_display, {:index_member_in, index, prefix, member, exprs} )} {:ok, index, {:range, prefix, range_member, bounds}} -> {left_name, left_alias, access_path_detail( table, left_display, {:index_member_range, index, prefix, range_member, bounds} )} :error -> nil end end else _ -> nil end end defp explain_join_left_table_detail( _db, _type, _left, _right_name, _right_alias, _constraint, _where ), do: nil defp explain_templates(db, {:table, name, alias_name}) do case plain_table(db, Table.key(name)) do %Table{} = table -> [table_frame(table, alias_name)] _ -> [] end end defp explain_templates(db, {:join, type, left, {:table, name, alias_name}, constraint}) do ltmpls = explain_templates(db, left) case plain_table(db, Table.key(name)) do %Table{} = table -> rtmpl = table_frame(table, alias_name) using = using_columns(type, constraint, ltmpls, rtmpl) ltmpls ++ [%{rtmpl | hidden: MapSet.union(rtmpl.hidden, MapSet.new(using))}] _ -> ltmpls end end defp explain_templates(_db, _from), do: [] defp access_path_detail(_table, display, {:rowid_eq, _value}), do: "SEARCH #{display} USING INTEGER PRIMARY KEY (rowid=?)" defp access_path_detail(_table, display, {:rowid_in, _exprs}), do: "SEARCH #{display} USING INTEGER PRIMARY KEY (rowid=?)" defp access_path_detail(_table, display, {:rowid_range, [{op, _expr} | _bounds]}) do op_text = %{lt: "<", le: "<=", gt: ">", ge: ">="}[op] "SEARCH #{display} USING INTEGER PRIMARY KEY (rowid#{op_text}?)" end defp access_path_detail(table, display, {:index_eq, index, n_columns}) do terms = index.columns |> Enum.take(n_columns) |> Enum.map_join(" AND ", &"#{display_column_name(table, &1)}=?") "SEARCH #{display} USING INDEX #{index.name} (#{terms})" end defp access_path_detail(table, display, {:index_range, index, [{op, _expr} | _bounds]}) do op_text = %{lt: "<", le: "<=", gt: ">", ge: ">="}[op] column = display_column_name(table, List.first(index.columns)) "SEARCH #{display} USING INDEX #{index.name} (#{column}#{op_text}?)" end defp access_path_detail(table, display, {:index_in, index, _exprs}) do column = display_column_name(table, List.first(index.columns)) "SEARCH #{display} USING INDEX #{index.name} (#{column}=?)" end defp access_path_detail(_table, display, {:expr_index_eq, index, _expr, _value}), do: "SEARCH #{display} USING INDEX #{index.name} (=?)" defp access_path_detail(_table, display, {:expr_index_in, index, _expr, _values}), do: "SEARCH #{display} USING INDEX #{index.name} (=?)" defp access_path_detail(_table, display, {:expr_index_or, index, _expr, _values}), do: "SEARCH #{display} USING INDEX #{index.name} (=?)" defp access_path_detail( _table, display, {:expr_index_range, index, _expr, [{op, _bound} | _bounds]} ) do op_text = %{lt: "<", le: "<=", gt: ">", ge: ">="}[op] "SEARCH #{display} USING INDEX #{index.name} (#{op_text}?)" end defp access_path_detail(table, display, {:index_member_eq, index, prefix}) do terms = prefix |> Enum.map_join(" AND ", fn {{:column, key}, _expr} -> "#{display_column_name(table, key)}=?" {{:expr, _indexed_expr}, _expr} -> "=?" end) "SEARCH #{display} USING INDEX #{index.name} (#{terms})" end defp access_path_detail(table, display, {:index_member_in, index, prefix, in_member, _exprs}) do prefix_terms = prefix |> Enum.map(fn {{:column, key}, _expr} -> "#{display_column_name(table, key)}=?" {{:expr, _indexed_expr}, _expr} -> "=?" end) in_term = case in_member do {:column, key} -> "#{display_column_name(table, key)}=?" {:expr, _indexed_expr} -> "=?" end terms = Enum.join(prefix_terms ++ [in_term], " AND ") "SEARCH #{display} USING INDEX #{index.name} (#{terms})" end defp access_path_detail( table, display, {:index_member_range, index, prefix, range_member, [{op, _bound} | _bounds]} ) do prefix_terms = prefix |> Enum.map(fn {{:column, key}, _expr} -> "#{display_column_name(table, key)}=?" {{:expr, _indexed_expr}, _expr} -> "=?" end) op_text = %{lt: "<", le: "<=", gt: ">", ge: ">="}[op] range_term = case range_member do {:column, key} -> "#{display_column_name(table, key)}#{op_text}?" {:expr, _indexed_expr} -> "#{op_text}?" end terms = Enum.join(prefix_terms ++ [range_term], " AND ") "SEARCH #{display} USING INDEX #{index.name} (#{terms})" end defp access_path_detail(_table, display, :scan), do: "SCAN #{display}" defp subquery_detail("SCAN CONSTANT ROW", nil), do: "SCAN CONSTANT ROW" defp subquery_detail("SCAN CONSTANT ROW", alias_name), do: "SCAN #{alias_name}" defp subquery_detail(detail, _alias_name), do: detail defp compound_query_plan_detail(:union_all), do: "UNION ALL" defp compound_query_plan_detail(:union), do: "UNION USING TEMP B-TREE" defp compound_query_plan_detail(:intersect), do: "INTERSECT USING TEMP B-TREE" defp compound_query_plan_detail(:except), do: "EXCEPT USING TEMP B-TREE" defp renumber_query_plan_rows(rows, start_id, root_parent) do id_map = rows |> Enum.map(&hd/1) |> Enum.with_index(start_id) |> Map.new() rows = Enum.map(rows, fn [old_id, old_parent, notused, detail] -> parent = if old_parent == 0, do: root_parent, else: Map.fetch!(id_map, old_parent) [Map.fetch!(id_map, old_id), parent, notused, detail] end) {rows, start_id + length(rows)} end # -- CREATE TABLE helpers ------------------------------------------------------- # Splits table-level constraints into composite_keys and composite_uniques. defp partition_table_constraints(constraints) do Enum.reduce(constraints, {[], []}, fn {:primary_key, name, cols}, {pks, uqs} -> {pks ++ [{name, cols}], uqs} {:unique, name, cols}, {pks, uqs} -> {pks, uqs ++ [{name, cols}]} {:check, _name, _expr}, acc -> acc {:foreign_key, _name, _cols, _ref_table, _ref_cols, _actions}, acc -> acc end) end # Returns only the CHECK constraints from table-level constraints. defp table_check_constraints(constraints) do for {:check, name, expr} <- constraints, do: {name, expr} end defp table_foreign_keys(constraints) do for {:foreign_key, _name, cols, ref_table, ref_cols, actions} <- constraints do {cols, ref_table, ref_cols, actions} end end defp ensure_valid_check_constraints!(table_name, columns, constraints) do Enum.each(columns, fn column -> ensure_valid_check_expr!(table_name, columns, column.check) end) Enum.each(constraints, fn {:check, _name, expr} -> ensure_valid_check_expr!(table_name, columns, expr) _constraint -> :ok end) end defp ensure_valid_check_expr!(_table_name, _columns, nil), do: :ok defp ensure_valid_check_expr!(table_name, columns, expr) do if contains_bind_parameter?(expr) do fail("parameters prohibited in CHECK constraints") end validate_check_expr_columns!( expr, Table.key(table_name), MapSet.new(columns, &Table.key(&1.name)) ) end defp validate_check_expr_columns!({:column, nil, name}, _table_key, column_keys) do unless MapSet.member?(column_keys, Table.key(name)) do fail("no such column: #{name}") end end defp validate_check_expr_columns!({:column, qualifier, name}, table_key, column_keys) do qualified_name = "#{qualifier}.#{name}" unless Table.key(qualifier) == table_key and MapSet.member?(column_keys, Table.key(name)) do fail("no such column: #{qualified_name}") end end defp validate_check_expr_columns!({:subquery, _query}, _table_key, _column_keys), do: fail("subqueries prohibited in CHECK constraints") defp validate_check_expr_columns!({:exists, _query}, _table_key, _column_keys), do: fail("subqueries prohibited in CHECK constraints") defp validate_check_expr_columns!( {:in, _expr, {:select, _query}, _negated}, _table_key, _column_keys ), do: fail("subqueries prohibited in CHECK constraints") defp validate_check_expr_columns!(%module{}, _table_key, _column_keys) when module in [Select, Compound, Values, With], do: fail("subqueries prohibited in CHECK constraints") defp validate_check_expr_columns!(tuple, table_key, column_keys) when is_tuple(tuple) do tuple |> Tuple.to_list() |> Enum.each(&validate_check_expr_columns!(&1, table_key, column_keys)) end defp validate_check_expr_columns!(list, table_key, column_keys) when is_list(list) do Enum.each(list, &validate_check_expr_columns!(&1, table_key, column_keys)) end defp validate_check_expr_columns!(_other, _table_key, _column_keys), do: :ok # -- ALTER TABLE helpers -------------------------------------------------------- defp rename_in_composites(composites, old_key, new_key) do Enum.map(composites, fn {cname, cols} -> {cname, Enum.map(cols, fn k -> if k == old_key, do: new_key, else: k end)} end) end defp alter_add_column(db, table, col_def) do ensure_valid_check_expr!(table.name, table.columns ++ [col_def], col_def.check) # Restrictions if col_def.primary_key do fail("Cannot add a PRIMARY KEY column") end if col_def.unique do fail("Cannot add a UNIQUE column") end # NOT NULL without a usable default is only an error if rows exist has_rows = table.rows != %{} if has_rows and col_def.not_null and not has_constant_default?(col_def) do fail("Cannot add a NOT NULL column with default value NULL") end if has_rows and not constant_default?(col_def) do fail("Cannot add a column with non-constant default") end # With foreign keys on, existing rows would all take the default value, # which cannot be checked against the parent table. if has_rows and db.foreign_keys and col_def.references != nil and has_constant_default?(col_def) do fail("Cannot add a REFERENCES column with non-NULL default value") end new_col_key = Table.key(col_def.name) default = column_default_value(db, col_def) new_rows = Map.new(Table.scan(table), fn {rowid, row} -> {rowid, Map.put(row, new_col_key, default)} end) new_checks = if col_def.check != nil do table.checks ++ [{col_def.check_name, col_def.check}] else table.checks end new_table = Table.narrow_all_rows(%{ table | columns: table.columns ++ [col_def], rows: new_rows, checks: new_checks, frame_columns: nil, column_index: nil }) {%Result{command: :alter_table}, db |> put_table(new_table) |> Database.schema_changed()} end defp alter_drop_column(db, table, col_name) do col = Table.column(table, col_name) unless col do fail(~s(no such column: "#{col_name}")) end if length(table.columns) == 1 do fail(~s(cannot drop column "#{col_name}": no other columns exist)) end if col.primary_key do fail(~s(cannot drop PRIMARY KEY column: "#{col_name}")) end if col.unique do fail(~s(cannot drop UNIQUE column: "#{col_name}")) end col_key = Table.key(col_name) # An index over the column would be left dangling; SQLite errors too. index_uses_column? = fn index -> Enum.any?(index_members(index), fn {:column, key} -> key == col_key {:expr, expr} -> expr_references_column?(expr, col_key) end) end case Enum.find(table.indexes, index_uses_column?) do nil -> :ok index -> fail("error in index #{index.name} after drop column: no such column: #{col_name}") end new_columns = Enum.reject(table.columns, &(Table.key(&1.name) == col_key)) new_rowid_alias = if table.rowid_alias == col_key, do: nil, else: table.rowid_alias # Remove column from all rows new_rows = Map.new(Table.scan(table), fn {rowid, row} -> {rowid, Map.delete(row, col_key)} end) # Remove column from composite constraints new_composite_keys = drop_from_composites(table.composite_keys, col_key) new_composite_uniques = drop_from_composites(table.composite_uniques, col_key) # Remove checks that reference only the dropped column (heuristic: drop column-level checks # on that specific column; table-level checks are kept as SQLite would error on them) new_checks = Enum.reject(table.checks, fn {_name, _expr} -> false end) new_table = Table.narrow_all_rows(%{ table | columns: new_columns, rowid_alias: new_rowid_alias, rows: new_rows, composite_keys: new_composite_keys, composite_uniques: new_composite_uniques, autoindexes: drop_from_autoindexes(table.autoindexes, col_key), checks: new_checks, frame_columns: nil, column_index: nil }) {%Result{command: :alter_table}, db |> put_table(new_table) |> Database.schema_changed()} end defp drop_from_composites(composites, col_key) do composites |> Enum.map(fn {cname, cols} -> {cname, Enum.reject(cols, &(&1 == col_key))} end) |> Enum.reject(fn {_cname, cols} -> cols == [] end) end # Returns true if the column has a constant (literal) default defp constant_default?(%{default: nil}), do: true defp constant_default?(%{default: {:literal, _}}), do: true defp constant_default?(%{default: {:negate, {:literal, _}}}), do: true defp constant_default?(%{default: {:column, nil, _}}), do: true defp constant_default?(_), do: false # Returns true if the column has a non-NULL constant default (satisfies NOT NULL) defp has_constant_default?(%{default: nil}), do: false defp has_constant_default?(%{default: {:literal, nil}}), do: false defp has_constant_default?(%{default: {:literal, _}}), do: true defp has_constant_default?(%{default: {:negate, {:literal, _}}}), do: true defp has_constant_default?(%{default: {:column, nil, _}}), do: true defp has_constant_default?(_), do: false # The value a DEFAULT clause produces for an omitted column. Literal and # bare-word forms are resolved directly; any other form is an expression # default (e.g. `DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ','now'))`) and is # evaluated like SQLite does at insert time, in an empty row environment # (DEFAULT expressions may not reference columns). Evaluation failures fall # back to NULL rather than aborting the insert. defp column_default_value(_db, %{default: nil}), do: nil defp column_default_value(_db, %{default: {:literal, value}, affinity: aff}), do: Value.apply_affinity(value, aff) defp column_default_value(_db, %{default: {:negate, {:literal, value}}, affinity: aff}) when is_number(value), do: Value.apply_affinity(-value, aff) defp column_default_value(_db, %{default: {:column, nil, word}, affinity: aff}), do: Value.apply_affinity(word, aff) defp column_default_value(db, %{default: expr, affinity: aff}) when expr != nil do env = %{db: db, frames: [], group: nil, outer: nil} expr |> eval(env) |> Value.apply_affinity(aff) rescue _ -> nil catch _ -> nil end defp column_default_value(_db, _), do: nil # Extract an integer LIMIT from the outermost SELECT/Compound/With for # use as a recursive CTE row cap. Returns nil if not present or not an integer literal. defp extract_query_limit(%Select{limit: {:literal, n}}, _db) when is_integer(n) and n > 0, do: n defp extract_query_limit(%Compound{limit: {:literal, n}}, _db) when is_integer(n) and n > 0, do: n defp extract_query_limit(%With{query: inner}, db), do: extract_query_limit(inner, db) defp extract_query_limit(_, _), do: nil # Any place a query can appear (subquery, EXISTS, IN, FROM) accepts a # simple select, a compound, bare VALUES, or a WITH. defp query_result(db, %Select{} = stmt, outer), do: select_result(db, stmt, outer) defp query_result(db, %Compound{} = stmt, outer), do: compound_result(db, stmt, outer) defp query_result(db, %Values{} = stmt, outer), do: values_result(db, stmt, outer) defp query_result(db, %With{} = stmt, outer) do outer_limit = extract_query_limit(stmt.query, db) # Inner WITH CTEs can shadow outer CTEs. We need to allow the inner names # to be evaluated even if they already exist in db.ctes. Save the outer ctes, # remove any keys that this WITH will shadow, then evaluate. inner_keys = MapSet.new(stmt.ctes, &Table.key(&1.name)) db_for_inner = %{db | ctes: Map.drop(db.ctes, MapSet.to_list(inner_keys))} db_with_ctes = resolve_ctes(db_for_inner, stmt.ctes, stmt.recursive, outer_limit) # Run the body with inner scope; discard inner CTEs from result (restore outer). result = query_result(db_with_ctes, stmt.query, outer) result end # A bare VALUES select: output columns are named column1..columnN. defp values_result(db, %Values{} = stmt, outer) do env = %{db: db, frames: [], group: nil, outer: outer} case stmt.rows |> Enum.map(&length/1) |> Enum.uniq() do [width] -> rows = Enum.map(stmt.rows, fn exprs -> Enum.map(exprs, &eval(&1, env)) end) names = Enum.map(1..width, &"column#{&1}") rows = rows |> compound_order(stmt.order_by, names, [names]) |> clamp(db, stmt.limit, stmt.offset) %Result{command: :select, columns: names, rows: rows, rows_affected: 0} _mixed -> fail("all VALUES must have the same number of terms") end end defp savepoint_index(stack, name) do target = {:savepoint, Table.key(name)} Enum.find_index(stack, fn {kind, _snapshot} -> kind == target end) end # -- INSERT helpers ------------------------------------------------------------- # The rows to insert, as evaluated value lists (or `:default` for # DEFAULT VALUES), with SQLite's count-mismatch errors. defp insert_rows(_db, %Insert{source: :default_values}, _table, _targets), do: [:default] defp insert_rows(db, %Insert{source: {:values, expr_rows}} = stmt, table, targets) do case expr_rows |> Enum.map(&length/1) |> Enum.uniq() do [width] -> check_insert_width(stmt, table, targets, width) env = constant_env(db) Enum.map(expr_rows, fn exprs -> Enum.map(exprs, &eval(&1, env)) end) _mixed -> fail("all VALUES must have the same number of terms") end end defp insert_rows(db, %Insert{source: {:select, query}} = stmt, table, targets) do result = query_result(db, query, nil) check_insert_width(stmt, table, targets, length(result.columns)) result.rows end defp check_insert_width(stmt, table, targets, width) do cond do stmt.columns == nil and width != length(targets) -> fail( "table #{table.name} has #{length(targets)} columns " <> "but #{width} values were supplied" ) stmt.columns != nil and width != length(targets) -> fail("#{width} values for #{length(targets)} columns") true -> :ok end end # Builds a full candidate row map (all columns, with defaults for missing values) # for CHECK constraint evaluation, without going through Table.insert. defp build_candidate_row(table, values, db) do # Column keys are static per table; reuse the cached folded keys # (`frame_columns`) rather than re-folding `Table.key(column.name)` for every # column on every inserted row — a hot spot on bulk/repeated inserts. table |> Table.frame_columns() |> Enum.zip(table.columns) |> Map.new(fn {{col_key, _name, _aff, _coll}, column} -> value = case Map.fetch(values, col_key) do {:ok, v} -> v :error -> column_default_value(db, column) end {col_key, value} end) end defp with_explicit_rowid(row, %{rowid_alias: alias_key}, rowid) when alias_key != nil and is_integer(rowid), do: Map.put(row, alias_key, rowid) defp with_explicit_rowid(row, _table, _rowid), do: row defp apply_generated_columns(row, db, table, rowid) do Enum.reduce(table.columns, row, fn %{generated: {_, expr}} = column, row -> env = table_env(db, table, rowid, row) value = expr |> eval(env) |> Value.apply_affinity(column.affinity) Map.put(row, Table.key(column.name), value) _column, row -> row end) end defp check_strict_types!(%{strict: false}, _row), do: :ok defp check_strict_types!(table, row) do Enum.each(table.columns, fn column -> value = Map.get(row, Table.key(column.name)) if column.generated == nil and value != nil and not strict_value_allowed?(column, value) do storage_type = value |> Value.type_of() |> Atom.to_string() |> String.upcase() declared_type = String.upcase(column.declared_type) fail( "cannot store #{storage_type} value in #{declared_type} column #{table.name}.#{column.name}" ) end end) end defp strict_value_allowed?(%{declared_type: type}, value) do case {String.upcase(type), Value.type_of(value)} do {"ANY", _storage_class} -> true {"INT", :integer} -> true {"INTEGER", :integer} -> true {"REAL", :real} -> true {"TEXT", :text} -> true {"BLOB", :blob} -> true _ -> false end end # -- foreign keys ------------------------------------------------------------- # # SQLite checks immediate foreign keys at the end of each statement: the # statement fails if it leaves more violations than it found (fkey.c keeps # a per-statement counter; the delta scan below is the tree-walking # equivalent). Deferred constraints wait until the outermost transaction # commits. RESTRICT actions fire as parent rows are deleted or updated, # regardless of deferral; CASCADE / SET NULL / SET DEFAULT rewrite child # rows before the statement-end check runs. defp with_fk_statement_check(db, table_name, fun) when is_binary(table_name) do entries = if db.foreign_keys do db |> relevant_fk_entries(Database.table_storage_key(nil, table_name)) |> immediate_fk_entries(db) else [] end if entries == [] do fun.() else # Unresolvable references (missing parent table, key mismatch) only # error on DML against the child table itself, as in SQLite. target_key = Table.key(table_name) {strict, lax} = Enum.split_with(entries, fn {child_key, _parent_key, _spec} -> child_key == target_key end) count = fn db -> fk_violation_count(db, strict, :raise) + fk_violation_count(db, lax, :skip) end violations_before = count.(db) {result, new_db} = fun.() if count.(new_db) > violations_before do fail("FOREIGN KEY constraint failed") end {result, new_db} end end defp with_fk_statement_check(db, %{schema: schema, table: table_name}, fun) when is_binary(table_name) do with_fk_statement_check(db, Database.table_storage_key(schema, table_name), fun) end defp with_fk_statement_check(_db, _stmt, fun), do: fun.() defp check_deferred_foreign_keys!(%{foreign_keys: false}), do: :ok defp check_deferred_foreign_keys!(db) do {_kind, snapshot} = List.last(db.txn_stack) db_before = Database.restore_schema(db, snapshot) violations_now = fk_violation_count(db, all_fk_entries(db), :skip) violations_before = fk_violation_count(db_before, all_fk_entries(db_before), :skip) if violations_now > violations_before do fail("FOREIGN KEY constraint failed") end :ok end defp all_fk_entries(db) do Enum.flat_map(db.tables, fn {child_key, child_table} -> Enum.map(foreign_key_specs(child_table), fn spec -> {child_key, fk_parent_key(child_table, spec), spec} end) end) end # Foreign keys a DML statement on `table_name` can affect: those whose # child or parent is the table itself or any table reachable from it # through delete/update actions. defp relevant_fk_entries(db, table_key) do entries = all_fk_entries(db) closure = fk_action_closure(entries, MapSet.new([table_key])) Enum.filter(entries, fn {child_key, parent_key, _spec} -> MapSet.member?(closure, child_key) or MapSet.member?(closure, parent_key) end) end defp fk_action_closure(entries, set) do additions = for {child_key, parent_key, _spec} <- entries, MapSet.member?(set, parent_key), not MapSet.member?(set, child_key), do: child_key case additions do [] -> set keys -> fk_action_closure(entries, MapSet.union(set, MapSet.new(keys))) end end # Inside a transaction, constraints declared DEFERRABLE INITIALLY DEFERRED # (or all of them under PRAGMA defer_foreign_keys) wait until COMMIT. # Outside a transaction every constraint is effectively immediate. defp immediate_fk_entries(entries, db) do if db.txn_stack == [] do entries else Enum.reject(entries, fn {_child_key, _parent_key, spec} -> spec.deferred or db.defer_foreign_keys end) end end defp fk_violation_count(db, entries, on_missing) do Enum.reduce(entries, 0, fn {child_key, _parent_key, spec}, acc -> case Map.fetch(db.tables, child_key) do {:ok, child_table} -> acc + fk_spec_violation_count(db, child_table, spec, on_missing) :error -> acc end end) end defp fk_parent_key(child_table, spec) do Database.table_storage_key(child_table.schema, spec.parent_table) end # `:skip` tolerates unresolvable references (missing parent table, key # mismatch) by not counting them — used at COMMIT, where SQLite only sees # the counter accumulated by statements that resolved successfully. defp fk_spec_violation_count(db, child_table, spec, :skip) do fk_spec_violation_count(db, child_table, spec, :raise) rescue _e in Error -> 0 end defp fk_spec_violation_count(db, child_table, spec, :raise) do {parent_table, parent_columns} = referenced_parent!(db, child_table, spec, child_table) Enum.count(Table.scan(child_table), fn {_rowid, row} -> child_values = Enum.map(spec.child_keys, &Map.get(row, &1)) not Enum.any?(child_values, &is_nil/1) and not parent_row_exists?(parent_table, parent_columns, child_values) end) end # With foreign keys on, DROP TABLE performs an implicit DELETE FROM the # table first: delete actions fire and remaining references fail the drop. defp drop_table_fk_cleanup(%{foreign_keys: false} = db, _schema, _name), do: db defp drop_table_fk_cleanup(db, schema, name) do table_key = Database.table_storage_key(schema, name) case Map.fetch(db.tables, table_key) do :error -> db {:ok, table} when map_size(table.rows) == 0 -> db {:ok, table} -> {_result, db} = with_fk_statement_check(db, table_key, fn -> deleted_rows = Enum.map(Table.scan(table), fn {_rowid, row} -> row end) emptied = Table.delete_rows(table, Map.keys(table.rows)) db = db |> put_table(emptied) |> apply_fk_delete_actions(emptied, deleted_rows) {nil, db} end) db end end defp apply_fk_delete_actions(%{foreign_keys: false} = db, _parent_table, _deleted_rows), do: db defp apply_fk_delete_actions(db, _parent_table, []), do: db defp apply_fk_delete_actions(db, parent_table, deleted_rows) do Enum.reduce(parent_reference_checks(db, parent_table), db, fn {child_table, spec, parent_columns}, db -> child_table = refetch_table(db, child_table) deleted_keys = deleted_rows |> Enum.map(fn row -> Enum.map(parent_columns, &Map.get(row, Table.key(&1.name))) end) |> Enum.reject(fn values -> Enum.any?(values, &is_nil/1) end) if deleted_keys == [] do db else matches = matching_child_rows(child_table, spec, parent_columns, deleted_keys) case spec.on_delete do :restrict -> if matches != [] or restrict_among_deleted?( child_table, parent_table, spec, parent_columns, deleted_rows ) do fail("FOREIGN KEY constraint failed") end db :cascade -> rowids = Enum.map(matches, &elem(&1, 0)) child_table = Table.delete_rows(child_table, rowids) db = put_table(db, child_table) apply_fk_delete_actions(db, child_table, Enum.map(matches, &elem(&1, 1))) :set_null -> fk_set_child_columns(db, child_table, spec, matches, fn _column -> nil end) :set_default -> fk_set_child_columns(db, child_table, spec, matches, &column_default_value(db, &1)) :no_action -> db end end end) end # A self-referential RESTRICT fires as each parent row is deleted, so a row # deleted later in the same statement still counts as a referencing child # of one deleted earlier. defp restrict_among_deleted?(child_table, parent_table, spec, parent_columns, deleted_rows) do Database.table_storage_key(child_table.schema, child_table.name) == Database.table_storage_key(parent_table.schema, parent_table.name) and deleted_rows |> Enum.with_index() |> Enum.any?(fn {row, index} -> parent_values = Enum.map(parent_columns, &Map.get(row, Table.key(&1.name))) not Enum.any?(parent_values, &is_nil/1) and deleted_rows |> Enum.drop(index + 1) |> Enum.any?(fn later -> child_values = Enum.map(spec.child_keys, &Map.get(later, &1)) not Enum.any?(child_values, &is_nil/1) and foreign_key_values_match?(child_values, parent_values, parent_columns) end) end) end defp apply_fk_update_actions(%{foreign_keys: false} = db, _parent_table, _changed_pairs), do: db defp apply_fk_update_actions(db, _parent_table, []), do: db defp apply_fk_update_actions(db, parent_table, changed_pairs) do Enum.reduce(parent_reference_checks(db, parent_table), db, fn {child_table, spec, parent_columns}, db -> changes = changed_pairs |> Enum.map(fn {old_row, new_row} -> {Enum.map(parent_columns, &Map.get(old_row, Table.key(&1.name))), Enum.map(parent_columns, &Map.get(new_row, Table.key(&1.name)))} end) |> Enum.reject(fn {old_values, new_values} -> Enum.any?(old_values, &is_nil/1) or foreign_key_parent_values_equal?(old_values, new_values) end) Enum.reduce(changes, db, fn {old_values, new_values}, db -> child_table = refetch_table(db, child_table) matches = matching_child_rows(child_table, spec, parent_columns, [old_values]) case spec.on_update do :restrict -> if matches != [], do: fail("FOREIGN KEY constraint failed") db :cascade -> new_by_key = spec.child_keys |> Enum.zip(new_values) |> Map.new() fk_set_child_columns(db, child_table, spec, matches, fn column -> new_by_key |> Map.fetch!(Table.key(column.name)) |> Value.apply_affinity(column.affinity) end) :set_null -> fk_set_child_columns(db, child_table, spec, matches, fn _column -> nil end) :set_default -> fk_set_child_columns(db, child_table, spec, matches, &column_default_value(db, &1)) :no_action -> db end end) end) end # REPLACE conflict resolution is a DELETE followed by an INSERT, so the # rows it removes fire ON DELETE foreign key actions, as in SQLite. The # removed rows are found by diffing the table against its pre-statement # rows, excluding rowids the statement updated in place. defp apply_replace_deleted_actions(%{foreign_keys: false} = db, _table, _old_rows, _excluded), do: db defp apply_replace_deleted_actions(db, table, old_rows, excluded_rowids) do deleted = for {rowid, tuple} <- old_rows, not MapSet.member?(excluded_rowids, rowid), Map.get(table.rows, rowid) != tuple, do: Table.row_to_map(table, tuple) apply_fk_delete_actions(db, table, deleted) end defp matching_child_rows(child_table, spec, parent_columns, parent_keys) do Enum.filter(Table.scan(child_table), fn {_rowid, row} -> child_values = Enum.map(spec.child_keys, &Map.get(row, &1)) not Enum.any?(child_values, &is_nil/1) and Enum.any?(parent_keys, &foreign_key_values_match?(child_values, &1, parent_columns)) end) end # Applies a SET NULL / SET DEFAULT / cascading-update rewrite to child rows, # enforcing the child table's own constraints, then propagates the change # to foreign keys that reference the child table in turn. defp fk_set_child_columns(db, child_table, spec, matches, value_fun) do {child_table, pairs} = Enum.reduce(matches, {child_table, []}, fn {rowid, row}, {table, pairs} -> new_row = spec.child_keys |> Enum.reduce(row, fn key, row -> Map.put(row, key, value_fun.(Table.column(table, key))) end) |> apply_generated_columns(db, table, rowid) check_strict_types!(table, new_row) check_env = table_env(db, table, rowid, new_row) case check_violations(db, table, new_row, check_env, :abort) do :ok -> :ok {:error, message} -> fail(message) end case resolve_unique_indexes(db, table, rowid, new_row, :abort) do {:ok, table} -> case Table.update_row(table, rowid, new_row) do {:ok, table} -> new_rowid = updated_rowid(table, rowid, new_row) {table, [{row, Table.fetch_row!(table, new_rowid)} | pairs]} {:error, message} -> fail(message) end {:error, message} -> fail(message) end end) db |> put_table(child_table) |> apply_fk_update_actions(child_table, Enum.reverse(pairs)) end defp refetch_table(db, table) do case Map.fetch(db.tables, Database.table_storage_key(table.schema, table.name)) do {:ok, table} -> table :error -> table end end # -- triggers ----------------------------------------------------------------- # # Triggers fire per affected row. OLD./NEW. references in the WHEN clause # and body are substituted with the row's values before execution, so body # statements run through the ordinary exec path. RAISE(IGNORE) abandons # the row operation and any later triggers without rolling back changes # already made (thrown as :raise_ignore, caught per body statement). defp triggers_for(db, table_name, timing, event) when is_binary(table_name) do triggers_for(db, nil, table_name, timing, event) end defp triggers_for(db, %{schema: schema, table: table_name}, timing, event), do: triggers_for(db, schema, table_name, timing, event) defp triggers_for(db, schema, table_name, timing, event) do key = Database.table_storage_key(schema, table_name) db.triggers |> Map.values() |> Enum.filter(&(&1.table_key == key and &1.timing == timing and &1.event == event)) |> Enum.sort_by(& &1.seq) end defp validate_trigger_statement!(stmt, trigger_schema, trigger_name) do case stmt do %Insert{source: :default_values} -> fail(~s(near "DEFAULT": syntax error)) %Insert{returning: [_ | _]} -> fail("cannot use RETURNING in a trigger") %Insert{target_qualified: true} -> fail(qualified_trigger_dml_message()) %Insert{} = insert -> validate_trigger_insert_statement!(insert, trigger_schema, trigger_name) %Update{target_qualified: true} -> fail(qualified_trigger_dml_message()) %Delete{target_qualified: true} -> fail(qualified_trigger_dml_message()) %Update{index_hint: :not_indexed} -> fail(not_indexed_trigger_dml_message()) %Delete{index_hint: :not_indexed} -> fail(not_indexed_trigger_dml_message()) %Update{index_hint: {:indexed_by, _}} -> fail(indexed_by_trigger_dml_message()) %Update{} = update -> validate_trigger_update_statement!(update) validate_trigger_update_sources!(update, trigger_schema, trigger_name) %Delete{index_hint: {:indexed_by, _}} -> fail(indexed_by_trigger_dml_message()) %Delete{returning: [_ | _]} -> fail(~s(near "RETURNING": syntax error)) %Delete{order_by: [_ | _]} -> fail(~s(near "ORDER": syntax error)) %Delete{limit: limit} when not is_nil(limit) -> fail(~s(near "LIMIT": syntax error)) %Delete{} = delete -> validate_trigger_delete_sources!(delete, trigger_schema, trigger_name) %Select{} = query -> validate_trigger_query_sources!(query, trigger_schema, trigger_name) %Compound{} = query -> validate_trigger_query_sources!(query, trigger_schema, trigger_name) %Values{} -> :ok %With{query: %Insert{}} -> fail(~s(near "INSERT": syntax error)) %With{query: %Update{}} -> fail(~s(near "UPDATE": syntax error)) %With{query: %Delete{}} -> fail(~s(near "DELETE": syntax error)) %With{query: query} = with_query -> validate_trigger_query_sources!(with_query, trigger_schema, trigger_name) validate_trigger_statement!(query, trigger_schema, trigger_name) _other -> fail("unsupported statement in trigger body") end end defp validate_trigger_update_statement!(%Update{returning: [_ | _]}), do: fail(~s(near "RETURNING": syntax error)) defp validate_trigger_update_statement!(%Update{order_by: [_ | _]}), do: fail(~s(near "ORDER": syntax error)) defp validate_trigger_update_statement!(%Update{limit: limit}) when not is_nil(limit), do: fail(~s(near "LIMIT": syntax error)) defp validate_trigger_update_statement!(%Update{}), do: :ok defp validate_trigger_insert_statement!(%Insert{} = stmt, trigger_schema, trigger_name) do source_schemas = case stmt.source do {:select, query} -> trigger_query_source_schemas(query) {:values, rows} -> rows |> List.flatten() |> trigger_exprs_source_schemas() _other -> [] end (source_schemas ++ trigger_upsert_source_schemas(stmt.upsert)) |> validate_trigger_source_schemas!(trigger_schema, trigger_name) end defp validate_trigger_update_sources!(%Update{} = stmt, trigger_schema, trigger_name) do assignment_exprs = Enum.map(stmt.assignments, fn {_name, expr} -> expr end) (trigger_from_source_schemas(stmt.from) ++ trigger_exprs_source_schemas(assignment_exprs) ++ trigger_expr_source_schemas(stmt.where)) |> validate_trigger_source_schemas!(trigger_schema, trigger_name) end defp validate_trigger_delete_sources!(%Delete{} = stmt, trigger_schema, trigger_name) do stmt.where |> trigger_expr_source_schemas() |> validate_trigger_source_schemas!(trigger_schema, trigger_name) end defp trigger_upsert_source_schemas(upserts) do Enum.flat_map(upserts, fn {:nothing, target} -> trigger_upsert_target_source_schemas(target) {:update, target, assignments, where} -> assignment_exprs = Enum.map(assignments, fn {_name, expr} -> expr end) trigger_upsert_target_source_schemas(target) ++ trigger_exprs_source_schemas(assignment_exprs) ++ trigger_expr_source_schemas(where) end) end defp trigger_upsert_target_source_schemas({_columns, where}), do: trigger_expr_source_schemas(where) defp trigger_upsert_target_source_schemas(_target), do: [] defp validate_trigger_query_sources!(_query, "temp", _trigger_name), do: :ok defp validate_trigger_query_sources!(query, trigger_schema, trigger_name) do query |> trigger_query_source_schemas() |> validate_trigger_source_schemas!(trigger_schema, trigger_name) end defp validate_trigger_source_schemas!(_source_schemas, "temp", _trigger_name), do: :ok defp validate_trigger_source_schemas!(source_schemas, trigger_schema, trigger_name) do trigger_schema = trigger_schema(trigger_schema) Enum.each(source_schemas, fn source_schema -> normalized_source_schema = trigger_schema(source_schema) if source_schema != nil and normalized_source_schema != trigger_schema do fail("trigger #{trigger_name} cannot reference objects in database #{source_schema}") end end) end defp trigger_from_source_schemas(nil), do: [] defp trigger_from_source_schemas({:table, {:schema, schema, _name}, _alias}), do: [schema] defp trigger_from_source_schemas({:table, _name, _alias}), do: [] defp trigger_from_source_schemas({:subquery, query, _alias}), do: trigger_query_source_schemas(query) defp trigger_from_source_schemas({:join, _type, left, right, constraint}), do: trigger_from_source_schemas(left) ++ trigger_from_source_schemas(right) ++ trigger_join_constraint_source_schemas(constraint) defp trigger_from_source_schemas(_other), do: [] defp trigger_join_constraint_source_schemas({:on, expr}), do: trigger_expr_source_schemas(expr) defp trigger_join_constraint_source_schemas(_constraint), do: [] defp trigger_query_source_schemas(%Select{} = query) do trigger_from_source_schemas(query.from) ++ trigger_select_column_source_schemas(query.columns) ++ trigger_expr_source_schemas(query.where) ++ trigger_exprs_source_schemas(query.group_by) ++ trigger_expr_source_schemas(query.having) ++ trigger_order_source_schemas(query.order_by) ++ trigger_expr_source_schemas(query.limit) ++ trigger_expr_source_schemas(query.offset) end defp trigger_query_source_schemas(%Compound{left: left, right: right}), do: trigger_query_source_schemas(left) ++ trigger_query_source_schemas(right) defp trigger_query_source_schemas(%With{ctes: ctes, query: query}) do Enum.flat_map(ctes, &trigger_query_source_schemas(&1.query)) ++ trigger_query_source_schemas(query) end defp trigger_query_source_schemas(_query), do: [] defp trigger_select_column_source_schemas(columns) do Enum.flat_map(columns, fn {expr, _alias} -> trigger_expr_source_schemas(expr) _other -> [] end) end defp trigger_order_source_schemas(order_by) do Enum.flat_map(order_by, fn {expr, _direction} -> trigger_expr_source_schemas(expr) end) end defp trigger_exprs_source_schemas(exprs), do: Enum.flat_map(exprs, &trigger_expr_source_schemas/1) defp trigger_expr_source_schemas({:select, query}), do: trigger_query_source_schemas(query) defp trigger_expr_source_schemas({:subquery, query}), do: trigger_query_source_schemas(query) defp trigger_expr_source_schemas({:exists, query}), do: trigger_query_source_schemas(query) defp trigger_expr_source_schemas(%Select{} = query), do: trigger_query_source_schemas(query) defp trigger_expr_source_schemas(%Compound{} = query), do: trigger_query_source_schemas(query) defp trigger_expr_source_schemas(%With{} = query), do: trigger_query_source_schemas(query) defp trigger_expr_source_schemas(term) when is_tuple(term) do term |> Tuple.to_list() |> Enum.flat_map(&trigger_expr_source_schemas/1) end defp trigger_expr_source_schemas(term) when is_list(term), do: Enum.flat_map(term, &trigger_expr_source_schemas/1) defp trigger_expr_source_schemas(term) when is_map(term) do term |> Map.values() |> Enum.flat_map(&trigger_expr_source_schemas/1) end defp trigger_expr_source_schemas(_term), do: [] defp validate_trigger_definition!(%CreateTrigger{} = stmt) do if contains_bind_parameter?(stmt.when) or Enum.any?(stmt.body, &contains_bind_parameter?/1) do fail("trigger cannot use variables") end Enum.each(stmt.body, &validate_trigger_statement!(&1, stmt.schema, stmt.name)) end defp qualified_trigger_dml_message do "qualified table names are not allowed on INSERT, UPDATE, and DELETE statements within triggers" end defp not_indexed_trigger_dml_message do "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements within triggers" end defp indexed_by_trigger_dml_message do "the INDEXED BY clause is not allowed on UPDATE or DELETE statements within triggers" end defp contains_bind_parameter?({:param, _index, _raw}), do: true defp contains_bind_parameter?(term) when is_tuple(term) do term |> Tuple.to_list() |> Enum.any?(&contains_bind_parameter?/1) end defp contains_bind_parameter?(term) when is_list(term), do: Enum.any?(term, &contains_bind_parameter?/1) defp contains_bind_parameter?(term) when is_map(term) do map = if Map.has_key?(term, :__struct__) do Map.from_struct(term) else term end map |> Map.values() |> Enum.any?(&contains_bind_parameter?/1) end defp contains_bind_parameter?(_term), do: false defp drop_trigger_key(db, nil, name) do Enum.find_value(trigger_lookup_order(db), fn schema -> key = Database.table_storage_key(schema, name) if Map.has_key?(db.triggers, key), do: key end) end defp drop_trigger_key(db, schema, name) do key = Database.table_storage_key(schema, name) if Map.has_key?(db.triggers, key), do: key end defp trigger_lookup_order(db), do: ["temp", nil] ++ Enum.map(db.attached_databases, & &1.name) defp drop_triggers_on(db, schema, table_name) do key = Database.table_storage_key(schema, table_name) %{ db | triggers: db.triggers |> Enum.reject(fn {_k, trigger} -> trigger.table_key == key end) |> Map.new() } end defp fire_triggers(db, triggers, table, old_row, new_row, changed_keys \\ nil) do Enum.reduce_while(triggers, {:ok, db}, fn trigger, {:ok, db} -> skip? = (trigger.key in db.active_triggers and not db.recursive_triggers) or (trigger.update_columns != nil and changed_keys != nil and not Enum.any?(trigger.update_columns, &(&1 in changed_keys))) if skip? do {:cont, {:ok, db}} else case fire_trigger(db, trigger, table, old_row, new_row) do {:ok, db} -> {:cont, {:ok, db}} {:ignored, db} -> {:halt, {:ignored, db}} end end end) end defp fire_trigger(db, trigger, table, old_row, new_row) do if length(db.active_triggers) >= @max_trigger_depth do fail("too many levels of trigger recursion") end saved_active = db.active_triggers db = %{db | active_triggers: [trigger.key | saved_active]} fires? = trigger.when == nil or trigger.when |> trigger_substitute(table, old_row, new_row) |> truth(%{db: db, frames: [], group: nil, outer: nil}) |> Kernel.==(true) result = if fires? do Enum.reduce_while(trigger.body, {:ok, db}, fn stmt, {:ok, db} -> stmt = stmt |> trigger_substitute(table, old_row, new_row) |> qualify_trigger_statement(trigger.schema) try do {_result, db} = exec(db, stmt) {:cont, {:ok, db}} catch :raise_ignore -> {:halt, {:ignored, db}} end end) else {:ok, db} end {status, db} = result {status, %{db | active_triggers: saved_active}} end defp trigger_substitute(ast, table, old_row, new_row) do trig_walk(ast, table, old_row, new_row) end defp trig_walk({:column, qualifier, name} = node, table, old_row, new_row) when is_binary(qualifier) do case Table.key(qualifier) do "old" when old_row != nil -> {:literal, trigger_row_value(table, old_row, name, qualifier)} "new" when new_row != nil -> {:literal, trigger_row_value(table, new_row, name, qualifier)} _ -> node end end defp trig_walk(tuple, table, old_row, new_row) when is_tuple(tuple) do tuple |> Tuple.to_list() |> Enum.map(&trig_walk(&1, table, old_row, new_row)) |> List.to_tuple() end defp trig_walk(list, table, old_row, new_row) when is_list(list) do Enum.map(list, &trig_walk(&1, table, old_row, new_row)) end defp trig_walk(%module{} = node, table, old_row, new_row) do struct!( module, node |> Map.from_struct() |> Enum.map(fn {key, value} -> {key, trig_walk(value, table, old_row, new_row)} end) ) end defp trig_walk(%{} = map, table, old_row, new_row) do Map.new(map, fn {key, value} -> {key, trig_walk(value, table, old_row, new_row)} end) end defp trig_walk(other, _table, _old_row, _new_row), do: other defp trigger_row_value(table, {:trigger_row, rowid, row}, name, qualifier) do key = Table.key(name) cond do table.rowid_alias != nil and key == table.rowid_alias and is_nil(Map.get(row, key)) -> rowid Map.has_key?(row, key) -> Map.get(row, key) key in @rowid_names and table.rowid_alias != nil -> Map.get(row, table.rowid_alias) || rowid key in @rowid_names and not table.without_rowid -> rowid true -> fail("no such column: #{qualifier}.#{name}") end end defp trigger_row_value(table, row, name, qualifier) do key = Table.key(name) cond do Map.has_key?(row, key) -> Map.get(row, key) key in @rowid_names and table.rowid_alias != nil -> Map.get(row, table.rowid_alias) true -> fail("no such column: #{qualifier}.#{name}") end end defp parent_reference_checks(db, parent_table) do Enum.flat_map(db.tables, fn {_key, child_table} -> child_table |> foreign_key_specs() |> Enum.flat_map(fn spec -> if fk_parent_key(child_table, spec) == Database.table_storage_key(parent_table.schema, parent_table.name) do [{child_table, spec, referenced_columns!(child_table, spec, parent_table)}] else [] end end) end) end defp referenced_parent!(db, child_table, spec, current_table) do referenced_table_name = spec.parent_table parent_table = if fk_parent_key(child_table, spec) == Database.table_storage_key(current_table.schema, current_table.name) do current_table else case fetch_fk_parent_table(db, child_table, referenced_table_name) do {:ok, table} -> table {:error, _message} -> fail("no such table: #{child_table.schema || "main"}.#{referenced_table_name}") end end {parent_table, referenced_columns!(child_table, spec, parent_table)} end defp fetch_fk_parent_table(db, child_table, parent_name) do key = Database.table_storage_key(child_table.schema, parent_name) case Map.fetch(db.tables, key) do {:ok, table} -> {:ok, table} :error -> {:error, "no such table: #{child_table.schema || "main"}.#{parent_name}"} end end defp referenced_columns!(child_table, spec, parent_table) do parent_column_keys = case spec.parent_keys do [] -> primary_key_column_keys(parent_table) || foreign_key_mismatch!(child_table, parent_table) keys -> keys end if length(parent_column_keys) != length(spec.child_keys) do foreign_key_mismatch!(child_table, parent_table) end parent_columns = Enum.map(parent_column_keys, fn key -> Table.column(parent_table, key) || foreign_key_mismatch!(child_table, parent_table) end) unless referenced_key_unique?(parent_table, parent_column_keys) do foreign_key_mismatch!(child_table, parent_table) end parent_columns end defp referenced_key_unique?(%{rowid_alias: rowid_alias}, [rowid_alias]) when rowid_alias != nil, do: true defp referenced_key_unique?(parent_table, keys) do inline_unique? = case keys do [key] -> case Table.column(parent_table, key) do %{primary_key: true} -> true %{unique: true} -> true _ -> false end _ -> false end composite_unique? = Enum.any?(parent_table.composite_keys ++ parent_table.composite_uniques, fn {_name, cols} -> cols == keys end) index_unique? = Enum.any?(parent_table.indexes, fn index -> index.unique and index.columns == keys end) inline_unique? or composite_unique? or index_unique? end defp primary_key_column_keys(%{rowid_alias: rowid_alias}) when rowid_alias != nil, do: [rowid_alias] defp primary_key_column_keys(table) do inline_keys = table.columns |> Enum.filter(& &1.primary_key) |> Enum.map(&Table.key(&1.name)) composite_keys = case table.composite_keys do [{_name, keys}] -> keys _ -> [] end case inline_keys ++ composite_keys do [] -> nil keys -> keys end end defp parent_row_exists?(parent_table, parent_columns, child_values) do Enum.any?(Table.scan(parent_table), fn {_rowid, parent_row} -> parent_values = Enum.map(parent_columns, &Map.get(parent_row, Table.key(&1.name))) foreign_key_values_match?(child_values, parent_values, parent_columns) end) end defp foreign_key_values_match?(child_values, parent_values, parent_columns) do Enum.zip([child_values, parent_values, parent_columns]) |> Enum.all?(fn {child_value, parent_value, parent_column} -> child_value |> Value.apply_affinity(parent_column.affinity) |> Value.compare(parent_value) |> Kernel.==(:eq) end) end defp foreign_key_parent_values_equal?(old_values, new_values) do Enum.zip(old_values, new_values) |> Enum.all?(fn {old_value, new_value} -> Value.compare(old_value, new_value) == :eq end) end defp foreign_key_specs(table) do column_specs = table.columns |> Enum.filter(& &1.references) |> Enum.map(fn column -> {parent_table, parent_keys, actions} = column.references Map.merge(actions, %{ child_keys: [Table.key(column.name)], parent_table: parent_table, parent_keys: Enum.map(parent_keys, &Table.key/1) }) end) table_specs = Enum.map(table.foreign_keys, fn {child_keys, parent_table, parent_keys, actions} -> Map.merge(actions, %{ child_keys: child_keys, parent_table: parent_table, parent_keys: parent_keys }) end) column_specs ++ table_specs end defp foreign_key_mismatch!(child_table, parent_table) do fail(~s(foreign key mismatch - "#{child_table.name}" referencing "#{parent_table.name}")) end defp resolve_unique_indexes(db, table, rowid, row, on_conflict) do conflicts = unique_index_conflicts(db, table, rowid, row) case {conflicts, on_conflict} do {[], _} -> {:ok, table} {_conflicts, :ignore} -> :ignore {conflicts, :replace} -> rowids = Enum.flat_map(conflicts, &elem(&1, 0)) |> Enum.uniq() {:ok, Table.delete_rows(table, rowids)} {[{_rowids, message} | _], _} -> {:error, message} end end defp resolve_unique_indexes_for_dml(db, table, rowid, row, on_conflict, opts \\ []) do case {replacement_conflict_rowids(db, table, rowid, row, opts), on_conflict} do {[], _} -> case resolve_unique_indexes(db, table, rowid, row, on_conflict) do {:ok, table} -> {:ok, db, table} other -> other end {_rowids, :ignore} -> :ignore {rowids, :replace} -> fire_replace_delete_triggers(db, table, rowids) {_rowids, _} -> case resolve_unique_indexes(db, table, rowid, row, on_conflict) do {:ok, table} -> {:ok, db, table} other -> other end end end defp replacement_conflict_rowids(db, table, rowid, row, opts) do excluded_rowids = opts |> Keyword.get(:excluding_rowids, []) |> Enum.reject(&is_nil/1) |> MapSet.new() (table_constraint_conflict_rowids(table, rowid, row, excluded_rowids) ++ explicit_index_conflict_rowids(db, table, rowid, row)) |> Enum.reject(&MapSet.member?(excluded_rowids, &1)) |> Enum.uniq() end defp fire_replace_delete_triggers(db, table, rowids) do if db.recursive_triggers do before_triggers = triggers_for(db, table.schema, table.name, :before, :delete) after_triggers = triggers_for(db, table.schema, table.name, :after, :delete) Enum.reduce(rowids, {:ok, db, table}, fn rowid, {:ok, db, table} -> case Table.fetch_row(table, rowid) do :error -> {:ok, db, table} {:ok, row} -> db = put_table(db, table) {status, db} = fire_triggers(db, before_triggers, table, trigger_row(rowid, row), nil) table = refetch_table(db, table) if status == :ignored or not Map.has_key?(table.rows, rowid) do {:ok, db, table} else table = Table.delete_rows(table, [rowid]) {db, table} = fire_after_row_triggers(db, table, after_triggers, trigger_row(rowid, row), nil) {:ok, db, table} end end end) else {:ok, db, Table.delete_rows(table, rowids)} end end defp table_constraint_conflict_rowids(table, rowid, row, excluded_rowids) do rowid_conflicts = if is_integer(rowid) and Map.has_key?(table.rows, rowid) do [rowid] else [] end single_unique_conflicts = for column <- table.columns, unique_constraint_column?(table, column), column_key = Table.key(column.name), value = Map.fetch!(row, column_key), not is_nil(value), {conflicting_rowid, existing} <- Table.scan(table), not MapSet.member?(excluded_rowids, conflicting_rowid), Value.compare(Map.fetch!(existing, column_key), value) == :eq do conflicting_rowid end composite_key_conflicts = table.composite_keys |> Enum.reject(fn {_name, column_keys} -> length(column_keys) == 1 and hd(column_keys) == table.rowid_alias end) |> composite_constraint_conflict_rowids(table, row, excluded_rowids) composite_unique_conflicts = composite_constraint_conflict_rowids(table.composite_uniques, table, row, excluded_rowids) rowid_conflicts ++ single_unique_conflicts ++ composite_key_conflicts ++ composite_unique_conflicts end defp unique_constraint_column?(table, column), do: (column.primary_key or column.unique) and Table.key(column.name) != table.rowid_alias defp composite_constraint_conflict_rowids(constraints, table, row, excluded_rowids) do Enum.flat_map(constraints, fn {_name, column_keys} -> values = Enum.map(column_keys, &Map.fetch!(row, &1)) if Enum.any?(values, &is_nil/1) do [] else for {rowid, existing} <- Table.scan(table), not MapSet.member?(excluded_rowids, rowid), Enum.zip(column_keys, values) |> Enum.all?(fn {key, value} -> Value.compare(Map.fetch!(existing, key), value) == :eq end) do rowid end end end) end defp explicit_index_conflict_rowids(db, table, rowid, row) do db |> unique_index_conflicts(table, rowid, row) |> Enum.flat_map(&elem(&1, 0)) end defp insert_conflict_rowid(_table, rowid, _row) when is_integer(rowid), do: rowid defp insert_conflict_rowid(table, nil, row) do case table.rowid_alias do nil -> table.next_rowid alias_key -> case Map.fetch!(row, alias_key) do nil -> next_insert_rowid(table) rowid when is_integer(rowid) -> rowid _other -> nil end end end defp insert_conflict_rowid(_table, _rowid, _row), do: nil defp next_insert_rowid(table) do cond do table.autoincrement and table.sequence_row -> max(table.sequence + 1, table.next_rowid) table.autoincrement -> table_next_available_rowid(table) true -> table.next_rowid end end defp table_next_available_rowid(table) do table.rows |> Map.keys() |> Enum.max(fn -> 0 end) |> Kernel.+(1) end # Explicit UNIQUE indexes are enforced here because index members can carry # collations, including connection-local callbacks stored on the database. defp unique_index_conflicts(db, table, rowid, row) do for index <- table.indexes, index.unique, index.where == nil or row_matches_partial_index?(db, table, rowid, row, index.where), index_values = index_member_values(db, table, rowid, row, index), not Enum.any?(index_values, &is_nil/1), conflicts = unique_index_duplicates(db, table, index, index_values, rowid), conflicts != [] do {conflicts, index_conflict_message(table, index)} end end # Fast path: for a binary-collation index with materialized entries, the # conflicting rowids are exactly `entries[values]` (a hash lookup), turning a # bulk insert into a unique-indexed table from O(n²) into O(n). The entry key # uses affinity-canonical member values, which match binary `Value.compare` # equality, so this is exact for binary collation; collated/custom indexes # (where two distinct keys may be equal) and un-materialized entries take the # row-scan fallback below. Entries are kept current by the per-row maintenance # in the insert/update paths, so a duplicate inserted earlier in the same # statement is already visible here. defp unique_index_duplicates(db, table, index, values, excluding_rowid) do entries = Map.get(index, :entries) if entries != nil and binary_collation_index?(index) do entries |> Map.get(List.to_tuple(values), []) |> Enum.reject(&(&1 == excluding_rowid)) else for {existing_rowid, existing_row} <- Table.scan(table), existing_rowid != excluding_rowid, index.where == nil or row_matches_partial_index?(db, table, existing_rowid, existing_row, index.where), existing_values = index_member_values(db, table, existing_rowid, existing_row, index), not Enum.any?(existing_values, &is_nil/1), index_values_equal?(db, index, existing_values, values) do existing_rowid end end end defp binary_collation_index?(index) do case Map.get(index, :collations) do nil -> true collations -> Enum.all?(collations, fn nil -> true name when is_binary(name) -> String.downcase(name) == "binary" _ -> false end) end end defp row_matches_partial_index?(db, table, rowid, row, where) do env = table_env(db, table, rowid, row) truth(where, env) == true end defp index_members(index), do: Map.get(index, :members) || Enum.map(index.columns, &{:column, &1}) defp lookup_indexes(table), do: table.indexes ++ table.autoindexes defp index_member_values(db, table, rowid, row, index) do Enum.map(index_members(index), fn # `row` may be the raw stored tuple (the hot per-insert maintenance path) or # a `key => value` map (full rebuilds via `Table.scan`). {:column, key} when is_tuple(row) -> Table.cell(table, row, key) {:column, key} -> case row do %{^key => value} -> value _ -> nil end {:expr, expr} -> eval(expr, table_env(db, table, rowid, row)) end) end defp index_values_equal?(db, index, left_values, right_values) do collations = Map.get(index, :collations) || List.duplicate(nil, length(left_values)) values_equal_with_collations?(db, left_values, right_values, collations) end defp values_equal_with_collations?(db, left_values, right_values, collations) do Enum.zip([left_values, right_values, collations]) |> Enum.all?(fn {left, right, collation} -> Value.compare(left, right, normalize_collation!(collation, %{db: db})) == :eq end) end defp index_conflict_message(table, index) do if index.columns != [] do "UNIQUE constraint failed: #{column_list(table, index.columns)}" else "UNIQUE constraint failed: index '#{index.name}'" end end # Deep scan for a column reference, for DROP COLUMN's dangling-index check. defp expr_references_column?({:column, _qualifier, name}, col_key), do: Table.key(name) == col_key defp expr_references_column?(tuple, col_key) when is_tuple(tuple) do tuple |> Tuple.to_list() |> Enum.any?(&expr_references_column?(&1, col_key)) end defp expr_references_column?(list, col_key) when is_list(list) do Enum.any?(list, &expr_references_column?(&1, col_key)) end defp expr_references_column?(_other, _col_key), do: false defp validate_index_expression!(table, expr) do case expr do {:column, nil, name} -> unless Table.column(table, name), do: fail("no such column: #{name}") {:column, _qualifier, name} -> unless Table.column(table, name), do: fail("no such column: #{name}") tuple when is_tuple(tuple) -> tuple |> Tuple.to_list() |> Enum.each(&validate_index_expression!(table, &1)) list when is_list(list) -> Enum.each(list, &validate_index_expression!(table, &1)) _other -> :ok end end defp index_member_collation(_table, %{collate: collation}, _member) when is_binary(collation), do: collation defp index_member_collation(table, _column_spec, {:column, key}) do case Table.column(table, key) do nil -> nil column -> column.collate end end defp index_member_collation(_table, %{collate: collation}, _member), do: collation defp column_list(table, col_keys) do Enum.map_join(col_keys, ", ", fn col_key -> "#{table.name}.#{display_column_name(table, col_key)}" end) end defp insert_or_upsert( db, table, values, explicit_rowid, %{upsert: []}, on_conflict, _before_update_triggers, _after_update_triggers ) do rowid = insert_conflict_rowid(table, explicit_rowid, values) case resolve_unique_indexes_for_dml(db, table, rowid, values, on_conflict) do {:ok, db, table} -> case Table.insert(table, values, rowid: explicit_rowid, on_conflict: on_conflict) do {:ok, table, rowid} -> {:inserted, db, maybe_add_index_entries(db, table, rowid), rowid} other -> other end other -> other end end defp insert_or_upsert( db, table, values, explicit_rowid, %{upsert: clauses}, on_conflict, before_update_triggers, after_update_triggers ) do candidate = upsert_candidate(db, table, values, explicit_rowid) Enum.each(clauses, &validate_upsert_target!(table, elem(&1, 1))) # The clauses chain: the first whose target the candidate conflicts with # handles the row; conflicts on untargeted constraints fail normally. match = Enum.find_value(clauses, fn clause -> case upsert_conflict_rowid(db, table, elem(clause, 1), candidate, explicit_rowid) do nil -> nil rowid -> {clause, rowid} end end) case match do nil -> # A trailing catch-all DO NOTHING also swallows partial-index # conflicts the targeted probe above cannot see. insert_algorithm = if Enum.any?(clauses, &(elem(&1, 1) == nil)), do: :ignore, else: on_conflict rowid = insert_conflict_rowid(table, explicit_rowid, values) case resolve_unique_indexes_for_dml(db, table, rowid, values, insert_algorithm) do {:ok, db, table} -> case Table.insert(table, values, rowid: explicit_rowid, on_conflict: on_conflict) do {:ok, table, rowid} -> # Keep index entries current as we go so the next row's unique # check (and the final store) need no full rebuild. {:inserted, db, maybe_add_index_entries(db, table, rowid), rowid} other -> other end other -> other end {{:nothing, _target}, _rowid} -> :ignore {{:update, _target, assignments, where}, rowid} -> existing = Table.fetch_row!(table, rowid) env = upsert_env(db, table, rowid, existing, candidate) if where != nil and truth(where, env) != true do :ignore else assignments = Enum.map(assignments, &update_assignment(table, &1)) {new_row, explicit_rowid} = updated_row_and_rowid(table, assignments, existing, env) new_row = apply_generated_columns(new_row, db, table, update_rowid_value(explicit_rowid, rowid)) check_strict_types!(table, new_row) changed_keys = Enum.map(assignments, &update_assignment_key/1) {trigger_status, db, table} = if before_update_triggers == [] do {:ok, db, table} else db = put_table(db, table) {status, db} = fire_triggers(db, before_update_triggers, table, existing, new_row, changed_keys) {status, db, refetch_table(db, table)} end cond do trigger_status == :ignored -> :ignore not Map.has_key?(table.rows, rowid) -> :ignore true -> new_row = if before_update_triggers == [] do new_row else table |> Table.fetch_row!(rowid) |> rebase_updated_row(table, assignments, new_row) |> apply_generated_columns( db, table, update_conflict_rowid(table, rowid, new_row, explicit_rowid) ) end check_strict_types!(table, new_row) conflict_rowid = update_conflict_rowid(table, rowid, new_row, explicit_rowid) case resolve_unique_indexes_for_dml( db, table, conflict_rowid, new_row, on_conflict, excluding_rowids: [rowid] ) do {:ok, db, table} -> opts = update_rowid_opts([on_conflict: on_conflict], explicit_rowid) case Table.update_row(table, rowid, new_row, opts) do {:ok, table} -> # An upsert UPDATE moves a row; rebuild this table's index # entries so later rows in the batch see a consistent set # (the incremental add path only knows about plain inserts). table = refresh_index_entries(db, table) new_rowid = updated_rowid(table, rowid, new_row, explicit_rowid) returning_row = Table.fetch_row!(table, new_rowid) {db, table} = fire_after_row_triggers( db, table, after_update_triggers, existing, returning_row ) {:updated, db, table, new_rowid, {rowid, existing, returning_row}} other -> other end other -> other end end end end end defp upsert_candidate(db, table, values, explicit_rowid) do candidate = build_candidate_row(table, values, db) case {table.rowid_alias, explicit_rowid} do {alias_key, rowid} when alias_key != nil and is_integer(rowid) -> Map.put(candidate, alias_key, rowid) _ -> candidate end end defp upsert_env(db, table, rowid, existing, candidate) do target_frame = %{table_frame(table, nil) | row: existing, rowid: rowid} excluded_frame = %{ table_frame(table, "excluded") | row: candidate, rowid: nil, hidden: MapSet.new(Enum.map(table.columns, &Table.key(&1.name))) } %{db: db, frames: [target_frame, excluded_frame], group: nil, outer: nil} end defp validate_upsert_target!(_table, nil), do: :ok defp validate_upsert_target!(table, {columns, target_where}) do Enum.each(columns, fn column_name -> unless Table.column(table, column_name) do fail("no such column: #{column_name}") end end) target_keys = Enum.map(columns, &Table.key/1) # A bare target matches full uniqueness constraints only; targeting a # partial unique index requires the WHERE clause, as in SQLite. matched? = if target_where == nil do Enum.any?(upsert_unique_targets(table), &(&1 == target_keys)) else Enum.any?(table.indexes, fn index -> index.unique and Map.get(index, :where) != nil and index.columns == target_keys end) end unless matched? do fail("ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint") end end defp upsert_conflict_rowid(db, table, nil, candidate, explicit_rowid) do table |> upsert_unique_targets() |> Enum.find_value(&upsert_conflict_rowid(db, table, {&1, nil}, candidate, explicit_rowid)) end defp upsert_conflict_rowid(db, table, {target_columns, target_where}, candidate, explicit_rowid) do target_keys = Enum.map(target_columns, &Table.key/1) cond do target_keys == [table.rowid_alias] and target_where == nil -> rowid_value = explicit_rowid || Map.get(candidate, table.rowid_alias) if is_integer(rowid_value) and Map.has_key?(table.rows, rowid_value), do: rowid_value true -> values = Enum.map(target_keys, &Map.get(candidate, &1)) cond do Enum.any?(values, &is_nil/1) -> nil # A partial index only conflicts when both the candidate and the # existing row satisfy the index predicate. target_where != nil and truth(target_where, table_env(db, table, nil, candidate)) != true -> nil true -> collations = upsert_target_collations(table, target_keys, target_where) Enum.find_value(Table.scan(table), fn {rowid, row} -> existing_values = Enum.map(target_keys, &Map.get(row, &1)) if values_equal_with_collations?(db, existing_values, values, collations) and (target_where == nil or truth(target_where, table_env(db, table, rowid, row)) == true) do rowid end end) end end end defp upsert_unique_targets(table) do rowid_targets = if table.rowid_alias != nil do [[table.rowid_alias]] else [] end column_targets = for column <- table.columns, (column.primary_key or column.unique) and Table.key(column.name) != table.rowid_alias, do: [Table.key(column.name)] index_targets = for index <- table.indexes, index.unique, is_nil(Map.get(index, :where)), do: index.columns composite_pk_targets = for {_name, keys} <- table.composite_keys, not (length(keys) == 1 and hd(keys) == table.rowid_alias), do: keys composite_unique_targets = for {_name, keys} <- table.composite_uniques, do: keys rowid_targets ++ column_targets ++ index_targets ++ composite_pk_targets ++ composite_unique_targets end defp upsert_target_collations(table, target_keys, target_where) do case matching_unique_index(table, target_keys, target_where) do nil -> Enum.map(target_keys, &column_collation_name(table, &1)) index -> Map.get(index, :collations) || Enum.map(target_keys, &column_collation_name(table, &1)) end end defp matching_unique_index(table, target_keys, target_where) do Enum.find(table.indexes, fn index -> index.unique and index.columns == target_keys and ((target_where == nil and is_nil(Map.get(index, :where))) or (target_where != nil and Map.get(index, :where) != nil)) end) end defp column_collation_name(table, key) do case Table.column(table, key) do nil -> nil column -> column.collate end end # Validates CHECK constraints for the candidate row. # Returns :ok, :ignore (on_conflict == :ignore), or {:error, message}. # SQLite: OR IGNORE suppresses CHECK failures; OR REPLACE does NOT bypass CHECK. defp check_violations( %{ignore_check_constraints: true}, _table, _candidate_row, _check_env, _on_conflict ), do: :ok defp check_violations(_db, table, _candidate_row, check_env, on_conflict) do Enum.find_value(table.checks, :ok, fn {cname, expr} -> result = truth(expr, check_env) # NULL result passes (only false fails) if result == false do if on_conflict == :ignore do :ignore else message = if cname do "CHECK constraint failed: #{cname}" else "CHECK constraint failed: #{check_text(expr)}" end {:error, message} end end end) end defp insert_values(_targets, :default), do: {%{}, nil} defp insert_values(targets, row) do targets |> Enum.zip(row) |> Enum.reduce({%{}, nil}, fn {:rowid, value}, {values, _rowid} -> if value == nil or is_integer(value), do: {values, value}, else: fail("datatype mismatch") {{column, key}, value}, {values, rowid} -> coerced = Value.apply_affinity(value, column.affinity) {Map.put(values, key, coerced), rowid} end) end defp select_result(db, stmt, outer) do # Try the JIT codegen path first (uncorrelated SELECTs only); it self-gates # and returns :fallback for unsupported shapes / one-shot queries. with nil <- outer, {:ok, result} <- ExSQL.Codegen.run_select(db, stmt) do result else _ -> case compile_vdbe(db, stmt, outer) do {:ok, plan} -> run_vdbe(plan) :unsupported -> select_result_treewalk(db, stmt, outer) end end end defp select_result_treewalk(db, stmt, outer) do check_window_placement!(stmt.where) Enum.each(stmt.group_by, &check_window_placement!/1) check_window_placement!(stmt.having) {templates, frame_rows} = planned_relation(db, stmt.from, stmt.where, outer) # Resolve the WHERE's column references once (single-table scans only) and # compile the predicate to a closure, rather than re-resolving names + # affinity/collation and re-dispatching the AST for every row. precompiled_where = precompile_scan_where(stmt.where, db, templates, outer) filter = compile_scan_filter(precompiled_where) # A column-free WHERE (e.g. `WHERE NULL IS NOT NULL`) is constant for the # whole scan — evaluate it once instead of per row. Constant false/NULL → no # rows; constant true → drop the filter (every row passes). {filter, frame_rows} = case constant_filter_value(precompiled_where, db) do :dynamic -> {filter, frame_rows} true -> {nil, frame_rows} _false_or_null -> {nil, []} end # Fuse the per-row env wrap and the WHERE filter into one pass, so a # filtered-out row never lands in an intermediate list. When the predicate # is row-local (only this frame's columns — no subquery, outer reference, # or db-dependent call), it reads nothing but `frames`, so filter against a # minimal map and build the full env only for surviving rows — skipping the # 4-key allocation for every row a selective filter rejects. envs = cond do filter == nil -> Enum.map(frame_rows, &%{db: db, frames: &1, group: nil, outer: outer}) match?([_], templates) and row_local?(precompiled_where) -> for frames <- frame_rows, filter.(%{group: nil, frames: frames}), do: %{db: db, frames: frames, group: nil, outer: outer} true -> for frames <- frame_rows, env = %{db: db, frames: frames, group: nil, outer: outer}, filter.(env), do: env end columns = stmt.columns |> expand_columns(templates) |> resolve_window_refs(stmt.windows) names = Enum.map(columns, &result_column_name(db, templates, &1)) envs = maybe_reverse_unordered_envs(db, stmt, columns, envs) aggregate? = aggregate_query?(db, stmt, columns) # ORDER BY expressions may reference output aliases (`ORDER BY 10-(x+y)`). # For non-aggregate queries the keys are evaluated per row against the scan # frames, so fastify their column refs (as projection/WHERE already are) to # skip per-row name resolution during the sort. Aggregate ORDER BY runs # against grouped envs, so leave it on the normal path. order_by = if aggregate? do Enum.map(stmt.order_by, fn {expr, direction} -> {substitute_aliases(expr, columns, templates), direction} end) else lookup = frame_column_lookup(db, templates) Enum.map(stmt.order_by, fn {expr, direction} -> {expr |> substitute_aliases(columns, templates) |> rewrite_frame_columns(lookup), direction} end) end window_exprs = collect_windows(columns) envs = apply_window_output_order(envs, window_exprs) window_values = compute_windows(window_exprs, envs) projected = if aggregate? do grouped = db |> grouped_envs(stmt, columns, templates, envs, outer) grouped |> Enum.map(fn genv -> {genv, Enum.map(columns, fn {expr, _} -> expr |> eval(genv) |> sql_value() end)} end) else eval_columns = precompile_scan_columns(columns, db, templates) # The vast majority of queries have no window functions; skip the per-row # `:windows` injection (and the `with_index`/`Map.get`/`Map.put` it needs) # entirely in that case — it's pure overhead on every projected row. Also # compile each output column to a closure once so the per-row projection # is a direct call rather than re-dispatching `eval/2` on the AST per # column per row (`compile_pred/1` falls back to `eval` for anything it # doesn't specialize, so the result is identical). if window_exprs == [] do compiled = Enum.map(eval_columns, fn {expr, _} -> compile_pred(expr) end) Enum.map(envs, fn env -> {env, Enum.map(compiled, fn c -> c.(env) |> sql_value() end)} end) else envs |> Enum.with_index() |> Enum.map(fn {env, index} -> env = Map.put(env, :windows, Map.get(window_values, index, %{})) {env, Enum.map(eval_columns, fn {expr, _} -> expr |> eval(env) |> sql_value() end)} end) end end template_env = %{db: db, frames: templates, group: nil, outer: outer} rows = projected |> distinct(stmt.distinct, columns, template_env) |> order(order_by, columns, names) |> Enum.map(&elem(&1, 1)) |> clamp(db, stmt.limit, stmt.offset) affinities = Enum.map(columns, fn {expr, _} -> expr_affinity(expr, template_env) end) %Result{ command: :select, columns: names, rows: rows, rows_affected: 0, affinities: affinities } end # -- VDBE compilation ----------------------------------------------------------- # # Compiles the common single-table scan/filter/project shape to an # `ExSQL.Vdbe` opcode program. Anything outside the supported subset returns # `:unsupported` so `select_result/3` falls back to the tree walker. The # compiler reuses the tree walker's own affinity/collation/name helpers, so a # compiled query produces byte-for-byte the same result as the interpreter. # # Supported: no correlation/DISTINCT/GROUP BY/HAVING/ORDER BY/window; FROM a # single plain table (not a view, CTE, or subquery); projection of `*` or # plain columns; WHERE nil or a conjunction of `column literal` terms; # integer LIMIT/OFFSET. The `reverse_unordered_selects` shuffle is honored by # declining (the tree walker owns that behavior). defp compile_vdbe(_db, _stmt, outer) when outer != nil, do: :unsupported defp compile_vdbe(db, %Select{} = stmt, _outer) do with false <- vdbe_disabled?(), true <- vdbe_simple_shape?(db, stmt), {:table, name, alias_name} when is_binary(name) <- stmt.from, %Table{} = table <- plain_table(db, relation_unqualified_table_key(db, name)), false <- vdbe_rowid_table?(table), false <- vdbe_seek_available?(db, table, stmt.where), template = table_frame(table, alias_name), columns = expand_columns(stmt.columns, [template]), false <- aggregate_query?(db, stmt, columns), [] <- collect_windows(columns), {:ok, limit} <- vdbe_int_literal(stmt.limit), {:ok, offset} <- vdbe_int_literal(stmt.offset), {:ok, projection} <- vdbe_projection(db, columns, [template], table), {:ok, filter} <- vdbe_filter(stmt.where, db, table, template), {:ok, order} <- vdbe_order(stmt.order_by, db, table, template, columns, projection.names) do program = vdbe_program(projection.loads, filter, order.key_loads) {:ok, %{ table: table, program: program, names: projection.names, affinities: projection.affinities, limit: limit, offset: offset, order: order.directions }} else _ -> :unsupported end end defp compile_vdbe(_db, _stmt, _outer), do: :unsupported # The register VM is disabled by default: the tree walker now compiles a # single-table scan's WHERE/projection columns to direct lookups, compiles the # predicate to a closure, and sorts via decorate-sort-undecorate — and that # measures faster than the VM at every scale tested (the VM rebuilds its state # map per opcode, more per-row allocation than one env + baked-in closures). # Set `EXSQL_USE_VDBE=1` to re-enable for A/B comparison. defp vdbe_disabled?, do: System.get_env("EXSQL_USE_VDBE") == nil defp vdbe_simple_shape?(db, %Select{} = stmt) do not db.reverse_unordered_selects and not stmt.distinct and stmt.group_by == [] and stmt.having == nil and map_size(stmt.windows) == 0 end # The per-row environment a compiled closure evaluates against: one frame for # the scanned table, no group/outer (single-table, uncorrelated). defp vdbe_env(db, template, row, rowid) do %{db: db, frames: [%{template | row: row, rowid: rowid}], group: nil, outer: nil} end # Selecting an INTEGER PRIMARY KEY (rowid alias) reads the rowid, not the # stored column; leave those to the tree walker for now. defp vdbe_rowid_table?(%Table{rowid_alias: alias}), do: alias != nil # Decline whenever the tree walker would satisfy this WHERE with a rowid or # index seek. The VDBE only knows how to full-scan, so compiling these would # silently drop the seek — changing not just performance but observable # behavior (e.g. how many rows a side-effecting WHERE term like a UDF visits). # `table_access_path/3` inspects index *definitions* (not materialized # entries), so this is a cheap check that mirrors the planner's own decision. defp vdbe_seek_available?(_db, _table, nil), do: false defp vdbe_seek_available?(db, table, where), do: table_access_path(db, table, where) != :scan defp vdbe_int_literal(nil), do: {:ok, nil} defp vdbe_int_literal({:literal, n}) when is_integer(n) and n >= 0, do: {:ok, n} defp vdbe_int_literal(_other), do: :unsupported # Resolves the projection to register loads plus result names and affinities. # A plain column of this table loads with the fast `:column` opcode; any other # expression compiles to an `:eval` closure over the tree walker's evaluator. defp vdbe_projection(db, columns, [template] = templates, table) do template_env = %{db: db, frames: templates, group: nil, outer: nil} {names, affs, loads, _reg} = Enum.reduce(columns, {[], [], [], 0}, fn {expr, _alias} = item, {names, affs, loads, reg} -> load = vdbe_proj_load(expr, table, db, template, reg) name = result_column_name(db, templates, item) affinity = expr_affinity(expr, template_env) {[name | names], [affinity | affs], [load | loads], reg + 1} end) {:ok, %{names: Enum.reverse(names), affinities: Enum.reverse(affs), loads: Enum.reverse(loads)}} end defp vdbe_proj_load({:column, _qualifier, name} = expr, table, db, template, reg) do key = Table.key(name) if Table.column(table, key), do: {:column, key, reg}, else: vdbe_eval_load(expr, db, template, reg) end defp vdbe_proj_load(expr, _table, db, template, reg), do: vdbe_eval_load(expr, db, template, reg) defp vdbe_eval_load(expr, db, template, reg) do {:eval, fn row, rowid -> eval(expr, vdbe_env(db, template, row, rowid)) end, reg} end # WHERE compiles to the fast `:cmp` chain when it is a conjunction of # `column literal` (binary collation); otherwise the whole predicate # becomes one `:filter` closure so any WHERE shape is still handled. defp vdbe_filter(nil, _db, _table, _template), do: {:ok, :none} defp vdbe_filter(where, db, table, template) do case vdbe_cmp_terms(where, db, table, template) do {:ok, terms} -> {:ok, {:cmp, terms}} :unsupported -> {:ok, {:filter, fn row, rowid -> matches_where?(where, vdbe_env(db, template, row, rowid)) end}} end end defp vdbe_cmp_terms({:binary, :and, left, right}, db, table, template) do with {:ok, l} <- vdbe_cmp_terms(left, db, table, template), {:ok, r} <- vdbe_cmp_terms(right, db, table, template) do {:ok, l ++ r} end end defp vdbe_cmp_terms({:binary, op, left, right}, db, table, template) when op in [:eq, :ne, :lt, :le, :gt, :ge] do env = %{db: db, frames: [template], group: nil, outer: nil} # Only the binary-collation hot path; user/column collations need the # collation callback resolved per comparison, which the tree walker owns. case {vdbe_operand(left, table, env), vdbe_operand(right, table, env), comparison_collation(left, right, env)} do {{:col, key, aff_a}, {:lit, value, aff_b}, :binary} -> {:ok, [{op, {:col, key, aff_a}, {:lit, value, aff_b}, :binary}]} {{:lit, value, aff_a}, {:col, key, aff_b}, :binary} -> {:ok, [{op, {:lit, value, aff_a}, {:col, key, aff_b}, :binary}]} _other -> :unsupported end end defp vdbe_cmp_terms(_other, _db, _table, _template), do: :unsupported defp vdbe_operand({:column, _qualifier, name} = expr, table, env) do key = Table.key(name) if Table.column(table, key), do: {:col, key, expr_affinity(expr, env)}, else: :error end defp vdbe_operand({:literal, value} = expr, _table, env) when not is_tuple(value) or value == nil, do: {:lit, value, expr_affinity(expr, env)} defp vdbe_operand(_expr, _table, _env), do: :error # ORDER BY plan: for each term, either reuse a projection register (integer # position, `ORDER BY 2`) or compile the (alias-substituted) expression to a # key `:eval` load. Only binary collation is handled; anything else declines # so the tree walker keeps ownership of collated ordering. defp vdbe_order([], _db, _table, _template, _columns, _names), do: {:ok, %{key_loads: [], directions: nil}} defp vdbe_order(order_by, db, table, template, columns, names) do env = %{db: db, frames: [template], group: nil, outer: nil} nproj = length(columns) # Key `:eval` registers sit above the projection regs and the two cmp scratch regs. key_base = nproj + 2 Enum.reduce_while(order_by, {[], [], key_base}, fn {expr, direction}, {loads, dirs, reg} -> if order_collation(expr, env, columns, names) == :binary do {load, next_reg} = vdbe_order_key(expr, db, table, template, columns, nproj, reg) {:cont, {[load | loads], [direction | dirs], next_reg}} else {:halt, :unsupported} end end) |> case do :unsupported -> :unsupported {loads, dirs, _reg} -> {:ok, %{key_loads: Enum.reverse(loads), directions: Enum.reverse(dirs)}} end end # `ORDER BY ` references the n-th projected column's register directly. defp vdbe_order_key({:literal, n}, _db, _table, _template, _columns, nproj, reg) when is_integer(n) and n >= 1 and n <= nproj, do: {{:reuse, n - 1}, reg} defp vdbe_order_key(expr, db, _table, template, columns, _nproj, reg) do subbed = substitute_aliases(expr, columns, [template]) {{:eval, fn row, rowid -> eval(subbed, vdbe_env(db, template, row, rowid)) end, reg}, reg + 1} end # Lays out the program: rewind; the filter (a `:cmp` chain or one `:filter` # closure, each failure jumping to Next); the projection loads; any ORDER BY # key `:eval` loads; ResultRow (projection + key registers); Next; Halt. defp vdbe_program(proj_loads, filter, key_loads) do nproj = length(proj_loads) proj_regs = Enum.to_list(0..(nproj - 1)) filter_ops = vdbe_filter_ops(filter, nproj) {key_eval_ops, key_regs} = vdbe_key_ops(key_loads, proj_regs) body = filter_ops ++ proj_loads ++ key_eval_ops ++ [{:result_row, proj_regs, key_regs}] next_addr = 1 + length(body) halt_addr = next_addr + 1 ops = [{:rewind, halt_addr}] ++ patch_filter_fail(body, next_addr) ++ [{:next, 1}, {:halt}] List.to_tuple(ops) end defp vdbe_filter_ops(:none, _scratch), do: [] defp vdbe_filter_ops({:filter, fun}, _scratch), do: [{:filter, fun, :fail}] defp vdbe_filter_ops({:cmp, terms}, scratch), do: Enum.flat_map(terms, &vdbe_cmp_ops(&1, scratch)) defp vdbe_cmp_ops({op, a, b, collation}, scratch) do {load_a, aff_a} = vdbe_operand_op(a, scratch) {load_b, aff_b} = vdbe_operand_op(b, scratch + 1) [load_a, load_b, {:cmp, op, scratch, aff_a, scratch + 1, aff_b, collation, :fail}] end # Returns the ORDER BY key-eval opcodes and the register list `result_row` # reads keys from (reused projection regs need no opcode). defp vdbe_key_ops(key_loads, proj_regs) do {ops, regs} = Enum.reduce(key_loads, {[], []}, fn {:reuse, proj_index}, {ops, regs} -> {ops, [Enum.at(proj_regs, proj_index) | regs]} {:eval, _fun, reg} = op, {ops, regs} -> {[op | ops], [reg | regs]} end) {Enum.reverse(ops), Enum.reverse(regs)} end defp vdbe_operand_op({:col, key, aff}, reg), do: {{:column, key, reg}, aff} defp vdbe_operand_op({:lit, value, aff}, reg), do: {{:value, value, reg}, aff} defp patch_filter_fail(ops, next_addr) do Enum.map(ops, fn {:cmp, op, ra, aa, rb, ab, coll, :fail} -> {:cmp, op, ra, aa, rb, ab, coll, next_addr} {:filter, fun, :fail} -> {:filter, fun, next_addr} other -> other end) end defp run_vdbe(%{table: table, program: program} = plan) do rows = ExSQL.Vdbe.run(Table.scan(table), program, plan.limit, plan.offset, plan.order) %Result{ command: :select, columns: plan.names, rows: rows, rows_affected: 0, affinities: plan.affinities } end defp maybe_reverse_unordered_envs(%{reverse_unordered_selects: false}, _stmt, _columns, envs), do: envs defp maybe_reverse_unordered_envs(db, stmt, columns, envs) do if stmt.order_by == [] or aggregate_query?(db, stmt, columns) do Enum.reverse(envs) else envs end end defp sql_value({:json, text}), do: text defp sql_value(value), do: value defp dml_result(%{count_changes: false}, _table, [], _returning_rows, command, count), do: %Result{command: command, rows_affected: count} defp dml_result(%{count_changes: true}, _table, [], _returning_rows, command, count) do %Result{ command: :select, columns: ["rows #{dml_count_changes_verb(command)}"], rows: [[count]], rows_affected: count, affinities: [:integer] } end defp dml_result(db, table, returning, returning_rows, _command, count) do template = table_frame(table, nil) columns = expand_columns(returning, [template]) names = Enum.map(columns, &result_column_name(db, [template], &1)) rows = Enum.map(returning_rows, fn {rowid, row} -> env = %{db: db, frames: [%{template | row: row, rowid: rowid}], group: nil, outer: nil} Enum.map(columns, fn {expr, _alias_name} -> expr |> eval(env) |> sql_value() end) end) template_env = %{db: db, frames: [template], group: nil, outer: nil} affinities = Enum.map(columns, fn {expr, _} -> expr_affinity(expr, template_env) end) %Result{ command: :select, columns: names, rows: rows, rows_affected: count, affinities: affinities } end defp dml_count_changes_verb(:insert), do: "inserted" defp dml_count_changes_verb(:update), do: "updated" defp dml_count_changes_verb(:delete), do: "deleted" defp update_assignment(table, {name, expr}) do case Table.column(table, name) do %{} = column -> if column.generated, do: fail("cannot UPDATE generated column \"#{column.name}\"") {:column, column, expr} nil -> key = Table.key(name) if key in @rowid_names and not table.without_rowid do {:rowid, key, expr} else fail("no such column: #{name}") end end end defp update_assignment_key({:column, column, _expr}), do: Table.key(column.name) defp update_assignment_key({:rowid, key, _expr}), do: key defp updated_row_and_rowid(table, assignments, row, env) do Enum.reduce(assignments, {row, :not_set}, fn {:column, column, expr}, {new_row, explicit_rowid} -> value = expr |> eval(env) |> Value.apply_affinity(column.affinity) {Map.put(new_row, Table.key(column.name), value), explicit_rowid} {:rowid, _key, expr}, {new_row, _explicit_rowid} -> value = expr |> eval(env) |> Value.apply_affinity(:integer) new_row = case table.rowid_alias do nil -> new_row alias_key -> Map.put(new_row, alias_key, value) end {new_row, value} end) end defp rebase_updated_row(current_row, table, assignments, new_row) do Enum.reduce(assignments, current_row, fn {:column, column, _expr}, row -> key = Table.key(column.name) Map.put(row, key, Map.fetch!(new_row, key)) {:rowid, _key, _expr}, row -> case table.rowid_alias do nil -> row alias_key -> Map.put(row, alias_key, Map.fetch!(new_row, alias_key)) end end) end defp update_rowid_value(:not_set, rowid), do: rowid defp update_rowid_value(explicit_rowid, _rowid), do: explicit_rowid defp update_conflict_rowid(_table, rowid, _row, :not_set) when is_nil(rowid), do: nil defp update_conflict_rowid(table, rowid, row, :not_set) do case table.rowid_alias do nil -> rowid alias_key -> Map.fetch!(row, alias_key) end end defp update_conflict_rowid(_table, rowid, _row, explicit_rowid), do: update_rowid_value(explicit_rowid, rowid) defp update_rowid_opts(opts, :not_set), do: opts defp update_rowid_opts(opts, explicit_rowid), do: Keyword.put(opts, :rowid, explicit_rowid) defp updated_rowid(table, old_rowid, row), do: updated_rowid(table, old_rowid, row, :not_set) defp updated_rowid(_table, _old_rowid, _row, explicit_rowid) when explicit_rowid != :not_set, do: explicit_rowid defp updated_rowid(table, old_rowid, row, :not_set) do case table.rowid_alias do nil -> old_rowid alias_key -> Map.fetch!(row, alias_key) end end # No LIMIT/OFFSET: the entire matched set is affected regardless of order, so # take the index/rowid access path (`planned_relation`) instead of a full # scan — the difference between O(matches·log n) and O(n) per statement. # Residual conjuncts the index doesn't cover are still applied by `filter`. defp dml_target_rows(db, table, %{limit: nil, offset: nil, schema: schema} = stmt) when schema in [nil, "main"] do {templates, frame_rows} = planned_relation(db, {:table, table.name, nil}, stmt.where, nil) filter = compile_scan_filter(precompile_scan_where(stmt.where, db, templates, nil)) matched = for [frame] <- frame_rows, keep_row?(filter, %{db: db, frames: [frame], group: nil, outer: nil}), do: {frame.rowid, frame_row_map(frame)} Enum.sort_by(matched, &elem(&1, 0)) end defp dml_target_rows(db, table, stmt) do # Build the frame template once and fastify/compile the predicate once, # rather than rebuilding the frame and re-resolving column names (with their # `downcase`) for every row — the same treatment SELECT scans get. template = table_frame(table, nil) filter = compile_scan_filter(precompile_scan_where(stmt.where, db, [template], nil)) matched = for {rowid, row} <- Table.scan(table), env = %{db: db, frames: [%{template | row: row, rowid: rowid}], group: nil, outer: nil}, keep_row?(filter, env), do: {rowid, row} limited = matched |> order_dml_targets(db, table, stmt.order_by) |> clamp(db, stmt.limit, stmt.offset) selected = MapSet.new(limited, &elem(&1, 0)) Enum.filter(matched, fn {rowid, _row} -> MapSet.member?(selected, rowid) end) end defp keep_row?(nil, _env), do: true defp keep_row?(filter, env), do: filter.(env) # Take the planner's rowid/index access path (not a full scan) when there's no # LIMIT/OFFSET to order by — so `UPDATE ... WHERE pk = ?` is an O(1) seek # instead of O(n), which (with incremental index maintenance) makes a table of # single-row updates O(n) rather than O(n²). defp update_target_rows( db, %Table{without_rowid: false} = table, %{from: nil, limit: nil, offset: nil, schema: schema} = stmt ) when schema in [nil, "main"] do {templates, frame_rows} = planned_relation(db, {:table, table.name, nil}, stmt.where, nil) filter = compile_scan_filter(precompile_scan_where(stmt.where, db, templates, nil)) matched = for [frame] <- frame_rows, env = %{db: db, frames: [frame], group: nil, outer: nil}, keep_row?(filter, env), do: {frame.rowid, frame_row_map(frame), env} finalize_update_targets(matched, db, stmt) end defp update_target_rows(db, table, %{from: nil} = stmt) do template = table_frame(table, nil) filter = compile_scan_filter(precompile_scan_where(stmt.where, db, [template], nil)) matched = table |> Table.scan() |> Enum.flat_map(fn {rowid, row} -> env = %{db: db, frames: [%{template | row: row, rowid: rowid}], group: nil, outer: nil} if keep_row?(filter, env), do: [{rowid, row, env}], else: [] end) finalize_update_targets(matched, db, stmt) end defp update_target_rows(db, table, stmt) do matched = table |> Table.scan() |> Enum.flat_map(fn {rowid, row} -> case update_match_env(db, table, rowid, row, stmt) do nil -> [] env -> [{rowid, row, env}] end end) finalize_update_targets(matched, db, stmt) end defp finalize_update_targets(matched, db, stmt) do limited = matched |> order_update_targets(stmt.order_by) |> clamp(db, stmt.limit, stmt.offset) selected = MapSet.new(limited, fn {rowid, _row, _env} -> rowid end) Enum.filter(matched, fn {rowid, _row, _env} -> MapSet.member?(selected, rowid) end) end defp update_match_env(db, table, rowid, row, %{from: from, where: where}) do target_frame = %{table_frame(table, nil) | row: row, rowid: rowid} {_templates, source_rows} = relation(db, from, nil) source_rows |> Enum.map(fn source_frames -> %{db: db, frames: [target_frame | source_frames], group: nil, outer: nil} end) |> Enum.filter(&matches_where?(where, &1)) |> List.last() end defp order_update_targets(rows, []), do: rows defp order_update_targets(rows, order_by) do Enum.sort(rows, fn {_rowid_a, _row_a, env_a}, {_rowid_b, _row_b, env_b} -> compare_term_values(order_by, env_a, env_b) end) end defp order_dml_targets(rows, _db, _table, []), do: rows defp order_dml_targets(rows, db, table, order_by) do Enum.sort(rows, fn {rowid_a, row_a}, {rowid_b, row_b} -> env_a = table_env(db, table, rowid_a, row_a) env_b = table_env(db, table, rowid_b, row_b) compare_term_values(order_by, env_a, env_b) end) end defp aggregate_query?(db, stmt, columns) do stmt.group_by != [] or stmt.having != nil or Enum.any?(columns, fn {expr, _} -> contains_aggregate?(expr, db) end) end defp collect_windows(columns) do columns |> Enum.flat_map(fn {expr, _} -> windows_in(expr) end) |> Enum.uniq() end defp windows_in({:window, _name, _args, _spec, _filter} = expr), do: [expr] defp windows_in(expr) when is_tuple(expr) do expr |> Tuple.to_list() |> Enum.flat_map(fn element when is_tuple(element) -> windows_in(element) elements when is_list(elements) -> Enum.flat_map(elements, &windows_in/1) _ -> [] end) end defp windows_in(_), do: [] defp contains_window?(%_struct{} = term), do: term |> Map.from_struct() |> contains_window?() defp contains_window?({:window, _name, _args, _spec, _filter}), do: true defp contains_window?(tuple) when is_tuple(tuple), do: tuple |> Tuple.to_list() |> contains_window?() defp contains_window?(list) when is_list(list), do: Enum.any?(list, &contains_window?/1) defp contains_window?(map) when is_map(map), do: map |> Map.values() |> contains_window?() defp contains_window?(_term), do: false defp apply_window_output_order(envs, []), do: envs defp apply_window_output_order(envs, [{:window, _name, _args, spec, _filter} | _]) do terms = Enum.map(spec.partition_by, &{&1, :asc}) ++ spec.order_by sort_envs_by_terms(envs, terms) end # Window functions may only appear in the SELECT list and ORDER BY; # WHERE, GROUP BY, and HAVING reject them. Subqueries get their own check # when they execute, so the walk does not descend into them. defp check_window_placement!(nil), do: :ok defp check_window_placement!({:window, name, _args, _spec, _filter}) do fail("misuse of window function #{name}()") end defp check_window_placement!(%Select{}), do: :ok defp check_window_placement!(%Compound{}), do: :ok defp check_window_placement!(%Values{}), do: :ok defp check_window_placement!(%With{}), do: :ok defp check_window_placement!(tuple) when is_tuple(tuple) do tuple |> Tuple.to_list() |> Enum.each(&check_window_placement!/1) end defp check_window_placement!(list) when is_list(list) do Enum.each(list, &check_window_placement!/1) end defp check_window_placement!(_other), do: :ok defp compute_windows([], _envs), do: %{} defp compute_windows(window_exprs, envs) do indexed_envs = Enum.with_index(envs) db = envs |> List.first(%{db: Database.new()}) |> Map.fetch!(:db) Enum.reduce(window_exprs, %{}, fn {:window, name, args, spec, filter} = expr, acc -> values = cond do aggregate_call?(db, name, args) -> aggregate_window_values(name, args, spec, filter, indexed_envs) name in @window_functions and filter == nil -> built_in_window_values(name, args, spec, indexed_envs) name in @window_functions -> fail("FILTER clause may only be used with aggregate window functions") true -> fail("#{name}() may not be used as a window function") end Enum.reduce(values, acc, fn {index, value}, acc -> Map.update(acc, index, %{expr => value}, &Map.put(&1, expr, value)) end) end) end defp aggregate_window_values(name, args, spec, filter, indexed_envs) do db = indexed_envs |> List.first({%{db: Database.new()}, 0}) |> elem(0) |> Map.fetch!(:db) case fetch_window_aggregate_function(db, name, args) do {:ok, %{kind: :incremental_window} = function} -> incremental_window_values(function, args, spec, filter, indexed_envs) _other -> spec |> window_partitions(indexed_envs) |> Enum.flat_map(fn {_key, ordered} -> Enum.with_index(ordered) |> Enum.map(fn {{env, index}, position} -> frame = spec |> window_frame_indexed(ordered, position) |> Enum.map(&elem(&1, 0)) |> filter_window_frame(filter) {index, aggregate(name, args, frame, env)} end) end) end end defp fetch_window_aggregate_function(db, name, args) when is_list(args), do: Database.fetch_aggregate_function(db, name, length(args)) defp fetch_window_aggregate_function(_db, _name, _args), do: :error defp filter_window_frame(frame, nil), do: frame defp filter_window_frame(frame, filter), do: Enum.filter(frame, &(truth(filter, &1) == true)) defp incremental_window_values(function, args, spec, filter, indexed_envs) do spec |> window_partitions(indexed_envs) |> Enum.flat_map(fn {_key, ordered} -> {_state, _previous_frame, values} = ordered |> Enum.with_index() |> Enum.reduce({call_incremental_window_init(function), [], []}, fn {{_env, index}, position}, {state, previous_frame, values} -> frame = spec |> window_frame_indexed(ordered, position) |> filter_window_indexed_frame(filter) state = previous_frame |> frame_difference(frame) |> Enum.reduce(state, fn {env, _index}, state -> call_incremental_window_update( function, :inverse, state, incremental_args(args, env) ) end) state = frame |> frame_difference(previous_frame) |> Enum.reduce(state, fn {env, _index}, state -> call_incremental_window_update(function, :step, state, incremental_args(args, env)) end) value = call_incremental_window_value(function, state) {state, frame, [{index, value} | values]} end) Enum.reverse(values) end) end defp filter_window_indexed_frame(frame, nil), do: frame defp filter_window_indexed_frame(frame, filter) do Enum.filter(frame, fn {env, _index} -> truth(filter, env) == true end) end defp frame_difference(left, right) do right_indexes = MapSet.new(right, &elem(&1, 1)) Enum.reject(left, fn {_env, index} -> MapSet.member?(right_indexes, index) end) end defp incremental_args(args, env), do: Enum.map(args, &eval(&1, env)) defp built_in_window_values(name, args, spec, indexed_envs) do validate_window_args!(name, args) spec |> window_partitions(indexed_envs) |> Enum.flat_map(fn {_key, ordered} -> peers = rank_peers(ordered, spec.order_by) ordered |> Enum.with_index() |> Enum.map(fn {{env, index}, position} -> frame = window_frame_indexed(spec, ordered, position) value = built_in_window_value(name, args, ordered, frame, peers, env, position) {index, value} end) end) end defp window_partitions(spec, indexed_envs) do indexed_envs |> Enum.group_by(fn {env, _index} -> Enum.map(spec.partition_by, &eval(&1, env)) end) |> Enum.map(fn {key, partition} -> {key, sort_indexed_envs_by_terms(partition, spec.order_by)} end) end defp rank_peers(ordered, []), do: Enum.map(ordered, fn _ -> 1 end) defp rank_peers(ordered, order_by) do ordered |> Enum.reduce({[], nil, 0, 0}, fn {env, _index}, {ranks, previous_key, row_number, rank} -> key = Enum.map(order_by, fn {expr, _direction} -> eval(expr, env) end) row_number = row_number + 1 rank = if key == previous_key, do: rank, else: row_number {[rank | ranks], key, row_number, rank} end) |> elem(0) |> Enum.reverse() end defp validate_window_args!("row_number", []), do: :ok defp validate_window_args!("row_number", _args), do: fail("wrong number of arguments to function row_number()") defp validate_window_args!("rank", []), do: :ok defp validate_window_args!("dense_rank", []), do: :ok defp validate_window_args!("percent_rank", []), do: :ok defp validate_window_args!("cume_dist", []), do: :ok defp validate_window_args!("lead", args) when length(args) in 1..3, do: :ok defp validate_window_args!("lag", args) when length(args) in 1..3, do: :ok defp validate_window_args!("first_value", [_arg]), do: :ok defp validate_window_args!("last_value", [_arg]), do: :ok defp validate_window_args!("nth_value", [_arg, _n]), do: :ok defp validate_window_args!("ntile", [_arg]), do: :ok defp validate_window_args!(name, _args), do: fail("wrong number of arguments to function #{name}()") defp built_in_window_value("row_number", [], _ordered, _frame, _peers, _env, position), do: position + 1 defp built_in_window_value("rank", [], _ordered, _frame, peers, _env, position), do: Enum.at(peers, position) defp built_in_window_value("dense_rank", [], _ordered, _frame, peers, _env, position), do: peers |> Enum.take(position + 1) |> Enum.uniq() |> length() defp built_in_window_value("lead", args, ordered, _frame, _peers, env, position) do offset = window_offset(args, env, 1) default = window_default(args, env) target = Enum.at(ordered, position + offset) window_arg_value(target, List.first(args), default) end defp built_in_window_value("lag", args, ordered, _frame, _peers, env, position) do offset = window_offset(args, env, 1) default = window_default(args, env) index = position - offset # Guard the negative index: `Enum.at/2` treats a negative index as counting # from the end, but a lag before the partition start must yield the default. target = if index >= 0, do: Enum.at(ordered, index) window_arg_value(target, List.first(args), default) end defp built_in_window_value("ntile", [arg], ordered, _frame, _peers, env, position) do buckets = eval(arg, env) unless is_integer(buckets) and buckets > 0 do fail("argument of ntile must be a positive integer") end ntile_bucket(position, length(ordered), buckets) end defp built_in_window_value("first_value", [arg], _ordered, frame, _peers, _env, _position) do window_arg_value(List.first(frame), arg, nil) end defp built_in_window_value("last_value", [arg], _ordered, frame, _peers, _env, _position) do frame |> List.last() |> window_arg_value(arg, nil) end defp built_in_window_value( "nth_value", [_arg, n_expr] = args, _ordered, frame, _peers, env, _position ) do n = eval(n_expr, env) unless is_integer(n) and n > 0 do fail("second argument to nth_value must be a positive integer") end frame |> Enum.at(n - 1) |> window_arg_value(List.first(args), nil) end defp built_in_window_value("percent_rank", [], ordered, _frame, peers, _env, position) do total = length(ordered) if total <= 1, do: 0.0, else: (Enum.at(peers, position) - 1) / (total - 1) end defp built_in_window_value("cume_dist", [], ordered, _frame, peers, _env, position) do rank = Enum.at(peers, position) last_peer = peers |> Enum.with_index() |> Enum.filter(fn {peer, _} -> peer == rank end) |> List.last() {_peer, last_index} = last_peer (last_index + 1) / length(ordered) end defp window_frame_indexed(%{frame: nil, order_by: []}, ordered, _position), do: ordered # The default frame with ORDER BY is RANGE BETWEEN UNBOUNDED PRECEDING AND # CURRENT ROW: peer rows of the current row are included. defp window_frame_indexed(%{frame: nil, order_by: order_by}, ordered, position) do {_first, last} = peer_bounds(ordered, order_by, position) Enum.take(ordered, last + 1) end defp window_frame_indexed(%{frame: frame, order_by: order_by}, ordered, position) do total = length(ordered) current_env = ordered |> Enum.at(position) |> elem(0) {first, last} = case frame.unit do :rows -> {rows_frame_index(frame.start, :start, current_env, position, total), rows_frame_index(frame.finish, :finish, current_env, position, total)} :range -> range_frame_bounds(frame, order_by, ordered, position, total) :groups -> groups_frame_bounds(frame, order_by, ordered, position, total) end positions = if last < first, do: [], else: Enum.to_list(first..last) positions = case Map.get(frame, :exclude, :no_others) do :no_others -> positions :current_row -> List.delete(positions, position) :group -> {peer_first, peer_last} = peer_bounds(ordered, order_by, position) Enum.reject(positions, &(&1 >= peer_first and &1 <= peer_last)) :ties -> {peer_first, peer_last} = peer_bounds(ordered, order_by, position) Enum.reject(positions, &(&1 != position and &1 >= peer_first and &1 <= peer_last)) end Enum.map(positions, &Enum.at(ordered, &1)) end defp rows_frame_index(:unbounded_preceding, _side, _env, _position, _total), do: 0 defp rows_frame_index(:unbounded_following, _side, _env, _position, total), do: total - 1 defp rows_frame_index(:current_row, _side, _env, position, _total), do: position defp rows_frame_index({:preceding, expr}, side, env, position, _total) do max(position - frame_offset!(expr, env, side), 0) end defp rows_frame_index({:following, expr}, side, env, position, total) do min(position + frame_offset!(expr, env, side), total - 1) end defp frame_offset!(expr, env, side) do case eval(expr, env) do n when is_integer(n) and n >= 0 -> n _ -> fail( "frame #{if side == :start, do: "starting", else: "ending"} offset must be a non-negative integer" ) end end # First and last position of the current row's peer group (rows with equal # ORDER BY keys). With no ORDER BY every row is a peer of every other. defp peer_bounds(ordered, [], _position), do: {0, length(ordered) - 1} defp peer_bounds(ordered, order_by, position) do keys = order_keys(ordered, order_by) current = Enum.at(keys, position) indexed = Enum.with_index(keys) first = Enum.find_value(indexed, fn {key, i} -> if key == current, do: i end) last = indexed |> Enum.reverse() |> Enum.find_value(fn {key, i} -> if key == current, do: i end) {first, last} end defp order_keys(ordered, order_by) do Enum.map(ordered, fn {env, _index} -> Enum.map(order_by, fn {expr, _direction} -> eval(expr, env) end) end) end # RANGE frames: CURRENT ROW means the current peer group; numeric offsets # require exactly one ORDER BY term and select rows whose key is within # the offset of the current row's key. defp range_frame_bounds(frame, order_by, ordered, position, total) do offset_frame? = match?({:preceding, _}, frame.start) or match?({:following, _}, frame.start) or match?({:preceding, _}, frame.finish) or match?({:following, _}, frame.finish) if offset_frame? do unless match?([_], order_by) do fail("RANGE with offset PRECEDING/FOLLOWING requires one ORDER BY expression") end range_offset_bounds(frame, order_by, ordered, position, total) else {peer_first, peer_last} = peer_bounds(ordered, order_by, position) first = case frame.start do :unbounded_preceding -> 0 :current_row -> peer_first :unbounded_following -> total - 1 end last = case frame.finish do :unbounded_following -> total - 1 :current_row -> peer_last :unbounded_preceding -> 0 end {first, last} end end defp range_offset_bounds(frame, [{order_expr, direction}], ordered, position, total) do current_env = ordered |> Enum.at(position) |> elem(0) current_key = eval(order_expr, current_env) if current_key == nil do # NULLs are peers of one another; an offset frame on a NULL key covers # exactly the NULL peer group. peer_bounds(ordered, [{order_expr, direction}], position) else # Signed distance from the current key, oriented along the sort # direction; NULL keys sort before everything and never match offsets. deltas = Enum.map(ordered, fn {env, _index} -> case eval(order_expr, env) do nil -> nil key -> if direction == :desc, do: numeric(current_key) - numeric(key), else: numeric(key) - numeric(current_key) end end) start_value = case frame.start do :unbounded_preceding -> nil {:preceding, expr} -> -range_offset!(expr, current_env, :start) :current_row -> 0 {:following, expr} -> range_offset!(expr, current_env, :start) end finish_value = case frame.finish do :unbounded_following -> nil {:preceding, expr} -> -range_offset!(expr, current_env, :finish) :current_row -> 0 {:following, expr} -> range_offset!(expr, current_env, :finish) end first = if start_value == nil do 0 else Enum.find_index(deltas, fn delta -> delta != nil and delta >= start_value end) || total end last = if finish_value == nil do total - 1 else case Enum.with_index(deltas) |> Enum.filter(fn {delta, _i} -> delta != nil and delta <= finish_value end) |> List.last() do {_delta, i} -> i nil -> -1 end end {first, last} end end defp numeric(value) when is_number(value), do: value defp numeric(_value), do: 0 defp range_offset!(expr, env, side) do case eval(expr, env) do n when is_number(n) and n >= 0 -> n _ -> fail( "frame #{if side == :start, do: "starting", else: "ending"} offset must be a non-negative number" ) end end # GROUPS frames count whole peer groups instead of rows. defp groups_frame_bounds(frame, order_by, ordered, position, total) do keys = order_keys(ordered, order_by) {group_numbers, _last_key, _n} = Enum.reduce(keys, {[], :none, -1}, fn key, {numbers, last_key, n} -> n = if key == last_key, do: n, else: n + 1 {[n | numbers], key, n} end) group_numbers = Enum.reverse(group_numbers) current_group = Enum.at(group_numbers, position) max_group = List.last(group_numbers) current_env = ordered |> Enum.at(position) |> elem(0) start_group = case frame.start do :unbounded_preceding -> 0 {:preceding, expr} -> current_group - frame_offset!(expr, current_env, :start) :current_row -> current_group {:following, expr} -> current_group + frame_offset!(expr, current_env, :start) end finish_group = case frame.finish do :unbounded_following -> max_group {:preceding, expr} -> current_group - frame_offset!(expr, current_env, :finish) :current_row -> current_group {:following, expr} -> current_group + frame_offset!(expr, current_env, :finish) end indexed = Enum.with_index(group_numbers) first = Enum.find_value(indexed, total, fn {group, i} -> if group >= start_group, do: i end) last = indexed |> Enum.reverse() |> Enum.find_value(-1, fn {group, i} -> if group <= finish_group, do: i end) {first, last} end defp ntile_bucket(position, total, buckets) when buckets >= total, do: position + 1 defp ntile_bucket(position, total, buckets) do base_size = div(total, buckets) larger_buckets = rem(total, buckets) larger_rows = larger_buckets * (base_size + 1) if position < larger_rows do div(position, base_size + 1) + 1 else larger_buckets + div(position - larger_rows, base_size) + 1 end end defp window_offset([_arg, offset_expr | _rest], env, _default) do case eval(offset_expr, env) do n when is_integer(n) and n >= 0 -> n _ -> 1 end end defp window_offset(_args, _env, default), do: default defp window_default([_arg, _offset, default_expr], env), do: eval(default_expr, env) defp window_default(_args, _env), do: nil defp window_arg_value(nil, _arg, default), do: default defp window_arg_value({target_env, _index}, arg, _default), do: eval(arg, target_env) # -- compound selects ---------------------------------------------------------- # # UNION/INTERSECT/EXCEPT run their distinct rows through a sorted temp # B-tree in SQLite, so their unordered output comes back sorted; UNION ALL # is plain concatenation. ORDER BY terms must name output columns (by # position, output name, or any component select's column name). defp compound_result(db, %Compound{} = stmt, outer) do {rows, leaf_names} = compound_rows(db, stmt, outer) names = hd(leaf_names) rows = rows |> compound_order(stmt.order_by, names, leaf_names) |> clamp(db, stmt.limit, stmt.offset) %Result{command: :select, columns: names, rows: rows, rows_affected: 0} end defp compound_rows(db, %Compound{} = stmt, outer) do {left_rows, left_names} = compound_rows(db, stmt.left, outer) {right_rows, right_names} = compound_rows(db, stmt.right, outer) if length(hd(left_names)) != length(hd(right_names)) do fail( "SELECTs to the left and right of #{compound_name(stmt.op)} " <> "do not have the same number of result columns" ) end rows = case stmt.op do :union_all -> left_rows ++ right_rows :union -> distinct_rows(left_rows ++ right_rows) :intersect -> right_set = row_key_set(right_rows) left_rows |> distinct_rows() |> Enum.filter(&MapSet.member?(right_set, row_key(&1))) :except -> right_set = row_key_set(right_rows) left_rows |> distinct_rows() |> Enum.reject(&MapSet.member?(right_set, row_key(&1))) end {rows, left_names ++ right_names} end defp compound_rows(db, stmt, outer) do result = query_result(db, stmt, outer) {result.rows, [result.columns]} end defp compound_name(:union_all), do: "UNION ALL" defp compound_name(:union), do: "UNION" defp compound_name(:intersect), do: "INTERSECT" defp compound_name(:except), do: "EXCEPT" defp distinct_rows(rows) do rows |> Enum.sort(&(compare_keys(&1, &2) != :gt)) |> Enum.reduce([], fn row, acc -> case acc do [previous | _] -> if compare_keys(row, previous) == :eq, do: acc, else: [row | acc] [] -> [row] end end) |> Enum.reverse() end # O(n+m) set membership for INTERSECT/EXCEPT: a canonical hash key per row # whose equality matches `compare_keys/2` (i.e. element-wise `Value.compare` # under the default binary collation, which is what compound set ops use). # NULLs are equal to each other; an integer and a numerically-equal float # collapse to the same key (Value.compare treats `1` and `1.0` as `:eq`); # text/JSON compare as text; blobs by bytes — each in its own rank bucket so # cross-type rows never collide. defp row_key_set(rows), do: MapSet.new(rows, &row_key/1) defp row_key(row), do: Enum.map(row, &value_key/1) defp value_key(nil), do: :null defp value_key(value) when is_integer(value), do: {:num, value} defp value_key(value) when is_float(value) do truncated = trunc(value) if truncated == value, do: {:num, truncated}, else: {:num, value} end defp value_key({:json, text}) when is_binary(text), do: {:text, text} defp value_key(value) when is_binary(value), do: {:text, value} defp value_key({:blob, bytes}), do: {:blob, bytes} defp value_key(other), do: {:other, other} defp compound_order(rows, [], _names, _leaf_names), do: rows defp compound_order(rows, order_by, names, leaf_names) do keys = order_by |> Enum.with_index(1) |> Enum.map(fn {{expr, direction}, term} -> {compound_order_position(expr, term, names, leaf_names), direction} end) Enum.sort(rows, fn a, b -> Enum.reduce_while(keys, true, fn {index, direction}, _ -> case Value.compare(Enum.at(a, index), Enum.at(b, index)) do :eq -> {:cont, true} :lt -> {:halt, direction == :asc} :gt -> {:halt, direction == :desc} end end) end) end defp compound_order_position({:literal, n}, term, names, _leaf_names) when is_integer(n) do if n < 1 or n > length(names) do fail( "#{ordinal(term)} ORDER BY term out of range - should be between 1 and #{length(names)}" ) end n - 1 end # The qualifier is ignored: `ORDER BY t1.log` matches output column `log`. defp compound_order_position({:column, _qualifier, name}, term, names, leaf_names) do key = Table.key(name) Enum.find_value([names | leaf_names], fn columns -> Enum.find_index(columns, &(Table.key(&1) == key)) end) || fail_unmatched_order_term(term) end defp compound_order_position(_expr, term, _names, _leaf_names), do: fail_unmatched_order_term(term) defp fail_unmatched_order_term(term) do fail("#{ordinal(term)} ORDER BY term does not match any column in the result set") end # -- FROM: tables, subqueries, joins ----------------------------------------- # # A relation is {templates, rows}: one frame template per source (carrying # name/columns/hidden), and each row as a list of frame instances. Joins are # nested loops; NATURAL/USING mark the right side's join columns hidden so # they appear once in `*` expansion and resolve unambiguously. # -- access planning ------------------------------------------------------------- # # The first sliver of a query planner: a single-table FROM whose WHERE # constrains the rowid with `=` becomes a point lookup into the row map # instead of a full scan. The WHERE clause is still applied afterwards, so # the lookup only needs to return a superset-safe restriction. # A provably constant-false WHERE (`WHERE NULL IS NOT NULL`, `WHERE NOT 35 IS # NOT NULL`, …) yields no rows regardless of the FROM, so skip materializing # the relation's rows entirely — critical for an unconstrained comma join, # whose cartesian product (e.g. `FROM t, t` over 1000 rows = 1e6 frames) would # otherwise be built only to be discarded. Templates still come from the table # schema (cheap) so the result columns resolve. Falls back to the normal # builder for FROM shapes whose templates aren't trivially derivable # (subqueries, views, table functions, NATURAL/USING joins). defp planned_relation(db, from, where, outer) do with true <- constant_false_where?(where, db), templates when is_list(templates) <- relation_templates(db, from) do {templates, []} else _ -> planned_relation_dispatch(db, from, where, outer) end end defp constant_false_where?(where, db) when not is_nil(where) do const_predicate?(where) and Value.truthy(eval(where, constant_env(db))) != true end defp constant_false_where?(_where, _db), do: false defp relation_templates(db, {:table, {:schema, schema, name}, alias_name}) do ensure_table_schema!(db, schema, name) table_template_only(db, Database.table_storage_key(schema, name), alias_name, name) end defp relation_templates(db, {:table, name, alias_name}) do relation_templates_unqualified(db, name, alias_name) end defp relation_templates( db, {:join, %{natural: false, left: false, right: false}, left, right, nil} ) do with l when is_list(l) <- relation_templates(db, left), r when is_list(r) <- relation_templates(db, right) do l ++ r else _ -> :unsupported end end defp relation_templates(_db, _other), do: :unsupported defp relation_templates_unqualified(db, name, alias_name) do key = relation_unqualified_table_key(db, name) table_template_only(db, key, alias_name, name) end defp table_template_only(db, table_key, alias_name, _name) do case Map.fetch(db.tables, table_key) do {:ok, table} -> [table_frame(table, alias_name)] :error -> :unsupported end end defp planned_relation_dispatch( db, {:table, {:schema, schema, name}, alias_name} = from, where, outer ) do ensure_table_schema!(db, schema, name) planned_named_relation( db, Database.table_storage_key(schema, name), alias_name, from, where, outer ) end defp planned_relation_dispatch(db, {:table, name, alias_name} = from, where, outer) do planned_named_relation(db, Table.key(name), alias_name, from, where, outer) end defp planned_relation_dispatch( db, {:join, type, left, {:table, name, alias_name}, constraint} = from, where, outer ) when constraint != nil or not (type.left or type.right) do # The rowid/index join planners drive a per-left-row probe of the right # table. That only pays off when a term actually correlates the right table # with the left (`right.col = left.col`, or an explicit ON/USING). Without a # correlation a single-table filter like `right.col IN (consts)` would be # re-probed once per left row, which is far slower than filtering the right # table once — so fall straight to the reordering/pushdown fallback. # A per-left-row probe must first materialize the whole left relation. When # the left is itself a multi-table comma join, that means building its cross # product up front — disastrous when those tables have no join predicates # tying them together (e.g. select4's 7-way joins blow up to ~10^7 rows). # For a plain inner comma join we therefore only peel when the left is a # single base table; a multi-table left goes to the reorder/hash-join # fallback, which orders and hashes the whole join globally. Explicit-ON and # OUTER joins keep the probe path (the inner-only fallback can't express # their semantics). peel? = join_correlated?(db, type, name, alias_name, constraint, where) and (constraint != nil or type.left or type.right or match?({:table, _, _}, left)) if peel? do # When the left is itself a join, build it once and share it across both # planners, otherwise an n-way join rebuilds the left ~2^n times. left_rel = case left do {:join, _t, _l, _r, _c} -> planned_relation(db, left, where, outer) _other -> nil end planned_rowid_join_relation( db, type, left, left_rel, name, alias_name, constraint, where, outer ) || planned_index_join_relation( db, type, left, left_rel, name, alias_name, constraint, where, outer ) || planned_left_rowid_inner_join_relation( db, type, left, {:table, name, alias_name}, constraint, where, outer ) || planned_left_index_inner_join_relation( db, type, left, {:table, name, alias_name}, constraint, where, outer ) || join_fallback_relation(db, from, where, outer) else join_fallback_relation(db, from, where, outer) end end defp planned_relation_dispatch( db, {:join, _type, _left, {:table, _name, _alias_name}, _constraint} = from, where, outer ), do: join_fallback_relation(db, from, where, outer) defp planned_relation_dispatch(db, from, where, outer), do: join_fallback_relation(db, from, where, outer) # True when a term ties the right table to another table — the precondition # for a useful per-left-row index/rowid probe. An explicit ON/USING join is # always treated as correlated; otherwise we look for an equality between a # right-table column and a column of some other table. defp join_correlated?(_db, %{natural: true}, _name, _alias_name, _constraint, _where), do: true defp join_correlated?(_db, _type, _name, _alias_name, constraint, _where) when constraint != nil, do: true defp join_correlated?(db, _type, name, alias_name, _constraint, where) do case plain_table(db, table_source_key(name)) do %Table{} = table -> right_qual = table_source_qualifier(name, alias_name) right_cols = MapSet.new(table.columns, &Table.key(&1.name)) Enum.any?(where_conjuncts(where), fn {:binary, :eq, {:column, _, _} = a, {:column, _, _} = b} -> ar = right_column?(a, right_qual, right_cols) br = right_column?(b, right_qual, right_cols) # A correlation only justifies peeling the right table to a # per-left-row probe when the probe can actually *seek* one side: # the right column drives a probe of this table, or the other # (left) column is the rowid/index prefix of its own table (the # left-index probe planner). A plain-column equi-join where neither # side is rowid/indexed is far better handled by the fallback's hash # join — peeling it strands the equi-join in a separate scope and # forces a cross product of the remaining tables. cond do ar and not br -> right_probe_column?(table, a) or column_probe_eligible?(db, b) br and not ar -> right_probe_column?(table, b) or column_probe_eligible?(db, a) true -> false end _other -> false end) _not_plain_table -> false end end defp right_column?({:column, nil, name}, _right_qual, right_cols), do: MapSet.member?(right_cols, Table.key(name)) defp right_column?({:column, qualifier, _name}, right_qual, _right_cols), do: Table.key(qualifier) == right_qual # The given column can drive a rowid/index seek probe on `table`. defp right_probe_column?(table, {:column, _, name}) do key = Table.key(name) cond do table.rowid_alias == key -> true key in @rowid_names and not table.without_rowid -> true Enum.any?(lookup_indexes(table), &(List.first(&1.columns) == key)) -> true true -> false end end # The other (left) side of a correlation can drive a left-table seek probe: # resolve the column's table(s) and ask whether it is rowid/index-eligible # there. Qualified names pick the named table; unqualified names check every # base table (the column name is unique to its table in practice). defp column_probe_eligible?(db, {:column, qualifier, _name} = column) do tables = case qualifier do nil -> for %Table{} = t <- Map.values(db.tables), do: t qual -> case plain_table(db, table_source_key(qual)) do %Table{} = t -> [t] _ -> [] end end Enum.any?(tables, &right_probe_column?(&1, column)) end # The generic nested-loop fallback (used once index/rowid join planning has # declined). Reorders comma joins to apply selective filters and equi-joins # early, pushes single-table WHERE conjuncts into each base table, and marks # equi-join nodes for hashing. When the join order is changed, the resulting # frames are permuted back to the original FROM order so projection (`*`, # positional refs) is unaffected. defp join_fallback_relation(db, from, where, outer) do {tree, perm} = optimize_inner_join(db, from, where) relation = relation(db, tree, outer) if perm, do: permute_relation(relation, perm), else: relation end # Reorders {tmpls, rows} so the per-source frames match `target_quals` # (original FROM order) instead of the optimized execution order. defp permute_relation({tmpls, rows}, target_quals) do order = Enum.map(target_quals, fn q -> Enum.find_index(tmpls, &(&1.name == q)) end) if Enum.any?(order, &is_nil/1) do {tmpls, rows} else new_tmpls = Enum.map(order, &Enum.at(tmpls, &1)) new_rows = Enum.map(rows, fn frames -> Enum.map(order, &Enum.at(frames, &1)) end) {new_tmpls, new_rows} end end # Composite hash-join key for one row; `:null` if any key column is NULL, # since SQL equality never matches on NULL. defp hash_join_key(exprs, env) do Enum.reduce_while(exprs, [], fn expr, acc -> case eval(expr, env) do nil -> {:halt, :null} value -> {:cont, [value | acc]} end end) end defp optimize_inner_join(db, from, where) do # An inner join's `ON` equi-conditions can drive hash joins just like a # comma join's WHERE keys, so explicit `a JOIN b ON a.k=b.k` hashes too. The # ON conjuncts feed *only* the hash-join annotation — not pushdown — so the # join node keeps evaluating its full ON exactly once (a side-effecting ON # term isn't duplicated into a base-table prefilter). WHERE conjuncts still # drive reordering and pushdown as before. # # Trees containing outer joins participate too: inner nodes hash from the # WHERE + inner-ON pool, LEFT nodes hash from their *own* ON only (see # annotate_hashjoins_with_qualifiers/3), pushdown skips the nullable side, # and reordering already restricts itself to pure comma joins. where_conjuncts = where_conjuncts(where) hash_conjuncts = where_conjuncts ++ inner_join_on_conjuncts(from) with {:ok, [_ | _] = sources, has_opaque?} <- join_sources(db, from) do source_lookup = source_lookup(sources, has_opaque?) {reordered, perm} = reorder_comma_join(from, where_conjuncts, source_lookup) nullable = nullable_qualifiers(reordered) tree = reordered |> pushdown_predicates(where_conjuncts, source_lookup, nullable) |> annotate_hashjoins(hash_conjuncts, source_lookup) {tree, perm} else _ -> {from, nil} end end # ON conjuncts of *inner* nodes only: an outer join's ON does not filter its # preserved side (unmatched rows null-extend instead), so feeding it into the # global pool would let an inner node below wrongly drop rows. defp inner_join_on_conjuncts({:join, %{left: false, right: false}, left, right, {:on, expr}}) do inner_join_on_conjuncts(left) ++ inner_join_on_conjuncts(right) ++ where_conjuncts(expr) end defp inner_join_on_conjuncts({:join, _type, left, right, _constraint}) do inner_join_on_conjuncts(left) ++ inner_join_on_conjuncts(right) end defp inner_join_on_conjuncts(_other), do: [] # Qualifiers whose frames can be null-extended: the right subtree of a LEFT # join, the left subtree of a RIGHT join (FULL marks both). A WHERE predicate # must not be pushed into these — filtering the nullable side pre-join turns # "matched then filtered out" rows into null-extended ones, which # null-tolerant predicates (IS NULL) would then keep. defp nullable_qualifiers({:join, type, left, right, _constraint}) do nullable_qualifiers(left) ++ nullable_qualifiers(right) ++ if(type.left, do: subtree_qualifiers(right), else: []) ++ if(type.right, do: subtree_qualifiers(left), else: []) end defp nullable_qualifiers(_other), do: [] # Greedy join ordering for pure comma joins over base tables: start from the # most-filtered table, then always extend along a join predicate (avoiding # cartesian blowups), preferring the most-filtered candidate. Inner joins are # freely reorderable and WHERE is re-applied downstream, so this only changes # execution order; `permute_relation/2` restores the original column order. defp reorder_comma_join(from, conjuncts, source_lookup) do with false <- source_lookup.has_opaque?, {:ok, [_, _ | _] = leaves} <- comma_join_leaves(from) do original = Enum.map(leaves, &leaf_qualifier/1) leaf_by_qual = Map.new(Enum.zip(original, leaves)) qsets = conjuncts |> Enum.map(&conjunct_qualifiers(&1, source_lookup)) |> Enum.reject(&(MapSet.size(&1) == 0)) filter_counts = filter_counts_by_qualifier(original, qsets) order = greedy_join_order(original, qsets, filter_counts) if order == original do {from, nil} else tree = order |> Enum.map(&Map.fetch!(leaf_by_qual, &1)) |> rebuild_left_deep() {tree, original} end else _ -> {from, nil} end end # Only pure comma joins (no NATURAL/USING/ON, no outer side) are freely # reorderable; a NATURAL join's shared-column semantics must not be rebuilt # into a plain cross join. defp comma_join_leaves({:join, %{natural: false, left: false, right: false}, left, right, nil}) do with {:ok, l} <- comma_join_leaves(left), {:ok, r} <- comma_join_leaves(right) do {:ok, l ++ r} end end defp comma_join_leaves({:table, _name, _alias} = table), do: {:ok, [table]} defp comma_join_leaves(_other), do: :no defp leaf_qualifier({:table, name, alias_name}), do: table_source_qualifier(name, alias_name) defp conjunct_qualifiers(expr, source_lookup) do expr |> expr_column_refs([]) |> Enum.reduce(MapSet.new(), fn ref, acc -> case column_owner(ref, source_lookup) do {:ok, qualifier} -> MapSet.put(acc, qualifier) _unknown -> acc end end) end defp greedy_join_order(quals, qsets, filter_counts) do start = Enum.max_by(quals, &Map.get(filter_counts, &1, 0)) extend_join_order( [start], List.delete(quals, start), MapSet.new([start]), join_adjacency(quals, qsets), filter_counts ) end defp extend_join_order(order, [], _chosen, _adjacency, _filter_counts), do: Enum.reverse(order) defp extend_join_order(order, remaining, chosen, adjacency, filter_counts) do connected_set = chosen |> Enum.reduce(MapSet.new(), fn qualifier, acc -> MapSet.union(acc, Map.get(adjacency, qualifier, MapSet.new())) end) |> MapSet.difference(chosen) connected = Enum.filter(remaining, &MapSet.member?(connected_set, &1)) pool = if connected == [], do: remaining, else: connected next = Enum.max_by(pool, &Map.get(filter_counts, &1, 0)) extend_join_order( [next | order], List.delete(remaining, next), MapSet.put(chosen, next), adjacency, filter_counts ) end defp filter_counts_by_qualifier(qualifiers, qsets) do empty_counts = Map.new(qualifiers, &{&1, 0}) Enum.reduce(qsets, empty_counts, fn qset, counts -> if MapSet.size(qset) == 1 do [qualifier] = MapSet.to_list(qset) Map.update!(counts, qualifier, &(&1 + 1)) else counts end end) end defp join_adjacency(qualifiers, qsets) do empty = Map.new(qualifiers, &{&1, MapSet.new()}) Enum.reduce(qsets, empty, fn qset, adjacency -> if MapSet.size(qset) > 1 do Enum.reduce(qset, adjacency, fn qualifier, adjacency -> Map.update!(adjacency, qualifier, &MapSet.union(&1, MapSet.delete(qset, qualifier))) end) else adjacency end end) end @comma_join_type %{natural: false, left: false, right: false} defp rebuild_left_deep([first | rest]) do Enum.reduce(rest, first, fn leaf, acc -> {:join, @comma_join_type, acc, leaf, nil} end) end defp pushdown_predicates(from, conjuncts, source_lookup, nullable_qualifiers) do nullable = MapSet.new(nullable_qualifiers) assignments = conjuncts |> Enum.filter(&pushable_predicate?/1) |> Enum.reduce(%{}, fn pred, acc -> case predicate_owner(pred, source_lookup) do {:ok, qualifier} -> # Never prefilter a null-extendable side: it would turn "matched # then filtered out" rows into null-extended ones. if MapSet.member?(nullable, qualifier) do acc else Map.update(acc, qualifier, [pred], &[pred | &1]) end :none -> acc end end) if assignments == %{}, do: from, else: apply_pushdown(from, assignments) end # Annotates each join node with the cross-table equi-join keys it can use for # a hash join. Only same-affinity numeric plain-column equalities qualify # (canonical storage, no collation, NULLs excluded), so the hash key is # exactly SQLite `=`. The full WHERE is re-applied later as a backstop. # # Inner nodes draw keys from the global pool (WHERE + inner-ON conjuncts); # a LEFT node draws keys ONLY from its own ON — its ON never filters, so no # foreign conjunct may be treated as its join condition — and the hash-join # execution null-extends unmatched left rows. RIGHT/FULL keep nested loops # (their unmatched-right bookkeeping needs every right row visited). defp annotate_hashjoins(from, conjuncts, source_lookup) do {from, _qualifiers} = annotate_hashjoins_with_qualifiers(from, conjuncts, source_lookup) from end defp annotate_hashjoins_with_qualifiers( {:join, type, left, right, constraint}, conjuncts, source_lookup ) do {left, la} = annotate_hashjoins_with_qualifiers(left, conjuncts, source_lookup) {right, ra} = annotate_hashjoins_with_qualifiers(right, conjuncts, source_lookup) qualifiers = la ++ ra node_conjuncts = cond do type.right -> [] type.left -> left_join_own_conjuncts(type, constraint) true -> conjuncts end equi = Enum.flat_map(node_conjuncts, fn conj -> case equi_join_key(conj, la, ra, source_lookup) do {:ok, lexpr, rexpr} -> [{lexpr, rexpr}] :no -> [] end end) node = if equi == [], do: {:join, type, left, right, constraint}, else: {:hashjoin, type, left, right, constraint, equi} {node, qualifiers} end defp annotate_hashjoins_with_qualifiers({:prefiltered, src, preds}, _conjuncts, _source_lookup), do: {{:prefiltered, src, preds}, subtree_qualifiers(src)} defp annotate_hashjoins_with_qualifiers( {:table, name, alias_name} = table, _conjuncts, _source_lookup ), do: {table, [table_source_qualifier(name, alias_name)]} defp annotate_hashjoins_with_qualifiers(from, _conjuncts, _source_lookup), do: {from, []} # NATURAL/USING joins share columns through `using_columns` rather than an ON # expression; keep those on the nested loop. defp left_join_own_conjuncts(%{natural: false}, {:on, expr}), do: where_conjuncts(expr) defp left_join_own_conjuncts(_type, _constraint), do: [] defp subtree_qualifiers({:join, _type, left, right, _constraint}), do: subtree_qualifiers(left) ++ subtree_qualifiers(right) # annotate_hashjoins/4 rewrites inner nodes bottom-up, so a node's left child # may already be a `:hashjoin`. Without this clause it falls through to the # `_opaque -> []` catch-all, the parent sees no left qualifiers, and every join # above the first hash join silently degrades to a cross product — the # dominant blowup in wide comma joins (select4's 8-way joins). defp subtree_qualifiers({:hashjoin, _type, left, right, _constraint, _keys}), do: subtree_qualifiers(left) ++ subtree_qualifiers(right) defp subtree_qualifiers({:prefiltered, src, _preds}), do: subtree_qualifiers(src) defp subtree_qualifiers({:table, name, alias_name}), do: [table_source_qualifier(name, alias_name)] defp subtree_qualifiers(_opaque), do: [] defp equi_join_key( {:binary, :eq, {:column, _, _} = e1, {:column, _, _} = e2}, la, ra, source_lookup ) do with {:ok, q1} <- column_owner(e1, source_lookup), {:ok, q2} <- column_owner(e2, source_lookup), a1 when a1 != nil <- column_affinity_in_sources(e1, source_lookup), ^a1 <- column_affinity_in_sources(e2, source_lookup), true <- hashable_equi_join?(a1, e1, e2, source_lookup) do cond do q1 in la and q2 in ra -> {:ok, e1, e2} q1 in ra and q2 in la -> {:ok, e2, e1} true -> :no end else _ -> :no end end defp equi_join_key(_conj, _la, _ra, _source_lookup), do: :no # The hash key is the raw evaluated value, so a value equal under `=` must # produce the same key. Numeric same-affinity columns are stored canonically # (the affinity equality above prevents int/float mixing). TEXT/BLOB are only # hashable under BINARY collation, where `=` is byte equality; NOCASE/RTRIM # would group equal values under different keys, so they keep the nested loop. defp hashable_equi_join?(affinity, _e1, _e2, _sources) when affinity in [:integer, :real, :numeric], do: true defp hashable_equi_join?(affinity, e1, e2, source_lookup) when affinity in [:text, :blob], do: binary_collation_in_sources?(e1, source_lookup) and binary_collation_in_sources?(e2, source_lookup) defp hashable_equi_join?(_affinity, _e1, _e2, _sources), do: false defp binary_collation_in_sources?({:column, nil, name}, source_lookup) do key = Table.key(name) case Map.get(source_lookup.by_column, key) do %{collations: collations} -> binary_collation_name?(Map.get(collations, key)) _ -> false end end defp binary_collation_in_sources?({:column, qualifier, name}, source_lookup) do qkey = Table.key(qualifier) key = Table.key(name) case Map.get(source_lookup.by_qualifier, qkey) do %{collations: collations} -> binary_collation_name?(Map.get(collations, key)) nil -> false end end defp column_affinity_in_sources({:column, nil, name}, source_lookup) do key = Table.key(name) case Map.get(source_lookup.by_column, key) do %{affinities: affinities} -> Map.get(affinities, key) _ -> nil end end defp column_affinity_in_sources({:column, qualifier, name}, source_lookup) do qkey = Table.key(qualifier) key = Table.key(name) case Map.get(source_lookup.by_qualifier, qkey) do %{affinities: affinities} -> Map.get(affinities, key) nil -> nil end end # Walks a join tree (outer joins included — reshaping is guarded separately: # reordering restricts itself to pure comma joins, pushdown skips nullable # qualifiers, and outer nodes hash only their own ON), returning the base # tables (qualifier key + column-key set) plus whether any opaque source # (subquery/view/CTE/table function) is present. defp join_sources(db, from) do inner_join_walk(db, from, [], false) end defp inner_join_walk(db, {:join, _type, left, right, _constraint}, acc, opaque?) do case inner_join_walk(db, left, acc, opaque?) do {:ok, acc, opaque?} -> inner_join_walk(db, right, acc, opaque?) end end defp inner_join_walk(db, {:table, {:schema, schema, name}, alias_name}, acc, opaque?) do inner_join_source( db, Database.table_storage_key(schema, name), name, alias_name, acc, opaque? ) end defp inner_join_walk(db, {:table, name, alias_name}, acc, opaque?) do inner_join_source( db, relation_unqualified_table_key(db, name), name, alias_name, acc, opaque? ) end defp inner_join_walk(_db, _other, acc, _opaque?), do: {:ok, acc, true} defp inner_join_source(db, table_key, name, alias_name, acc, opaque?) do case plain_table(db, table_key) do %Table{} = table -> source = %{ qualifier: table_source_qualifier(name, alias_name), columns: MapSet.new(table.columns, &Table.key(&1.name)), affinities: Map.new(table.columns, &{Table.key(&1.name), &1.affinity}), collations: Map.new(table.columns, &{Table.key(&1.name), &1.collate || "BINARY"}) } {:ok, [source | acc], opaque?} _not_plain_table -> # A view/CTE materialized as a table-name source: treat as opaque. {:ok, acc, true} end end defp source_lookup(sources, has_opaque?) do by_qualifier = Map.new(sources, &{&1.qualifier, &1}) by_column = Enum.reduce(sources, %{}, fn source, columns -> Enum.reduce(source.columns, columns, fn key, columns -> case Map.fetch(columns, key) do :error -> Map.put(columns, key, source) {:ok, _existing} -> Map.put(columns, key, :ambiguous) end end) end) %{has_opaque?: has_opaque?, by_qualifier: by_qualifier, by_column: by_column} end # A predicate is safe to evaluate early when it is row-local: no subquery, # aggregate, or window references. defp pushable_predicate?(expr), do: not expr_unpushable?(expr) defp expr_unpushable?({:subquery, _, _}), do: true defp expr_unpushable?({:subquery, _}), do: true defp expr_unpushable?({:exists, _}), do: true defp expr_unpushable?({:scalar_subquery, _}), do: true defp expr_unpushable?({:in, _expr, {:select, _}, _negated}), do: true defp expr_unpushable?({:window, _, _, _, _}), do: true defp expr_unpushable?({:function, name, _args}) when is_binary(name), do: name in @aggregate_functions defp expr_unpushable?(tuple) when is_tuple(tuple), do: tuple |> Tuple.to_list() |> Enum.any?(&expr_unpushable?/1) defp expr_unpushable?(list) when is_list(list), do: Enum.any?(list, &expr_unpushable?/1) defp expr_unpushable?(_other), do: false # Returns `{:ok, qualifier}` when every column the predicate references # belongs to a single base table; `:none` otherwise. defp predicate_owner(expr, source_lookup) do case expr_column_refs(expr, []) do [] -> :none refs -> owners = Enum.map(refs, &column_owner(&1, source_lookup)) case Enum.uniq(owners) do [{:ok, qualifier}] -> {:ok, qualifier} _ambiguous_or_unknown -> :none end end end defp column_owner({:column, nil, name}, source_lookup) do # Unqualified: only safe when ownership is unambiguous and there is no # opaque source that might also expose the column. key = Table.key(name) case Map.get(source_lookup.by_column, key) do %{qualifier: qualifier} when not source_lookup.has_opaque? -> {:ok, qualifier} _ -> :unknown end end defp column_owner({:column, qualifier, _name}, source_lookup) do key = Table.key(qualifier) case Map.get(source_lookup.by_qualifier, key) do %{qualifier: qualifier} -> {:ok, qualifier} nil -> :unknown end end defp expr_column_refs({:column, _qualifier, _name} = ref, acc), do: [ref | acc] defp expr_column_refs(tuple, acc) when is_tuple(tuple), do: tuple |> Tuple.to_list() |> Enum.reduce(acc, &expr_column_refs/2) defp expr_column_refs(list, acc) when is_list(list), do: Enum.reduce(list, acc, &expr_column_refs/2) defp expr_column_refs(_other, acc), do: acc defp apply_pushdown({:join, type, left, right, constraint}, assignments) do {:join, type, apply_pushdown(left, assignments), apply_pushdown(right, assignments), constraint} end defp apply_pushdown({:table, name, alias_name} = table, assignments) do case Map.get(assignments, table_source_qualifier(name, alias_name)) do nil -> table preds -> {:prefiltered, table, preds} end end defp apply_pushdown(other, _assignments), do: other defp table_source_key({:schema, schema, name}), do: Database.table_storage_key(schema, name) defp table_source_key(name), do: Table.key(name) defp table_source_name({:schema, _schema, name}), do: name defp table_source_name(name), do: name defp table_source_display(name, alias_name), do: alias_name || table_source_name(name) defp table_source_qualifier(name, alias_name), do: Table.key(table_source_display(name, alias_name)) defp planned_named_relation(db, key, alias_name, from, where, outer) do case plain_table(db, key) do %Table{} = table -> case table_access_path(db, table, where) do {:rowid_eq, {:literal, value}} -> planned_rowid_eq_relation(table, alias_name, value) {:rowid_in, exprs} -> planned_rowid_in_relation(table, alias_name, exprs) {:rowid_range, bounds} -> planned_rowid_range_relation(table, alias_name, bounds) _other -> planned_index_relation(db, table, alias_name, where) || relation(db, from, outer) end _not_plain_table -> relation(db, from, outer) end end defp planned_rowid_eq_relation(table, alias_name, value) do tmpl = table_frame(table, alias_name) rows = case Value.apply_affinity(value, :integer) do rowid when is_integer(rowid) -> case Table.fetch_row(table, rowid) do {:ok, row} -> [[%{tmpl | row: row, rowid: rowid}]] :error -> [] end _not_integer -> [] end {[tmpl], rows} end defp planned_rowid_in_relation(table, alias_name, exprs) do tmpl = table_frame(table, alias_name) rows = exprs |> rowid_in_lookup_values() |> Enum.flat_map(fn rowid -> case Table.fetch_row(table, rowid) do {:ok, row} -> [[%{tmpl | row: row, rowid: rowid}]] :error -> [] end end) {[tmpl], rows} end defp planned_rowid_range_relation(table, alias_name, bounds) do tmpl = table_frame(table, alias_name) rows = case rowid_range_lookup_bounds(bounds) do {:ok, bounds} -> table |> Table.scan() |> Enum.filter(fn {rowid, _row} -> rowid_range_match?(rowid, bounds) end) |> Enum.map(fn {rowid, row} -> [%{tmpl | row: row, rowid: rowid}] end) :error -> [] end {[tmpl], rows} end defp rowid_in_lookup_values(exprs) do exprs |> Enum.flat_map(fn {:literal, value} -> case Value.apply_affinity(value, :integer) do rowid when is_integer(rowid) -> [rowid] _not_integer -> [] end _expr -> [] end) |> Enum.uniq() |> Enum.sort() end defp rowid_range_lookup_bounds(bounds) do bounds |> Enum.reduce_while({:ok, []}, fn {op, {:literal, value}}, {:ok, acc} -> {:cont, {:ok, [{op, Value.apply_affinity(value, :integer)} | acc]}} _bound, _acc -> {:halt, :error} end) |> case do {:ok, bounds} -> {:ok, Enum.reverse(bounds)} :error -> :error end end defp rowid_range_match?(rowid, bounds) do Enum.all?(bounds, fn {op, value} -> Value.compare_op(op, rowid, value) == true end) end defp rowid_in_constraint(table, conjuncts) do Enum.find_value(conjuncts, fn {:in, expr, list, false} when is_list(list) -> expr = strip_collation(expr) if rowid_column_ref?(table, expr) and Enum.all?(list, &constant_expr?/1) do list end _other -> nil end) end defp rowid_or_literal_constraint(table, where) do where |> where_conjuncts() |> Enum.find_value(fn term -> disjuncts = where_disjuncts(term) with true <- multiple_terms?(disjuncts), values <- Enum.map(disjuncts, &rowid_or_literal_disjunct(table, &1)), true <- Enum.all?(values, &match?([_ | _], &1)) do Enum.flat_map(values, & &1) else _other -> nil end end) end defp rowid_or_literal_disjunct(table, {:binary, :eq, left, right}) do cond do rowid_column_ref?(table, strip_collation(left)) and constant_expr?(strip_collation(right)) -> [strip_collation(right)] rowid_column_ref?(table, strip_collation(right)) and constant_expr?(strip_collation(left)) -> [strip_collation(left)] true -> [] end end defp rowid_or_literal_disjunct(table, {:in, expr, list, false}) when is_list(list) do expr = strip_collation(expr) if rowid_column_ref?(table, expr) and Enum.all?(list, &constant_expr?/1) do list else [] end end defp rowid_or_literal_disjunct(_table, _term), do: [] defp rowid_range_constraints(table, conjuncts) do conjuncts |> Enum.flat_map(fn {:binary, op, left, right} when op in [:lt, :le, :gt, :ge] -> case rowid_range_constraint(table, left, right, op) do nil -> [] bound -> [bound] end {:between, expr, low, high, false} -> case rowid_between_constraint(table, expr, low, high) do nil -> [] bounds -> bounds end _other -> [] end) |> case do [] -> nil bounds -> bounds end end defp rowid_range_constraint(table, left, right, op) do left_base = strip_collation(left) right_base = strip_collation(right) cond do rowid_column_ref?(table, left_base) and constant_expr?(right_base) -> {op, right_base} rowid_column_ref?(table, right_base) and constant_expr?(left_base) -> {flip_range_op(op), left_base} true -> nil end end defp rowid_between_constraint(table, expr, low, high) do expr_base = strip_collation(expr) low_base = strip_collation(low) high_base = strip_collation(high) if rowid_column_ref?(table, expr_base) and constant_expr?(low_base) and constant_expr?(high_base) do [{:ge, low_base}, {:le, high_base}] end end defp planned_index_relation(db, table, alias_name, where) do # Plan against the index *definitions* first; only materialize entries once # an index lookup is actually chosen. Ensuring up front rebuilt (and then # discarded) every index on the table for queries that end up full-scanning. with {:ok, index, values} <- planned_index_lookup(table, where) do table = ensure_index_entries(db, table) index = ensured_index(table, index) tmpl = table_frame(table, alias_name) rows = db |> index_lookup_rowids(index, values) |> Enum.flat_map(fn rowid -> case Table.fetch_row(table, rowid) do {:ok, row} -> [[%{tmpl | row: row, rowid: rowid}]] :error -> [] end end) {[tmpl], rows} else _ -> nil end end defp planned_rowid_join_relation( db, type, left, left_rel, right_name, right_alias, constraint, where, outer ) do with join_kind when join_kind in [:inner, :left, :right, :full] <- indexed_join_kind(type), %Table{without_rowid: false} = table <- plain_table(db, table_source_key(right_name)) do rtmpl = table_frame(table, right_alias) {ltmpls, lrows} = left_rel || planned_relation(db, left, where, outer) using = using_columns(type, constraint, ltmpls, rtmpl) right_qualifier = table_source_qualifier(right_name, right_alias) rtmpl = %{rtmpl | hidden: MapSet.union(rtmpl.hidden, MapSet.new(using))} lookup_terms = join_lookup_terms(join_kind, constraint, where) ++ using_lookup_terms(using, right_qualifier) case join_rowid_lookup_plan(table, right_qualifier, ltmpls, lookup_terms) do {:ok, lookup_plan} -> {rows, matched_right_rowids} = Enum.map_reduce(lrows, MapSet.new(), fn lframes, matched_right_rowids -> env = %{db: db, frames: lframes, group: nil, outer: outer} raw_matches = join_rowid_matches(db, table, rtmpl, lookup_plan, env, constraint, using, outer) matches = case raw_matches do [] when join_kind in [:left, :full] -> [lframes ++ [null_frame(rtmpl)]] matches -> matches end matched_right_rowids = track_matched_right_rowids(raw_matches, matched_right_rowids) {matches, matched_right_rowids} end) |> then(fn {rows, matched_right_rowids} -> {Enum.flat_map(rows, & &1), matched_right_rowids} end) right_rows = if join_kind in [:right, :full] do for {rowid, row} <- Table.scan(table), not MapSet.member?(matched_right_rowids, rowid) do right_unmatched_row(ltmpls, %{rtmpl | row: row, rowid: rowid}, using) end else [] end {ltmpls ++ [rtmpl], rows ++ right_rows} :error -> nil end else _ -> nil end end defp join_rowid_matches(db, table, rtmpl, lookup_plan, env, constraint, using, outer) do lookup_plan |> join_rowid_lookup_rowids(table, env) |> Enum.flat_map(fn rowid -> case Table.fetch_row(table, rowid) do {:ok, row} -> rframe = %{rtmpl | row: row, rowid: rowid} if join_match?(db, constraint, using, env.frames, rframe, outer) do [env.frames ++ [rframe]] else [] end :error -> [] end end) end defp join_rowid_lookup_rowids({:eq, expr}, table, env) do case join_rowid_lookup_value(table, expr, env) do rowid when is_integer(rowid) -> [rowid] _not_integer -> [] end end defp join_rowid_lookup_rowids({:in, exprs}, table, env) do exprs |> Enum.flat_map(fn expr -> case join_rowid_lookup_value(table, expr, env) do rowid when is_integer(rowid) -> [rowid] _not_integer -> [] end end) |> Enum.uniq() |> Enum.sort() end defp join_rowid_lookup_rowids({:range, bounds}, table, env) do case join_rowid_range_bounds(table, bounds, env) do {:ok, bounds} -> table |> Table.scan() |> Enum.filter(fn {rowid, _row} -> rowid_range_match?(rowid, bounds) end) |> Enum.map(fn {rowid, _row} -> rowid end) :error -> [] end end defp join_rowid_range_bounds(table, bounds, env) do bounds |> Enum.reduce_while({:ok, []}, fn {op, expr}, {:ok, acc} -> case join_rowid_lookup_value(table, expr, env) do value when is_integer(value) -> {:cont, {:ok, [{op, value} | acc]}} _not_integer -> {:halt, :error} end end) |> case do {:ok, bounds} -> {:ok, Enum.reverse(bounds)} :error -> :error end end defp join_rowid_lookup_value(table, expr, env) do expr |> eval(env) |> Value.apply_affinity(:integer) |> rowid_lookup_value(table) end defp rowid_lookup_value(value, %{without_rowid: false}) when is_integer(value), do: value defp rowid_lookup_value(_value, _table), do: nil defp track_matched_right_rowids(raw_matches, matched_right_rowids) do Enum.reduce(raw_matches, matched_right_rowids, fn frames, matched_right_rowids -> frames |> List.last() |> Map.fetch!(:rowid) |> then(&MapSet.put(matched_right_rowids, &1)) end) end defp planned_index_join_relation( db, type, left, left_rel, right_name, right_alias, constraint, where, outer ) do with join_kind when join_kind in [:inner, :left, :right, :full] <- indexed_join_kind(type), %Table{} = table <- plain_table(db, table_source_key(right_name)) do rtmpl = table_frame(table, right_alias) {ltmpls, lrows} = left_rel || planned_relation(db, left, where, outer) using = using_columns(type, constraint, ltmpls, rtmpl) right_qualifier = table_source_qualifier(right_name, right_alias) rtmpl = %{rtmpl | hidden: MapSet.union(rtmpl.hidden, MapSet.new(using))} lookup_terms = join_lookup_terms(join_kind, constraint, where) ++ using_lookup_terms(using, right_qualifier) # Plan against the index definitions first; materialize entries only once # an index lookup is chosen (see planned_index_relation/4). with {:ok, index, lookup_plan} <- join_index_lookup_plan(table, right_name, right_alias, ltmpls, lookup_terms) do table = ensure_index_entries(db, table) index = ensured_index(table, index) {rows, matched_right_rowids} = Enum.map_reduce(lrows, MapSet.new(), fn lframes, matched_right_rowids -> env = %{db: db, frames: lframes, group: nil, outer: outer} raw_matches = db |> join_probe_rowids(index, table, lookup_plan, env) |> Enum.flat_map(fn rowid -> case Table.fetch_row(table, rowid) do {:ok, row} -> rframe = %{rtmpl | row: row, rowid: rowid} if join_match?(db, constraint, using, lframes, rframe, outer) do [lframes ++ [rframe]] else [] end :error -> [] end end) matches = case raw_matches do [] when join_kind in [:left, :full] -> [lframes ++ [null_frame(rtmpl)]] matches -> matches end matched_right_rowids = track_matched_right_rowids(raw_matches, matched_right_rowids) {matches, matched_right_rowids} end) |> then(fn {rows, matched_right_rowids} -> {Enum.flat_map(rows, & &1), matched_right_rowids} end) right_rows = if join_kind in [:right, :full] do for {rowid, row} <- Table.scan(table), not MapSet.member?(matched_right_rowids, rowid) do right_unmatched_row(ltmpls, %{rtmpl | row: row, rowid: rowid}, using) end else [] end {ltmpls ++ [rtmpl], rows ++ right_rows} else _ -> nil end else _ -> nil end end defp planned_left_index_inner_join_relation( db, type, {:table, left_name, left_alias}, {:table, right_name, right_alias}, constraint, where, outer ) do with :inner <- indexed_join_kind(type), %Table{} = table <- plain_table(db, table_source_key(left_name)) do ltmpl = table_frame(table, left_alias) rfrom = {:table, right_name, right_alias} {rtmpls, rrows} = planned_relation(db, rfrom, nil, outer) [rtmpl] = rtmpls using = using_columns(type, constraint, [ltmpl], rtmpl) left_qualifier = table_source_qualifier(left_name, left_alias) rtmpl = %{rtmpl | hidden: MapSet.union(rtmpl.hidden, MapSet.new(using))} lookup_terms = join_lookup_terms(:inner, constraint, where) ++ using_lookup_terms(using, left_qualifier) # Plan against the index definitions first; materialize entries only once # an index lookup is chosen (see planned_index_relation/4). with {:ok, index, lookup_plan} <- join_index_lookup_plan(table, left_name, left_alias, rtmpls, lookup_terms) do table = ensure_index_entries(db, table) index = ensured_index(table, index) rows = Enum.flat_map(rrows, fn [rframe] -> env = %{db: db, frames: [rframe], group: nil, outer: outer} db |> join_probe_rowids(index, table, lookup_plan, env) |> Enum.flat_map(fn rowid -> case Table.fetch_row(table, rowid) do {:ok, row} -> lframe = %{ltmpl | row: row, rowid: rowid} rframe = %{rframe | hidden: rtmpl.hidden} if join_match?(db, constraint, using, [lframe], rframe, outer) do [[lframe, rframe]] else [] end :error -> [] end end) end) {[ltmpl, rtmpl], rows} else _ -> nil end else _ -> nil end end defp planned_left_index_inner_join_relation( _db, _type, _left, _right, _constraint, _where, _outer ), do: nil defp planned_left_rowid_inner_join_relation( db, type, {:table, left_name, left_alias}, {:table, right_name, right_alias}, constraint, where, outer ) do with :inner <- indexed_join_kind(type), %Table{without_rowid: false} = table <- plain_table(db, table_source_key(left_name)) do ltmpl = table_frame(table, left_alias) rfrom = {:table, right_name, right_alias} {rtmpls, rrows} = planned_relation(db, rfrom, nil, outer) [rtmpl] = rtmpls using = using_columns(type, constraint, [ltmpl], rtmpl) left_qualifier = table_source_qualifier(left_name, left_alias) rtmpl = %{rtmpl | hidden: MapSet.union(rtmpl.hidden, MapSet.new(using))} lookup_terms = join_lookup_terms(:inner, constraint, where) ++ using_lookup_terms(using, left_qualifier) case join_rowid_lookup_plan(table, left_qualifier, rtmpls, lookup_terms) do {:ok, lookup_plan} -> rows = left_rowid_inner_join_rows( db, table, ltmpl, rtmpl, rrows, lookup_plan, constraint, using, outer ) {[ltmpl, rtmpl], rows} :error -> nil end else _ -> nil end end defp planned_left_rowid_inner_join_relation( _db, _type, _left, _right, _constraint, _where, _outer ), do: nil defp left_rowid_inner_join_rows( db, table, ltmpl, rtmpl, rrows, lookup_plan, constraint, using, outer ) do Enum.flat_map(rrows, fn [rframe] -> env = %{db: db, frames: [rframe], group: nil, outer: outer} lookup_plan |> join_rowid_lookup_rowids(table, env) |> Enum.flat_map( &left_rowid_inner_join_match( db, table, ltmpl, rtmpl, &1, rframe, constraint, using, outer ) ) end) end defp left_rowid_inner_join_match( db, table, ltmpl, rtmpl, rowid, rframe, constraint, using, outer ) do case Table.fetch_row(table, rowid) do {:ok, row} -> lframe = %{ltmpl | row: row, rowid: rowid} rframe = %{rframe | hidden: rtmpl.hidden} if join_match?(db, constraint, using, [lframe], rframe, outer) do [[lframe, rframe]] else [] end :error -> [] end end defp indexed_join_kind(%{natural: false, left: false, right: false}), do: :inner defp indexed_join_kind(%{natural: false, left: true, right: false}), do: :left defp indexed_join_kind(%{natural: false, left: false, right: true}), do: :right defp indexed_join_kind(%{natural: false, left: true, right: true}), do: :full defp indexed_join_kind(%{natural: true, left: false, right: false}), do: :inner defp indexed_join_kind(%{natural: true, left: true, right: false}), do: :left defp indexed_join_kind(%{natural: true, left: false, right: true}), do: :right defp indexed_join_kind(%{natural: true, left: true, right: true}), do: :full defp indexed_join_kind(_type), do: :unsupported defp join_lookup_terms(:inner, constraint, where), do: join_constraint_terms(constraint) ++ where_conjuncts(where) defp join_lookup_terms(join_kind, constraint, where) when join_kind in [:left, :right, :full], do: join_constraint_terms(constraint) ++ where_conjuncts(where) defp using_lookup_terms(using, right_qualifier) do Enum.map(using, fn key -> {:using_eq, key, right_qualifier} end) end defp join_index_lookup_plan(table, right_name, right_alias, ltmpls, terms) do case join_in_index_lookup(table, right_name, right_alias, ltmpls, terms) do {:ok, index, prefix, member, exprs} -> {:ok, index, {:in, prefix, member, exprs}} :error -> case join_index_lookup(table, right_name, right_alias, ltmpls, terms) do {:ok, index, prefix} -> {:ok, index, {:eq, prefix}} :error -> case join_range_index_lookup(table, right_name, right_alias, ltmpls, terms) do {:ok, index, prefix, range_member, bounds} -> {:ok, index, {:range, prefix, range_member, bounds}} :error -> :error end end end end defp join_probe_rowids(db, index, table, {:eq, prefix}, env) do lookup_values = Enum.map(prefix, &join_lookup_value(table, &1, env)) index_lookup_rowids(db, index, {:eq, lookup_values}) end defp join_probe_rowids(db, index, table, {:range, prefix, range_member, bounds}, env) do prefix_values = Enum.map(prefix, &join_lookup_value(table, &1, env)) bound_values = Enum.map(bounds, fn {op, expr} -> {op, join_lookup_value(table, {range_member, expr}, env)} end) lookup = if prefix_values == [] do {:range, bound_values} else {:member_range, prefix_values, bound_values} end index_lookup_rowids(db, index, lookup) end defp join_probe_rowids(db, index, table, {:in, prefix, member, exprs}, env) do prefix_values = Enum.map(prefix, &join_lookup_value(table, &1, env)) values = Enum.map(exprs, &join_lookup_value(table, {member, &1}, env)) lookup = if prefix_values == [] do {:in, values} else {:member_in, prefix_values, values} end index_lookup_rowids(db, index, lookup) end defp join_index_lookup(table, right_name, right_alias, ltmpls, terms) do right_qualifier = table_source_qualifier(right_name, right_alias) local_terms = Enum.map(terms, &localize_index_term(&1, right_qualifier, table)) lookup_indexes(table) |> Enum.filter(&join_index_usable?(table, &1, local_terms)) |> Enum.map(fn index -> member_collations = index_member_collation_pairs(index) prefix = member_collations |> Enum.reduce_while({:ok, []}, fn {member, collation}, {:ok, prefix} -> case join_member_equality_constraint( table, right_qualifier, ltmpls, terms, member, collation ) do nil -> {:halt, :error} expr -> {:cont, {:ok, [{member, expr} | prefix]}} end end) |> case do {:ok, prefix} -> Enum.reverse(prefix) :error -> [] end {index, prefix} end) |> Enum.filter(fn {_index, prefix} -> prefix != [] end) |> Enum.max_by( fn {index, prefix} -> {length(prefix), if(index.unique, do: 1, else: 0)} end, fn -> nil end ) |> case do nil -> :error {index, prefix} -> {:ok, index, prefix} end end defp join_range_index_lookup(table, right_name, right_alias, ltmpls, terms) do right_qualifier = table_source_qualifier(right_name, right_alias) local_terms = Enum.map(terms, &localize_index_term(&1, right_qualifier, table)) lookup_indexes(table) |> Enum.filter(&join_index_usable?(table, &1, local_terms)) |> Enum.map(fn index -> member_collations = index_member_collation_pairs(index) {prefix, remaining} = join_equality_prefix(table, right_qualifier, ltmpls, terms, member_collations) range_member = List.first(remaining) range_collation = index_member_collation(index, range_member) bounds = join_member_range_constraints( table, right_qualifier, terms, range_member, range_collation ) {index, prefix, range_member, bounds} end) |> Enum.filter(fn {_index, _prefix, range_member, bounds} -> range_member != nil and bounds != [] end) |> Enum.max_by( fn {index, prefix, _range_member, bounds} -> {length(prefix), length(bounds), if(index.unique, do: 1, else: 0)} end, fn -> nil end ) |> case do nil -> :error {index, prefix, range_member, bounds} -> {:ok, index, prefix, range_member, bounds} end end defp join_in_index_lookup(table, right_name, right_alias, ltmpls, terms) do right_qualifier = table_source_qualifier(right_name, right_alias) local_terms = Enum.map(terms, &localize_index_term(&1, right_qualifier, table)) lookup_indexes(table) |> Enum.filter(&join_index_usable?(table, &1, local_terms)) |> Enum.map(fn index -> member_collations = index_member_collation_pairs(index) {prefix, remaining} = join_equality_prefix(table, right_qualifier, ltmpls, terms, member_collations) member = List.first(remaining) member_collation = index_member_collation(index, member) exprs = join_member_in_constraint( table, right_qualifier, ltmpls, terms, member, member_collation ) {index, prefix, member, exprs} end) |> Enum.filter(fn {_index, _prefix, member, exprs} -> member != nil and exprs != [] end) |> Enum.max_by( fn {index, prefix, _member, exprs} -> {length(prefix), length(exprs), if(index.unique, do: 1, else: 0)} end, fn -> nil end ) |> case do nil -> :error {index, prefix, member, exprs} -> {:ok, index, prefix, member, exprs} end end defp index_member_collation_pairs(index) do members = index_members(index) collations = Map.get(index, :collations) || [] members |> Enum.with_index() |> Enum.map(fn {member, index} -> {member, Enum.at(collations, index)} end) end defp index_member_collation(_index, nil), do: nil defp index_member_collation(index, member) do members = index_members(index) collations = Map.get(index, :collations) || [] case Enum.find_index(members, &(&1 == member)) do nil -> nil member_index -> Enum.at(collations, member_index) end end defp join_equality_prefix(table, right_qualifier, ltmpls, terms, member_collations) do member_collations |> Enum.reduce_while({[], member_collations}, fn {member, collation} = member_collation, {prefix, [_member | rest]} -> case join_member_equality_constraint( table, right_qualifier, ltmpls, terms, member, collation ) do nil -> {:halt, {Enum.reverse(prefix), Enum.map([member_collation | rest], &elem(&1, 0))}} expr -> {:cont, {[{member, expr} | prefix], rest}} end end) |> case do {prefix, []} -> {Enum.reverse(prefix), []} other -> other end end defp join_lookup_value(table, {{:column, key}, expr}, env) do value = eval(expr, env) column = Table.column(table, key) Value.apply_affinity(value, column.affinity) end defp join_lookup_value(_table, {{:expr, _indexed_expr}, expr}, env), do: eval(expr, env) defp join_constraint_terms({:on, expr}), do: where_conjuncts(expr) defp join_constraint_terms(_constraint), do: [] defp join_member_equality_constraint( table, right_qualifier, ltmpls, terms, {:column, key}, index_collation ) do Enum.find_value(terms, fn term -> case join_equality_constraint(table, right_qualifier, ltmpls, term, index_collation) do {^key, expr} -> expr _ -> nil end end) end defp join_member_equality_constraint( table, right_qualifier, _ltmpls, terms, {:expr, indexed_expr}, _index_collation ) do Enum.find_value(terms, fn {:binary, :eq, left, right} -> cond do explicit_collation_node?(left) or explicit_collation_node?(right) -> nil expression_equivalent?(localize_index_term(left, right_qualifier, table), indexed_expr) and not expr_references_right_table?(right, right_qualifier, table) -> right expression_equivalent?(localize_index_term(right, right_qualifier, table), indexed_expr) and not expr_references_right_table?(left, right_qualifier, table) -> left true -> nil end _term -> nil end) end defp join_member_range_constraints(_table, _right_qualifier, _terms, nil, _collation), do: [] defp join_member_range_constraints( table, right_qualifier, terms, {:column, key}, index_collation ) do terms |> Enum.flat_map(fn {:binary, op, left, right} when op in [:lt, :le, :gt, :ge] -> cond do right_join_column(table, right_qualifier, [], strip_collation(left)) == key and collation_names_equal?(range_term_collation(table, key, left, right), index_collation) and not expr_references_right_table?(right, right_qualifier, table) -> [{op, right}] right_join_column(table, right_qualifier, [], strip_collation(right)) == key and collation_names_equal?(range_term_collation(table, key, left, right), index_collation) and not expr_references_right_table?(left, right_qualifier, table) -> [{flip_range_op(op), left}] true -> [] end {:between, expr, low, high, false} -> if right_join_column(table, right_qualifier, [], strip_collation(expr)) == key and collation_names_equal?( range_term_collation(table, key, expr, low), index_collation ) and collation_names_equal?( range_term_collation(table, key, expr, high), index_collation ) and not expr_references_right_table?(low, right_qualifier, table) and not expr_references_right_table?(high, right_qualifier, table) do [{:ge, low}, {:le, high}] else [] end _term -> [] end) end defp join_member_range_constraints( table, right_qualifier, terms, {:expr, indexed_expr}, _index_collation ) do terms |> Enum.flat_map(fn {:binary, op, left, right} when op in [:lt, :le, :gt, :ge] -> cond do explicit_collation_node?(left) or explicit_collation_node?(right) -> [] expression_equivalent?(localize_index_term(left, right_qualifier, table), indexed_expr) and not expr_references_right_table?(right, right_qualifier, table) -> [{op, right}] expression_equivalent?(localize_index_term(right, right_qualifier, table), indexed_expr) and not expr_references_right_table?(left, right_qualifier, table) -> [{flip_range_op(op), left}] true -> [] end {:between, expr, low, high, false} -> if not (explicit_collation_node?(expr) or explicit_collation_node?(low) or explicit_collation_node?(high)) and expression_equivalent?( localize_index_term(expr, right_qualifier, table), indexed_expr ) and not expr_references_right_table?(low, right_qualifier, table) and not expr_references_right_table?(high, right_qualifier, table) do [{:ge, low}, {:le, high}] else [] end _term -> [] end) end defp join_member_in_constraint(_table, _right_qualifier, _ltmpls, _terms, nil, _collation), do: [] defp join_member_in_constraint( table, right_qualifier, ltmpls, terms, {:column, key}, index_collation ) do Enum.find_value(terms, [], fn {:in, expr, list, false} when is_list(list) -> if right_join_column(table, right_qualifier, ltmpls, strip_collation(expr)) == key and not explicit_collation_node?(list) and collation_names_equal?(in_term_collation(table, key, expr), index_collation) and Enum.all?(list, &join_lookup_expr_usable?(&1, table, right_qualifier)) do list end term -> join_member_or_lookup_constraint( table, right_qualifier, ltmpls, key, index_collation, term ) end) end defp join_member_in_constraint( table, right_qualifier, _ltmpls, terms, {:expr, indexed_expr}, _index_collation ) do Enum.find_value(terms, [], fn {:in, expr, list, false} when is_list(list) -> if expression_equivalent?(localize_index_term(expr, right_qualifier, table), indexed_expr) and not explicit_collation_node?(list) and Enum.all?(list, &join_lookup_expr_usable?(&1, table, right_qualifier)) do list end term -> join_expression_or_lookup_constraint(table, right_qualifier, indexed_expr, term) end) end defp join_expression_or_lookup_constraint(table, right_qualifier, indexed_expr, term) do disjuncts = where_disjuncts(term) with true <- multiple_terms?(disjuncts), constraints <- Enum.map( disjuncts, &join_expression_or_lookup_disjunct(table, right_qualifier, indexed_expr, &1) ), true <- Enum.all?(constraints, &match?([_ | _], &1)) do Enum.flat_map(constraints, & &1) else _other -> nil end end defp join_expression_or_lookup_disjunct( table, right_qualifier, indexed_expr, {:binary, :eq, left, right} ) do cond do explicit_collation_node?(left) or explicit_collation_node?(right) -> nil expression_equivalent?(localize_index_term(left, right_qualifier, table), indexed_expr) and join_lookup_expr_usable?(right, table, right_qualifier) -> [right] expression_equivalent?(localize_index_term(right, right_qualifier, table), indexed_expr) and join_lookup_expr_usable?(left, table, right_qualifier) -> [left] true -> nil end end defp join_expression_or_lookup_disjunct( table, right_qualifier, indexed_expr, {:in, expr, list, false} ) when is_list(list) do if expression_equivalent?(localize_index_term(expr, right_qualifier, table), indexed_expr) and not explicit_collation_node?(list) and Enum.all?(list, &join_lookup_expr_usable?(&1, table, right_qualifier)) do list end end defp join_expression_or_lookup_disjunct( _table, _right_qualifier, _indexed_expr, _term ), do: nil defp join_member_or_lookup_constraint( table, right_qualifier, ltmpls, key, index_collation, term ) do disjuncts = where_disjuncts(term) with true <- multiple_terms?(disjuncts), constraints <- Enum.map( disjuncts, &join_member_or_lookup_disjunct( table, right_qualifier, ltmpls, key, index_collation, &1 ) ), true <- Enum.all?(constraints, &match?([_ | _], &1)) do Enum.flat_map(constraints, & &1) else _other -> nil end end defp join_member_or_lookup_disjunct( table, right_qualifier, ltmpls, key, index_collation, {:binary, :eq, left, right} ) do left_base = strip_collation(left) right_base = strip_collation(right) cond do right_join_column(table, right_qualifier, ltmpls, left_base) == key -> join_member_or_lookup_candidate( table, right_qualifier, index_collation, join_term_collation(table, key, left, right), right ) right_join_column(table, right_qualifier, ltmpls, right_base) == key -> join_member_or_lookup_candidate( table, right_qualifier, index_collation, join_term_collation(table, key, left, right), left ) true -> nil end end defp join_member_or_lookup_disjunct( table, right_qualifier, ltmpls, key, index_collation, {:in, expr, list, false} ) when is_list(list) do expr_base = strip_collation(expr) if right_join_column(table, right_qualifier, ltmpls, expr_base) == key and not explicit_collation_node?(list) and collation_names_equal?(in_term_collation(table, key, expr), index_collation) and Enum.all?(list, &join_lookup_expr_usable?(&1, table, right_qualifier)) do list end end defp join_member_or_lookup_disjunct( _table, _right_qualifier, _ltmpls, _key, _index_collation, _term ), do: nil defp join_member_or_lookup_candidate( table, right_qualifier, index_collation, term_collation, expr ) do if collation_names_equal?(term_collation, index_collation) and join_lookup_expr_usable?(expr, table, right_qualifier) do [expr] end end defp join_equality_constraint( table, right_qualifier, ltmpls, {:using_eq, key, right_qualifier}, _index_collation ) do if Table.column(table, key) != nil and Enum.any?(ltmpls, &visible?(&1, key)), do: {key, {:column, nil, key}}, else: nil end defp join_equality_constraint( table, right_qualifier, ltmpls, {:binary, :eq, left, right}, index_collation ) do cond do key = right_join_column(table, right_qualifier, ltmpls, strip_collation(left)) -> join_equality_constraint_candidate( table, key, right, right_qualifier, join_term_collation(table, key, left, right), index_collation ) key = right_join_column(table, right_qualifier, ltmpls, strip_collation(right)) -> join_equality_constraint_candidate( table, key, left, right_qualifier, join_term_collation(table, key, left, right), index_collation ) true -> nil end end defp join_equality_constraint(_table, _right_qualifier, _ltmpls, _term, _index_collation), do: nil defp join_equality_constraint_candidate( table, key, left_expr, right_qualifier, term_collation, index_collation ) do if expr_references_right_table?(left_expr, right_qualifier, table) or not collation_names_equal?(term_collation, index_collation) do nil else {key, left_expr} end end defp join_term_collation(table, key, left, right) do explicit_collation_name(left) || explicit_collation_name(right) || column_collation_name(table, key) || :binary end defp join_index_usable?(table, index, local_terms) do members = index_members(index) collations = Map.get(index, :collations) || [] members != [] and index_predicate_usable?(index, terms_where(local_terms)) and members |> Enum.zip(collations) |> Enum.all?(fn {{:column, key}, collation} -> if join_terms_have_explicit_collation_for_column?(table, key, local_terms) do join_terms_match_index_collation?(table, key, collation, local_terms) else collation_names_equal?(column_collation_name(table, key), collation) or join_terms_match_index_collation?(table, key, collation, local_terms) end {{:expr, _indexed_expr}, _collation} -> true end) end defp join_terms_have_explicit_collation_for_column?(table, key, terms) do Enum.any?(terms, fn {:binary, op, left, right} when op in [:eq, :lt, :le, :gt, :ge] -> (localized_column_expr?(table, key, strip_collation(left)) and explicit_collation_node?(left)) or (localized_column_expr?(table, key, strip_collation(right)) and explicit_collation_node?(right)) {:between, expr, _low, _high, false} -> localized_column_expr?(table, key, strip_collation(expr)) and explicit_collation_node?(expr) {:in, expr, list, false} when is_list(list) -> localized_column_expr?(table, key, strip_collation(expr)) and explicit_collation_node?(expr) term -> disjuncts = where_disjuncts(term) multiple_terms?(disjuncts) and join_terms_have_explicit_collation_for_column?(table, key, disjuncts) end) end defp join_terms_match_index_collation?(table, key, index_collation, terms) do Enum.any?(terms, fn {:binary, :eq, left, right} -> (localized_column_expr?(table, key, strip_collation(left)) or localized_column_expr?(table, key, strip_collation(right))) and collation_names_equal?( join_term_collation(table, key, left, right), index_collation ) {:binary, op, left, right} when op in [:lt, :le, :gt, :ge] -> (localized_column_expr?(table, key, strip_collation(left)) or localized_column_expr?(table, key, strip_collation(right))) and collation_names_equal?( range_term_collation(table, key, left, right), index_collation ) {:between, expr, low, high, false} -> localized_column_expr?(table, key, strip_collation(expr)) and collation_names_equal?( range_term_collation(table, key, expr, low), index_collation ) and collation_names_equal?( range_term_collation(table, key, expr, high), index_collation ) {:in, expr, list, false} when is_list(list) -> localized_column_expr?(table, key, strip_collation(expr)) and not explicit_collation_node?(list) and collation_names_equal?(in_term_collation(table, key, expr), index_collation) term -> join_or_terms_match_index_collation?(table, key, index_collation, term) or member_or_literal_constraint(term, table, key, index_collation) != nil end) end defp join_or_terms_match_index_collation?(table, key, index_collation, term) do disjuncts = where_disjuncts(term) multiple_terms?(disjuncts) and Enum.all?(disjuncts, fn {:binary, :eq, left, right} -> (localized_column_expr?(table, key, strip_collation(left)) or localized_column_expr?(table, key, strip_collation(right))) and collation_names_equal?(join_term_collation(table, key, left, right), index_collation) {:in, expr, list, false} when is_list(list) -> localized_column_expr?(table, key, strip_collation(expr)) and not explicit_collation_node?(list) and collation_names_equal?(in_term_collation(table, key, expr), index_collation) _other -> false end) end defp localized_column_expr?(table, key, {:column, nil, name}), do: Table.key(name) == key and Table.column(table, key) != nil defp localized_column_expr?(_table, _key, _expr), do: false defp terms_where([]), do: nil defp terms_where([term]), do: term defp terms_where([term | rest]), do: {:binary, :and, term, terms_where(rest)} defp localize_index_term({:column, qualifier, name} = expr, target_qualifier, table) do key = Table.key(name) cond do qualifier != nil and Table.key(qualifier) == target_qualifier -> {:column, nil, name} qualifier == nil and Table.column(table, key) != nil -> {:column, nil, name} true -> expr end end defp localize_index_term(expr, target_qualifier, table) when is_tuple(expr) do expr |> Tuple.to_list() |> Enum.map(&localize_index_term(&1, target_qualifier, table)) |> List.to_tuple() end defp localize_index_term(expr, target_qualifier, table) when is_list(expr), do: Enum.map(expr, &localize_index_term(&1, target_qualifier, table)) defp localize_index_term(expr, _target_qualifier, _table), do: expr defp join_rowid_lookup_plan(table, right_qualifier, ltmpls, terms) do case join_rowid_point_lookup_plan(table, right_qualifier, ltmpls, terms) do {:ok, lookup_plan} -> {:ok, lookup_plan} :error -> join_rowid_range_lookup_plan(table, right_qualifier, ltmpls, terms) end end defp join_rowid_point_lookup_plan(table, right_qualifier, ltmpls, terms) do Enum.find_value(terms, :error, fn term -> case join_rowid_lookup_term(table, right_qualifier, ltmpls, term) do {:eq, _expr} = lookup_plan -> {:ok, lookup_plan} {:in, _exprs} = lookup_plan -> {:ok, lookup_plan} _other -> nil end end) end defp join_rowid_range_lookup_plan(table, right_qualifier, ltmpls, terms) do terms |> Enum.flat_map(fn term -> case join_rowid_lookup_term(table, right_qualifier, ltmpls, term) do {:range, bounds} -> bounds _other -> [] end end) |> case do [] -> :error bounds -> {:ok, {:range, bounds}} end end defp join_rowid_lookup_term( table, right_qualifier, ltmpls, {:using_eq, key, right_qualifier} ) do if table.rowid_alias == key and Enum.any?(ltmpls, &has_column?(&1, key)) do {:eq, {:column, nil, key}} end end defp join_rowid_lookup_term(table, right_qualifier, ltmpls, {:binary, :eq, left, right}) do left_base = strip_collation(left) right_base = strip_collation(right) cond do right_join_rowid_column?(table, right_qualifier, ltmpls, left_base) -> join_rowid_equality_candidate(table, right_qualifier, ltmpls, right) right_join_rowid_column?(table, right_qualifier, ltmpls, right_base) -> join_rowid_equality_candidate(table, right_qualifier, ltmpls, left) true -> nil end end defp join_rowid_lookup_term( table, right_qualifier, ltmpls, {:in, expr, exprs, false} ) when is_list(exprs) do expr = strip_collation(expr) if right_join_rowid_column?(table, right_qualifier, ltmpls, expr) and Enum.all?(exprs, &join_rowid_lookup_expr_usable?(&1, table, right_qualifier, ltmpls)) do {:in, exprs} end end defp join_rowid_lookup_term(table, right_qualifier, ltmpls, {:binary, op, left, right}) when op in [:lt, :le, :gt, :ge] do left_base = strip_collation(left) right_base = strip_collation(right) cond do right_join_rowid_column?(table, right_qualifier, ltmpls, left_base) -> join_rowid_range_candidate(table, right_qualifier, ltmpls, op, right) right_join_rowid_column?(table, right_qualifier, ltmpls, right_base) -> join_rowid_range_candidate(table, right_qualifier, ltmpls, flip_range_op(op), left) true -> nil end end defp join_rowid_lookup_term( table, right_qualifier, ltmpls, {:between, expr, low, high, false} ) do expr = strip_collation(expr) if right_join_rowid_column?(table, right_qualifier, ltmpls, expr) and join_rowid_lookup_expr_usable?(low, table, right_qualifier, ltmpls) and join_rowid_lookup_expr_usable?(high, table, right_qualifier, ltmpls) do {:range, [{:ge, low}, {:le, high}]} end end defp join_rowid_lookup_term( table, right_qualifier, ltmpls, {:binary, :or, _left, _right} = term ) do disjuncts = where_disjuncts(term) with true <- multiple_terms?(disjuncts), values <- Enum.map(disjuncts, &join_rowid_or_lookup_disjunct(table, right_qualifier, ltmpls, &1)), true <- Enum.all?(values, &match?([_ | _], &1)) do {:in, Enum.flat_map(values, & &1)} else _other -> nil end end defp join_rowid_lookup_term(_table, _right_qualifier, _ltmpls, _term), do: nil defp join_rowid_equality_candidate(table, right_qualifier, ltmpls, expr) do if expr_references_right_table?(expr, right_qualifier, table) or not lookup_value_resolvable?(expr, ltmpls) do nil else {:eq, expr} end end defp join_rowid_range_candidate(table, right_qualifier, ltmpls, op, expr) do if join_rowid_lookup_expr_usable?(expr, table, right_qualifier, ltmpls) do {:range, [{op, expr}]} end end defp join_rowid_or_lookup_disjunct(table, right_qualifier, ltmpls, disjunct) do case join_rowid_lookup_term(table, right_qualifier, ltmpls, disjunct) do {:eq, expr} -> [expr] {:in, exprs} -> exprs _other -> [] end end defp join_rowid_lookup_expr_usable?(expr, table, right_qualifier, ltmpls), do: join_lookup_expr_usable?(expr, table, right_qualifier) and lookup_value_resolvable?(expr, ltmpls) defp join_lookup_expr_usable?(expr, table, right_qualifier), do: not expr_references_right_table?(expr, right_qualifier, table) # A per-left-row seek value must be evaluable against the left relation alone # (the probe runs before the right row is fetched). When a comma join is # reordered/split, a join predicate can correlate the right table with a table # that is NOT yet in this left relation (e.g. `t31.rowid = t55.b55` while only # t51/t29 are built); using it as a seek key would evaluate `t55.b55` against # the wrong frames and raise "no such column". Requiring every referenced # column to resolve in `ltmpls` declines those — the full WHERE is still # applied downstream once every table is joined, so results are unchanged. defp lookup_value_resolvable?(expr, ltmpls) do expr |> expr_column_refs([]) |> Enum.all?(&column_resolvable_in_templates?(&1, ltmpls)) end defp column_resolvable_in_templates?({:column, nil, name}, ltmpls) do key = Table.key(name) Enum.any?(ltmpls, &visible?(&1, key)) or (key in @rowid_names and Enum.any?(ltmpls, & &1.has_rowid)) end defp column_resolvable_in_templates?({:column, qualifier, name}, ltmpls) do qkey = Table.key(qualifier) key = Table.key(name) Enum.any?(ltmpls, fn t -> t.name == qkey and (has_column?(t, key) or (key in @rowid_names and t.has_rowid)) end) end defp right_join_rowid_column?(%{without_rowid: true}, _right_qualifier, _ltmpls, _expr), do: false defp right_join_rowid_column?(table, right_qualifier, ltmpls, {:column, qualifier, name}) do key = Table.key(name) cond do qualifier != nil -> Table.key(qualifier) == right_qualifier and rowid_column_ref?(table, {:column, nil, name}) table.rowid_alias == key -> not Enum.any?(ltmpls, &visible?(&1, key)) key in @rowid_names -> not Enum.any?(ltmpls, & &1.has_rowid) true -> false end end defp right_join_rowid_column?(_table, _right_qualifier, _ltmpls, _expr), do: false defp right_join_column(table, right_qualifier, ltmpls, {:column, qualifier, name}) do key = Table.key(name) cond do qualifier != nil -> if Table.key(qualifier) == right_qualifier and Table.column(table, key) != nil, do: key Table.column(table, key) != nil and not Enum.any?(ltmpls, &visible?(&1, key)) -> key true -> nil end end defp right_join_column(_table, _right_qualifier, _ltmpls, _expr), do: nil defp expr_references_right_table?({:column, qualifier, name}, right_qualifier, table) do cond do qualifier != nil -> Table.key(qualifier) == right_qualifier Table.column(table, name) -> true true -> false end end defp expr_references_right_table?(expr, right_qualifier, table) when is_tuple(expr) do expr |> Tuple.to_list() |> Enum.any?(&expr_references_right_table?(&1, right_qualifier, table)) end defp expr_references_right_table?(expr, right_qualifier, table) when is_list(expr), do: Enum.any?(expr, &expr_references_right_table?(&1, right_qualifier, table)) defp expr_references_right_table?(_expr, _right_qualifier, _table), do: false defp planned_index_lookup(table, where) do case table_access_path(nil, table, where) do {:index_eq, index, n_columns} -> with true <- index_lookup_usable?(table, index, n_columns, where), {:ok, values} <- index_lookup_values(table, index, n_columns, where) do {:ok, index, {:eq, values}} end {:index_range, index, bounds} -> with true <- index_range_lookup_usable?(table, index, where), {:ok, bounds} <- index_range_lookup_bounds(table, index, bounds) do {:ok, index, {:range, bounds}} end {:index_in, index, exprs} -> with true <- index_in_lookup_usable?(table, index, where), {:ok, values} <- index_in_lookup_values(table, index, exprs) do {:ok, index, {:in, values}} end {:expr_index_eq, index, _expr, {:literal, value}} -> with true <- expression_index_lookup_usable?(index, where) do {:ok, index, {:eq, [value]}} end {:expr_index_in, index, _expr, exprs} -> with true <- expression_index_lookup_usable?(index, where), {:ok, values} <- expression_index_in_lookup_values(exprs) do {:ok, index, {:in, values}} end {:expr_index_or, index, _expr, values} -> with true <- expression_index_lookup_usable?(index, where) do {:ok, index, {:in, values}} end {:expr_index_range, index, _expr, bounds} -> with true <- expression_index_lookup_usable?(index, where), {:ok, bounds} <- expression_index_range_lookup_bounds(bounds) do {:ok, index, {:range, bounds}} end {:index_member_eq, index, prefix} -> with true <- member_index_lookup_usable?(table, index, prefix, where), {:ok, values} <- index_member_lookup_values(table, prefix) do {:ok, index, {:eq, values}} end {:index_member_range, index, prefix, range_member, bounds} -> with true <- member_index_lookup_usable?(table, index, prefix, where), {:ok, prefix_values} <- index_member_lookup_values(table, prefix), {:ok, bounds} <- index_member_range_lookup_bounds(table, range_member, bounds) do {:ok, index, {:member_range, prefix_values, bounds}} end {:index_member_in, index, prefix, in_member, exprs} -> with true <- member_index_lookup_usable?(table, index, prefix, where), {:ok, prefix_values} <- index_member_lookup_values(table, prefix), {:ok, values} <- index_member_in_lookup_values(table, in_member, exprs) do {:ok, index, {:member_in, prefix_values, values}} end _other -> :error end end # The table itself — not shadowed by a CTE or one of the virtual tables. defp plain_table(db, key) do if Map.has_key?(db.ctes, key) or Map.has_key?(db.pending_ctes, key) or key in ["sqlite_schema", "sqlite_master", "sqlite_sequence"] do nil else Map.get(db.tables, key) end end # How a WHERE clause can drive access to `table`: # `{:rowid_eq, expr}`, `{:index_eq, index, n_columns}`, # `{:index_range, index, op_text}`, or `:scan`. # Each access-path analysis is computed lazily inside its `cond` branch and # short-circuits at the first match, rather than eagerly computing all ~15 # (several of which traverse the table's indexes) for every query. `eq_keys` # stays eager — it's cheap (a conjunct scan) and the common `index_eq` branch # needs it in both its guard and body. defp table_access_path(_db, table, where) do conjuncts = where_conjuncts(where) eq_keys = equality_keys(table, conjuncts) cond do (rowid_eq = rowid_equality_constraint(table, conjuncts)) != nil -> {:rowid_eq, rowid_eq} (rowid_in = rowid_in_constraint(table, conjuncts)) != nil -> {:rowid_in, rowid_in} (rowid_or = rowid_or_literal_constraint(table, where)) != nil -> {:rowid_in, rowid_or} (rowid_range = rowid_range_constraints(table, conjuncts)) != nil -> {:rowid_range, rowid_range} (member_in = best_member_in_index(table, conjuncts)) != nil -> {index, prefix, in_member, exprs} = member_in {:index_member_in, index, prefix, in_member, exprs} (member_range = best_member_range_index(table, conjuncts)) != nil -> {index, prefix, range_member, bounds} = member_range {:index_member_range, index, prefix, range_member, bounds} index = best_equality_index(table, eq_keys, where) -> prefix = Enum.take_while(index.columns, &MapSet.member?(eq_keys, &1)) {:index_eq, index, length(prefix)} result = range_access_path(table, conjuncts, where) -> result result = in_access_path(table, conjuncts, where) -> result result = or_access_path(table, where) -> result (expression_eq = best_expression_equality_index(table, conjuncts)) != nil -> {index, expr, value} = expression_eq {:expr_index_eq, index, expr, value} (expression_in = best_expression_in_index(table, conjuncts)) != nil -> {index, expr, values} = expression_in {:expr_index_in, index, expr, values} (expression_or = best_expression_or_literal_index(table, conjuncts)) != nil -> {index, expr, values} = expression_or {:expr_index_or, index, expr, values} (expression_range = best_expression_range_index(table, conjuncts)) != nil -> {index, expr, bounds} = expression_range {:expr_index_range, index, expr, bounds} (member_eq = best_member_equality_index(table, conjuncts)) != nil -> {index, prefix} = member_eq {:index_member_eq, index, prefix} true -> :scan end end defp equality_keys(table, conjuncts) do Enum.reduce(conjuncts, MapSet.new(), fn {:binary, :eq, left, right}, acc -> case equality_constraint(table, left, right) do {key, _expr} -> MapSet.put(acc, key) nil -> acc end _other, acc -> acc end) end defp range_access_path(table, conjuncts, where) do constraints = range_constraints(table, conjuncts) index = Enum.find(lookup_indexes(table), fn index -> Map.has_key?(constraints, List.first(index.columns)) and index_range_lookup_usable?(table, index, where) end) if index, do: {:index_range, index, Map.fetch!(constraints, List.first(index.columns))} end defp in_access_path(table, conjuncts, where) do constraints = in_constraints(table, conjuncts) index = Enum.find(lookup_indexes(table), fn index -> Map.has_key?(constraints, List.first(index.columns)) and index_in_lookup_usable?(table, index, where) end) if index, do: {:index_in, index, Map.fetch!(constraints, List.first(index.columns))} end defp or_access_path(table, where) do constraints = or_literal_constraints(table, where) index = Enum.find( lookup_indexes(table), &(Map.has_key?(constraints, List.first(&1.columns)) and index_in_lookup_usable?(table, &1, where)) ) if index, do: {:index_in, index, Map.fetch!(constraints, List.first(index.columns))} end defp rowid_equality_constraint(table, conjuncts) do Enum.find_value(conjuncts, fn {:binary, :eq, left, right} -> cond do rowid_column_ref?(table, left) and constant_expr?(right) -> right rowid_column_ref?(table, right) and constant_expr?(left) -> left true -> nil end _other -> nil end) end defp where_conjuncts(nil), do: [] defp where_conjuncts({:binary, :and, left, right}), do: where_conjuncts(left) ++ where_conjuncts(right) defp where_conjuncts(expr), do: [expr] defp where_disjuncts({:binary, :or, left, right}), do: where_disjuncts(left) ++ where_disjuncts(right) defp where_disjuncts(expr), do: [expr] defp multiple_terms?([_, _ | _]), do: true defp multiple_terms?(_terms), do: false defp range_constraints(table, conjuncts) do Enum.reduce(conjuncts, %{}, fn {:binary, op, left, right}, acc when op in [:lt, :le, :gt, :ge] -> case range_constraint(table, left, right, op) do {key, constraint} -> Map.update(acc, key, [constraint], &[constraint | &1]) nil -> acc end {:between, expr, low, high, false}, acc -> case between_constraint(table, expr, low, high) do {key, constraints} -> reversed_constraints = Enum.reverse(constraints) Map.update(acc, key, reversed_constraints, &(reversed_constraints ++ &1)) nil -> acc end _other, acc -> acc end) |> Map.new(fn {key, constraints} -> {key, Enum.reverse(constraints)} end) end defp in_constraints(table, conjuncts) do Enum.reduce(conjuncts, %{}, fn {:in, expr, list, false}, acc when is_list(list) -> case in_constraint(table, expr, list) do {key, exprs} -> Map.put(acc, key, exprs) nil -> acc end _other, acc -> acc end) end defp or_literal_constraints(table, where) do where |> where_conjuncts() |> Enum.reduce(%{}, fn term, acc -> Map.merge(acc, or_literal_constraint_group(table, term)) end) end defp or_literal_constraint_group(table, term) do terms = where_disjuncts(term) with true <- length(terms) > 1, constraints <- Enum.map(terms, &or_literal_constraint(table, &1)), true <- Enum.all?(constraints, &match?({_, [_ | _]}, &1)), [{key, _exprs} | _rest] <- constraints, true <- Enum.all?(constraints, &(elem(&1, 0) == key)) do %{key => Enum.flat_map(constraints, &elem(&1, 1))} else _other -> %{} end end defp constant_expr?({:literal, _value}), do: true defp constant_expr?({:collate, expr, _name}), do: constant_expr?(expr) defp constant_expr?(_expr), do: false defp rowid_column_ref?(table, {:column, nil, name}) do key = Table.key(name) key in @rowid_names or (table.rowid_alias != nil and key == table.rowid_alias) end defp rowid_column_ref?(_table, _expr), do: false defp constrained_column_key(table, left, right) do case equality_constraint(table, left, right) do {key, _expr} -> key nil -> nil end end defp equality_constraint(table, left, right) do left_base = strip_collation(left) right_base = strip_collation(right) cond do match?({:column, nil, _}, left_base) and constant_expr?(right_base) -> {:column, nil, name} = left_base if key = column_key(table, name), do: {key, right_base} match?({:column, nil, _}, right_base) and constant_expr?(left_base) -> {:column, nil, name} = right_base if key = column_key(table, name), do: {key, left_base} true -> nil end end defp strip_collation({:collate, expr, _name}), do: strip_collation(expr) defp strip_collation(expr), do: expr defp range_constraint(table, left, right, op) do left_base = strip_collation(left) right_base = strip_collation(right) cond do match?({:column, nil, _}, left_base) and constant_expr?(right_base) -> {:column, nil, name} = left_base if key = column_key(table, name), do: {key, {op, right_base}} match?({:column, nil, _}, right_base) and constant_expr?(left_base) -> {:column, nil, name} = right_base if key = column_key(table, name), do: {key, {flip_range_op(op), left_base}} true -> nil end end defp between_constraint(table, expr, low, high) do expr_base = strip_collation(expr) low_base = strip_collation(low) high_base = strip_collation(high) with {:column, nil, name} <- expr_base, key when not is_nil(key) <- column_key(table, name), true <- constant_expr?(low_base), true <- constant_expr?(high_base) do {key, [{:ge, low_base}, {:le, high_base}]} else _other -> nil end end defp flip_range_op(:lt), do: :gt defp flip_range_op(:le), do: :ge defp flip_range_op(:gt), do: :lt defp flip_range_op(:ge), do: :le defp flip_range_op(:eq), do: :eq defp in_constraint(table, expr, list) do expr = strip_collation(expr) with {:column, nil, name} <- expr, key when not is_nil(key) <- column_key(table, name), true <- Enum.all?(list, &constant_expr?/1) do {key, list} else _other -> nil end end defp or_literal_constraint(table, {:binary, :eq, left, right}) do case equality_constraint(table, left, right) do nil -> nil {key, expr} -> {key, [expr]} end end defp or_literal_constraint(table, {:in, expr, list, false}) when is_list(list) do case in_constraint(table, expr, list) do nil -> nil {key, exprs} -> {key, exprs} end end defp or_literal_constraint(_table, _expr), do: nil defp column_key(table, name) do if Table.column(table, name), do: Table.key(name) end defp index_lookup_usable?(table, index, n_columns, where) do prefix_columns = Enum.take(index.columns, n_columns) prefix_collations = Enum.take(Map.get(index, :collations) || [], n_columns) index_predicate_usable?(index, where) and index.columns != [] and equality_terms_match_index_collations?(table, prefix_columns, prefix_collations, where) end defp index_range_lookup_usable?(table, index, where) do first_key = List.first(index.columns) index_collation = List.first(Map.get(index, :collations) || []) index_predicate_usable?(index, where) and index.columns != [] and range_terms_match_index_collation?(table, first_key, index_collation, where) end defp index_in_lookup_usable?(table, index, where) do first_key = List.first(index.columns) index_collation = List.first(Map.get(index, :collations) || []) index_predicate_usable?(index, where) and index.columns != [] and (in_terms_match_index_collation?(table, first_key, index_collation, where) or or_literal_terms_match_index_collation?(table, first_key, index_collation, where)) end defp expression_index_lookup_usable?(index, where) do index_predicate_usable?(index, where) and match?([{:expr, _expr}], index_members(index)) end defp member_index_lookup_usable?(table, index, prefix, where), do: index_predicate_usable?(index, where) and member_index_collations_compatible?(table, index, prefix) defp member_index_collations_compatible?(table, index, prefix) do collations = Map.get(index, :collations) || [] prefix |> Enum.zip(collations) |> Enum.all?(fn {{{:column, key}, _expr}, collation} -> collation == column_collation_name(table, key) {{{:expr, _indexed_expr}, _expr}, _collation} -> true end) end defp index_predicate_usable?(%{where: nil}, _where), do: true defp index_predicate_usable?(%{where: index_where}, query_where) do query_terms = where_conjuncts(query_where) index_disjuncts = where_disjuncts(index_where) if length(index_disjuncts) > 1 do query_terms_imply_index_or?(query_terms, index_disjuncts) else index_where |> where_conjuncts() |> Enum.all?(&query_terms_imply_index_conjunct?(query_terms, &1)) end end defp query_terms_imply_index_term?(query_terms, index_term) do index_term |> where_conjuncts() |> Enum.all?(&query_terms_imply_index_conjunct?(query_terms, &1)) end defp query_terms_imply_index_conjunct?(query_terms, index_conjunct) do index_disjuncts = where_disjuncts(index_conjunct) if length(index_disjuncts) > 1 do query_terms_imply_index_or?(query_terms, index_disjuncts) else Enum.any?(query_terms, &query_term_implies_index_term?(&1, index_conjunct)) end end defp query_terms_imply_index_or?(query_terms, index_disjuncts) do Enum.any?(query_terms, &query_term_implies_index_or?(&1, index_disjuncts)) or Enum.any?(index_disjuncts, fn index_term -> query_terms_imply_index_term?(query_terms, index_term) end) end defp query_term_implies_index_term?(query_term, index_term) do query_term_implies_index_term?(query_term, index_term, :default) end defp query_term_implies_index_term?( {:binary, :and, left, right}, index_term, :default ) when not (is_tuple(index_term) and tuple_size(index_term) == 4 and elem(index_term, 0) == :binary and (elem(index_term, 1) == :and or elem(index_term, 1) == :or)), do: query_term_implies_index_term?(left, index_term, :default) or query_term_implies_index_term?(right, index_term, :default) defp query_term_implies_index_term?( {:binary, :or, left, right}, index_term, :default ) when not (is_tuple(index_term) and tuple_size(index_term) == 4 and elem(index_term, 0) == :binary and (elem(index_term, 1) == :and or elem(index_term, 1) == :or)), do: query_term_implies_index_term?(left, index_term, :default) and query_term_implies_index_term?(right, index_term, :default) defp query_term_implies_index_term?( {:binary, :and, _left, _right} = query_term, {:binary, :and, index_left, index_right}, _mode ), do: query_term_implies_index_term?(query_term, index_left) and query_term_implies_index_term?(query_term, index_right) defp query_term_implies_index_term?( {:binary, :or, _left, _right} = query_term, {:binary, :or, index_left, index_right}, _mode ), do: query_term_implies_index_term?(query_term, index_left) or query_term_implies_index_term?(query_term, index_right) defp query_term_implies_index_term?(query_term, index_term, :default) do expression_equivalent?(query_term, index_term) or range_term_implies_index_term?(query_term, index_term) or not_null_predicate_implied?(query_term, index_term) or range_predicate_implied?(query_term, index_term) or in_predicate_implied?(query_term, index_term) end defp range_term_implies_index_term?( {:binary, query_op, query_left, query_right}, {:binary, index_op, index_left, index_right} ) when query_op in [:lt, :le, :gt, :ge] and index_op in [:lt, :le, :gt, :ge] do cond do expression_equivalent?(query_left, index_left) and expression_equivalent?(query_right, index_right) -> range_operator_implies?(query_op, index_op) expression_equivalent?(query_left, index_right) and expression_equivalent?(query_right, index_left) -> query_op |> flip_range_op() |> range_operator_implies?(index_op) true -> false end end defp range_term_implies_index_term?(_query_term, _index_term), do: false defp range_operator_implies?(:gt, :gt), do: true defp range_operator_implies?(:gt, :ge), do: true defp range_operator_implies?(:ge, :ge), do: true defp range_operator_implies?(:lt, :lt), do: true defp range_operator_implies?(:lt, :le), do: true defp range_operator_implies?(:le, :le), do: true defp range_operator_implies?(_query_op, _index_op), do: false defp query_term_implies_index_or?(query_term, index_disjuncts) do query_term_implies_index_or?(query_term, index_disjuncts, :default) end defp query_term_implies_index_or?({:binary, :and, _, _} = query_term, index_disjuncts, _mode), do: Enum.any?(index_disjuncts, &query_term_implies_index_term?(query_term, &1)) defp query_term_implies_index_or?({:binary, :or, left, right}, index_disjuncts, _mode), do: query_term_implies_index_or?(left, index_disjuncts) and query_term_implies_index_or?(right, index_disjuncts) defp query_term_implies_index_or?(query_term, index_disjuncts, :default) do with {:ok, {index_expr, index_values}} <- or_literal_values(index_disjuncts), {:ok, {query_expr, query_values}} <- in_implication_values(query_term), true <- expression_equivalent?(query_expr, index_expr) do Enum.all?(query_values, fn query_value -> Enum.any?(index_values, &literal_values_equal?(query_value, &1)) end) else _other -> Enum.any?(index_disjuncts, &query_term_implies_index_term?(query_term, &1)) end end defp or_literal_values(index_disjuncts) do index_disjuncts |> Enum.reduce_while({:ok, nil, []}, fn {:binary, :eq, left, {:literal, value}}, {:ok, nil, values} -> {:cont, {:ok, left, [value | values]}} {:binary, :eq, left, {:literal, value}}, {:ok, expr, values} -> if expression_equivalent?(left, expr) do {:cont, {:ok, expr, [value | values]}} else {:halt, :error} end {:binary, :eq, {:literal, value}, right}, {:ok, nil, values} -> {:cont, {:ok, right, [value | values]}} {:binary, :eq, {:literal, value}, right}, {:ok, expr, values} -> if expression_equivalent?(right, expr) do {:cont, {:ok, expr, [value | values]}} else {:halt, :error} end {:in, expr, list, false}, {:ok, nil, values} when is_list(list) -> case literal_values(list) do {:ok, literal_values} -> {:cont, {:ok, expr, Enum.reverse(literal_values) ++ values}} :error -> {:halt, :error} end {:in, in_expr, list, false}, {:ok, expr, values} when is_list(list) -> with true <- expression_equivalent?(in_expr, expr), {:ok, literal_values} <- literal_values(list) do {:cont, {:ok, expr, Enum.reverse(literal_values) ++ values}} else _other -> {:halt, :error} end _other, _acc -> {:halt, :error} end) |> case do {:ok, nil, _values} -> :error {:ok, expr, values} -> {:ok, {expr, Enum.reverse(values)}} :error -> :error end end defp not_null_predicate_implied?(query_term, {:is_not, expr, {:literal, nil}}), do: comparison_excludes_null?(query_term, expr) defp not_null_predicate_implied?(_query_term, _index_term), do: false defp literal_boolean_constraint({:not, expr}), do: {:ok, {expr, false}} defp literal_boolean_constraint(_query_term), do: :error defp comparison_excludes_null?({:not, {:is, query_expr, {:literal, nil}}}, expr), do: expression_equivalent?(query_expr, expr) defp comparison_excludes_null?({:not, {:is_not, _query_expr, {:literal, nil}}}, _expr), do: false defp comparison_excludes_null?({:binary, op, left, right}, expr) when op in [:eq, :ne, :lt, :le, :gt, :ge], do: expression_equivalent?(left, expr) or expression_equivalent?(right, expr) defp comparison_excludes_null?({:between, between_expr, _low, _high, _negated}, expr), do: expression_equivalent?(between_expr, expr) defp comparison_excludes_null?({:in, in_expr, _source, _negated}, expr), do: expression_equivalent?(in_expr, expr) defp comparison_excludes_null?({:is, is_expr, {:literal, value}}, expr) when not is_nil(value), do: expression_equivalent?(is_expr, expr) defp comparison_excludes_null?({:is, {:literal, value}, is_expr}, expr) when not is_nil(value), do: expression_equivalent?(is_expr, expr) defp comparison_excludes_null?({:like, like_expr, _pattern, _escape, _negated}, expr), do: expression_equivalent?(like_expr, expr) defp comparison_excludes_null?({:glob, glob_expr, _pattern, _negated}, expr), do: expression_equivalent?(glob_expr, expr) defp comparison_excludes_null?({:regexp, regexp_expr, _pattern, _negated}, expr), do: expression_equivalent?(regexp_expr, expr) defp comparison_excludes_null?(query_term, expr) do with {:ok, {query_expr, _truth}} <- literal_boolean_constraint(query_term) do expression_equivalent?(query_expr, expr) else _other -> false end end defp range_predicate_implied?(query_term, index_term) do with {:ok, query_constraints} <- comparison_constraints(query_term), {:ok, index_constraints} <- comparison_constraints(index_term) do Enum.all?(index_constraints, fn index_constraint -> Enum.any?(query_constraints, &comparison_constraint_implies?(&1, index_constraint)) end) else _other -> false end end defp comparison_constraints({:binary, op, left, {:literal, value}}) when op in [:eq, :lt, :le, :gt, :ge], do: {:ok, [{left, op, value}]} defp comparison_constraints({:binary, op, {:literal, value}, right}) when op in [:eq, :lt, :le, :gt, :ge], do: {:ok, [{right, flip_range_op(op), value}]} defp comparison_constraints({:between, expr, {:literal, low}, {:literal, high}, false}), do: {:ok, [{expr, :ge, low}, {expr, :le, high}]} defp comparison_constraints(_term), do: :error defp comparison_constraint_implies?( {query_expr, query_op, query_value}, {index_expr, index_op, index_value} ) do expression_equivalent?(query_expr, index_expr) and comparable_implication_values?(query_value, index_value) and comparison_constraint_implies?(query_op, query_value, index_op, index_value) end defp comparable_implication_values?(left, right) when is_number(left) and is_number(right), do: true defp comparable_implication_values?(left, right) when is_binary(left) and is_binary(right), do: true defp comparable_implication_values?(_left, _right), do: false defp comparison_constraint_implies?(:eq, query_value, index_op, index_value), do: Value.compare_op(index_op, query_value, index_value) == true defp comparison_constraint_implies?(query_op, query_value, index_op, index_value) when query_op in [:gt, :ge] and index_op in [:gt, :ge], do: lower_bound_implies?(query_op, Value.compare(query_value, index_value), index_op) defp comparison_constraint_implies?(query_op, query_value, index_op, index_value) when query_op in [:lt, :le] and index_op in [:lt, :le], do: upper_bound_implies?(query_op, Value.compare(query_value, index_value), index_op) defp comparison_constraint_implies?(_query_op, _query_value, _index_op, _index_value), do: false defp lower_bound_implies?(:gt, cmp, :gt), do: cmp in [:gt, :eq] defp lower_bound_implies?(:ge, :gt, :gt), do: true defp lower_bound_implies?(:gt, cmp, :ge), do: cmp in [:gt, :eq] defp lower_bound_implies?(:ge, cmp, :ge), do: cmp in [:gt, :eq] defp lower_bound_implies?(_query_op, _cmp, _index_op), do: false defp upper_bound_implies?(:lt, cmp, :lt), do: cmp in [:lt, :eq] defp upper_bound_implies?(:le, :lt, :lt), do: true defp upper_bound_implies?(:lt, cmp, :le), do: cmp in [:lt, :eq] defp upper_bound_implies?(:le, cmp, :le), do: cmp in [:lt, :eq] defp upper_bound_implies?(_query_op, _cmp, _index_op), do: false defp in_predicate_implied?(query_term, {:in, index_expr, index_list, false}) when is_list(index_list) do with {:ok, index_values} <- literal_values(index_list), {:ok, {query_expr, query_values}} <- in_implication_values(query_term), true <- expression_equivalent?(query_expr, index_expr) do Enum.all?(query_values, fn query_value -> Enum.any?(index_values, &literal_values_equal?(query_value, &1)) end) else _other -> false end end defp in_predicate_implied?(_query_term, _index_term), do: false defp in_implication_values({:binary, :eq, left, {:literal, value}}), do: {:ok, {left, [value]}} defp in_implication_values({:binary, :eq, {:literal, value}, right}), do: {:ok, {right, [value]}} defp in_implication_values({:in, expr, list, false}) when is_list(list) do case literal_values(list) do {:ok, values} -> {:ok, {expr, values}} :error -> :error end end defp in_implication_values(_term), do: :error defp literal_values(list) do list |> Enum.reduce_while({:ok, []}, fn {:literal, value}, {:ok, values} -> {:cont, {:ok, [value | values]}} _expr, _acc -> {:halt, :error} end) |> case do {:ok, values} -> {:ok, Enum.reverse(values)} :error -> :error end end defp literal_values_equal?(left, right), do: Value.compare_op(:eq, left, right) == true defp range_terms_match_index_collation?(table, key, index_collation, where) do terms = where |> where_conjuncts() |> Enum.flat_map(fn {:binary, op, left, right} when op in [:lt, :le, :gt, :ge] -> case range_constraint(table, left, right, op) do {^key, _constraint} -> [{left, right}] _other -> [] end {:between, expr, low, high, false} -> case between_constraint(table, expr, low, high) do {^key, _constraints} -> [{expr, low}, {expr, high}] _other -> [] end _other -> [] end) terms != [] and Enum.all?(terms, fn {left, right} -> collation_names_equal?(range_term_collation(table, key, left, right), index_collation) end) end defp range_term_collation(table, key, left, right) do explicit_collation_name(left) || explicit_collation_name(right) || column_collation_name(table, key) || :binary end defp in_terms_match_index_collation?(table, key, index_collation, where) do terms = where |> where_conjuncts() |> Enum.flat_map(fn {:in, expr, list, false} when is_list(list) -> case in_constraint(table, expr, list) do {^key, _exprs} -> [{expr, list}] _other -> [] end _other -> [] end) terms != [] and Enum.all?(terms, fn {expr, list} -> not explicit_collation_node?(list) and collation_names_equal?(in_term_collation(table, key, expr), index_collation) end) end defp in_term_collation(table, key, expr), do: explicit_collation_name(expr) || column_collation_name(table, key) || :binary defp or_literal_terms_match_index_collation?(table, key, index_collation, where) do terms = where |> where_conjuncts() |> Enum.flat_map(fn term -> disjuncts = where_disjuncts(term) if multiple_terms?(disjuncts) do Enum.flat_map(disjuncts, fn {:binary, :eq, left, right} -> case equality_constraint(table, left, right) do {^key, _expr} -> [{:eq, left, right}] _other -> [] end {:in, expr, list, false} when is_list(list) -> case in_constraint(table, expr, list) do {^key, _exprs} -> [{:in, expr, list}] _other -> [] end _other -> [] end) else [] end end) terms != [] and Enum.all?(terms, fn {:eq, left, right} -> collation_names_equal?( equality_term_collation(table, key, left, right), index_collation ) {:in, expr, list} -> not explicit_collation_node?(list) and collation_names_equal?(in_term_collation(table, key, expr), index_collation) end) end defp equality_terms_match_index_collations?(table, keys, collations, where) do keys |> Enum.zip(collations) |> Enum.all?(fn {key, index_collation} -> terms = equality_terms_for_key(table, key, where) terms != [] and Enum.all?(terms, fn {_expr, term_collation} -> collation_names_equal?(term_collation, index_collation) end) end) end defp equality_terms_for_key(table, key, where) do where |> where_conjuncts() |> Enum.flat_map(fn {:binary, :eq, left, right} -> case equality_constraint(table, left, right) do {^key, expr} -> [{expr, equality_term_collation(table, key, left, right)}] _other -> [] end _other -> [] end) end defp equality_term_collation(table, key, left, right) do explicit_collation_name(left) || explicit_collation_name(right) || column_collation_name(table, key) || :binary end defp explicit_collation_name({:collate, _expr, name}), do: name defp explicit_collation_name(expr) when is_tuple(expr) do expr |> Tuple.to_list() |> Enum.find_value(&explicit_collation_name/1) end defp explicit_collation_name(expr) when is_list(expr), do: Enum.find_value(expr, &explicit_collation_name/1) defp explicit_collation_name(_expr), do: nil defp collation_names_equal?(left, right), do: normalize_collation_name(left) == normalize_collation_name(right) defp normalize_collation_name(nil), do: "binary" defp normalize_collation_name(:binary), do: "binary" defp normalize_collation_name(name) when is_binary(name), do: String.downcase(name) defp normalize_collation_name(name), do: name defp explicit_collation_node?({:collate, _expr, _name}), do: true defp explicit_collation_node?(expr) when is_tuple(expr) do expr |> Tuple.to_list() |> Enum.any?(&explicit_collation_node?/1) end defp explicit_collation_node?(expr) when is_list(expr), do: Enum.any?(expr, &explicit_collation_node?/1) defp explicit_collation_node?(_expr), do: false defp best_expression_equality_index(table, conjuncts) do lookup_indexes(table) |> Enum.find_value(fn index -> case index_members(index) do [{:expr, indexed_expr}] -> expression_equality_constraint(index, indexed_expr, conjuncts) _other -> nil end end) end defp best_member_equality_index(table, conjuncts) do lookup_indexes(table) |> Enum.find_value(fn index -> members = index_members(index) cond do members == [] -> nil Enum.all?(members, &match?({:column, _key}, &1)) -> nil true -> case member_equality_prefix(table, members, conjuncts) do [] -> nil prefix -> {index, prefix} end end end) end defp member_equality_prefix(table, members, conjuncts) do members |> Enum.reduce_while([], fn member, acc -> case member_equality_constraint(table, member, conjuncts) do {:ok, expr} -> {:cont, [{member, expr} | acc]} :error -> {:halt, acc} end end) |> Enum.reverse() end defp best_member_range_index(table, conjuncts) do lookup_indexes(table) |> Enum.find_value(fn index -> members = index_members(index) cond do members == [] -> nil true -> prefix = member_equality_prefix(table, members, conjuncts) range_member = Enum.at(members, length(prefix)) cond do prefix == [] or is_nil(range_member) -> nil bounds = member_range_constraints(table, index, range_member, conjuncts) -> {index, prefix, range_member, bounds} true -> nil end end end) end defp best_member_in_index(table, conjuncts) do lookup_indexes(table) |> Enum.find_value(fn index -> members = index_members(index) cond do members == [] -> nil true -> prefix = member_equality_prefix(table, members, conjuncts) in_member = Enum.at(members, length(prefix)) cond do prefix == [] or is_nil(in_member) -> nil exprs = member_in_constraint(table, index, in_member, conjuncts) -> {index, prefix, in_member, exprs} true -> nil end end end) end defp member_equality_constraint(table, {:column, key}, conjuncts) do Enum.find_value(conjuncts, :error, fn {:binary, :eq, left, right} -> constrained_key = constrained_column_key(table, left, right) cond do constrained_key != key -> nil explicit_collation_node?(left) or explicit_collation_node?(right) -> nil match?({:column, nil, _}, left) -> {:ok, right} true -> {:ok, left} end _other -> nil end) end defp member_equality_constraint(_table, {:expr, indexed_expr}, conjuncts) do Enum.find_value(conjuncts, :error, fn {:binary, :eq, left, right} -> cond do expression_equivalent?(left, indexed_expr) and constant_expr?(right) and not explicit_collation_node?(right) -> {:ok, right} expression_equivalent?(right, indexed_expr) and constant_expr?(left) and not explicit_collation_node?(left) -> {:ok, left} true -> nil end _other -> nil end) end defp member_range_constraints(table, index, {:column, key}, conjuncts) do members = index_members(index) member_index = Enum.find_index(members, &(&1 == {:column, key})) index_collation = Enum.at(Map.get(index, :collations) || [], member_index || 0) conjuncts |> Enum.flat_map(fn {:binary, op, left, right} when op in [:lt, :le, :gt, :ge] -> case range_constraint(table, left, right, op) do {^key, bound} -> if collation_names_equal?( range_term_collation(table, key, left, right), index_collation ) do [bound] else [] end _other -> [] end {:between, expr, low, high, false} -> case between_range_constraint(table, key, expr, low, high, index_collation) do nil -> [] bounds -> bounds end _other -> [] end) |> case do [] -> nil bounds -> bounds end end defp member_range_constraints(_table, _index, {:expr, indexed_expr}, conjuncts) do case expression_range_constraints(indexed_expr, conjuncts) do [] -> nil bounds -> bounds end end defp member_range_constraints(_table, _index, _member, _conjuncts), do: nil defp between_range_constraint(table, key, expr, low, high, index_collation) do expr_base = strip_collation(expr) low_base = strip_collation(low) high_base = strip_collation(high) with {:column, nil, name} <- expr_base, ^key <- column_key(table, name), true <- constant_expr?(low_base), true <- constant_expr?(high_base), true <- collation_names_equal?(range_term_collation(table, key, expr, low), index_collation), true <- collation_names_equal?(range_term_collation(table, key, expr, high), index_collation) do [{:ge, low_base}, {:le, high_base}] else _other -> nil end end defp member_in_constraint(table, index, {:column, key}, conjuncts) do index_collation = index_member_collation(index, {:column, key}) Enum.find_value(conjuncts, fn {:in, expr, list, false} when is_list(list) -> case in_constraint(table, expr, list) do {^key, exprs} -> if not explicit_collation_node?(list) and collation_names_equal?(in_term_collation(table, key, expr), index_collation) do exprs end _other -> nil end term -> member_or_literal_constraint(term, table, key, index_collation) end) end defp member_in_constraint(_table, _index, {:expr, indexed_expr}, conjuncts) do Enum.find_value(conjuncts, fn term -> case term do {:in, expr, list, false} when is_list(list) -> if expression_equivalent?(expr, indexed_expr) and not explicit_collation_node?(list) and Enum.all?(list, &constant_expr?/1) do list end _other -> disjuncts = where_disjuncts(term) with true <- multiple_terms?(disjuncts), {:ok, {query_expr, values}} <- or_literal_values(disjuncts), true <- expression_equivalent?(query_expr, indexed_expr) do Enum.map(values, &{:literal, &1}) else _other -> nil end end end) end defp member_in_constraint(_table, _index, _member, _conjuncts), do: nil defp member_or_literal_constraint(term, table, key, index_collation) do disjuncts = where_disjuncts(term) with true <- multiple_terms?(disjuncts), constraints <- Enum.map(disjuncts, &member_or_literal_disjunct(table, key, index_collation, &1)), true <- Enum.all?(constraints, &match?([_ | _], &1)) do Enum.flat_map(constraints, & &1) else _other -> nil end end defp member_or_literal_disjunct(table, key, index_collation, {:binary, :eq, left, right}) do case equality_constraint(table, left, right) do {^key, expr} -> if collation_names_equal?( equality_term_collation(table, key, left, right), index_collation ) do [expr] end _other -> nil end end defp member_or_literal_disjunct(table, key, index_collation, {:in, expr, list, false}) when is_list(list) do case in_constraint(table, expr, list) do {^key, exprs} -> if not explicit_collation_node?(list) and collation_names_equal?(in_term_collation(table, key, expr), index_collation) do exprs end _other -> nil end end defp member_or_literal_disjunct(_table, _key, _index_collation, _term), do: nil defp expression_equality_constraint(index, indexed_expr, conjuncts) do Enum.find_value(conjuncts, fn {:binary, :eq, left, right} -> cond do expression_equivalent?(left, indexed_expr) and constant_expr?(right) and not explicit_collation_node?(right) -> {index, indexed_expr, right} expression_equivalent?(right, indexed_expr) and constant_expr?(left) and not explicit_collation_node?(left) -> {index, indexed_expr, left} true -> nil end _other -> nil end) end defp best_expression_in_index(table, conjuncts) do lookup_indexes(table) |> Enum.find_value(fn index -> case index_members(index) do [{:expr, indexed_expr}] -> expression_in_constraint(index, indexed_expr, conjuncts) _other -> nil end end) end defp expression_in_constraint(index, indexed_expr, conjuncts) do Enum.find_value(conjuncts, fn {:in, expr, list, false} when is_list(list) -> if expression_equivalent?(expr, indexed_expr) and not explicit_collation_node?(list) and Enum.all?(list, &constant_expr?/1) do {index, indexed_expr, list} end _other -> nil end) end defp best_expression_or_literal_index(table, conjuncts) do lookup_indexes(table) |> Enum.find_value(fn index -> case index_members(index) do [{:expr, indexed_expr}] -> expression_or_literal_constraint(index, indexed_expr, conjuncts) _other -> nil end end) end defp expression_or_literal_constraint(index, indexed_expr, conjuncts) do Enum.find_value(conjuncts, fn term -> disjuncts = where_disjuncts(term) with true <- multiple_terms?(disjuncts), {:ok, {query_expr, values}} <- or_literal_values(disjuncts), true <- expression_equivalent?(query_expr, indexed_expr) do {index, indexed_expr, values} else _other -> nil end end) end defp best_expression_range_index(table, conjuncts) do lookup_indexes(table) |> Enum.find_value(fn index -> case index_members(index) do [{:expr, indexed_expr}] -> case expression_range_constraints(indexed_expr, conjuncts) do [] -> nil bounds -> {index, indexed_expr, bounds} end _other -> nil end end) end defp expression_range_constraints(indexed_expr, conjuncts) do conjuncts |> Enum.flat_map(fn {:binary, op, left, {:literal, _value} = right} when op in [:lt, :le, :gt, :ge] -> if expression_equivalent?(left, indexed_expr) and not explicit_collation_node?(right) do [{op, right}] else [] end {:binary, op, {:literal, _value} = left, right} when op in [:lt, :le, :gt, :ge] -> if expression_equivalent?(right, indexed_expr) and not explicit_collation_node?(left) do [{flip_range_op(op), left}] else [] end {:between, expr, {:literal, _low} = low, {:literal, _high} = high, false} -> if expression_equivalent?(expr, indexed_expr) do [{:ge, low}, {:le, high}] else [] end _other -> [] end) end defp expression_equivalent?({:binary, :eq, left_a, right_a}, {:binary, :eq, left_b, right_b}) do equivalent_binary_terms?(left_a, right_a, left_b, right_b) end defp expression_equivalent?({:binary, :ne, left_a, right_a}, {:binary, :ne, left_b, right_b}) do equivalent_binary_terms?(left_a, right_a, left_b, right_b) end defp expression_equivalent?({:is, left_a, right_a}, {:is, left_b, right_b}) do equivalent_binary_terms?(left_a, right_a, left_b, right_b) end defp expression_equivalent?({:is_not, left_a, right_a}, {:is_not, left_b, right_b}) do equivalent_binary_terms?(left_a, right_a, left_b, right_b) end defp expression_equivalent?(left, right), do: normalize_index_expr(left) == normalize_index_expr(right) defp equivalent_binary_terms?(left_a, right_a, left_b, right_b) do normalized_left_a = normalize_index_expr(left_a) normalized_right_a = normalize_index_expr(right_a) normalized_left_b = normalize_index_expr(left_b) normalized_right_b = normalize_index_expr(right_b) (normalized_left_a == normalized_left_b and normalized_right_a == normalized_right_b) or (normalized_left_a == normalized_right_b and normalized_right_a == normalized_left_b) end defp normalize_index_expr({:column, qualifier, name}) do {:column, normalize_identifier(qualifier), Table.key(name)} end defp normalize_index_expr({:function, name, args}) when is_list(args) do {:function, Table.key(name), Enum.map(args, &normalize_index_expr/1)} end defp normalize_index_expr({:function, name, {:distinct, args}}) do {:function, Table.key(name), {:distinct, Enum.map(args, &normalize_index_expr/1)}} end defp normalize_index_expr(tuple) when is_tuple(tuple) do tuple |> Tuple.to_list() |> Enum.map(&normalize_index_expr/1) |> List.to_tuple() end defp normalize_index_expr(list) when is_list(list), do: Enum.map(list, &normalize_index_expr/1) defp normalize_index_expr(other), do: other defp normalize_identifier(nil), do: nil defp normalize_identifier(name), do: Table.key(name) defp index_lookup_values(table, index, n_columns, where) do constraints = equality_constraints(table, where) values = index.columns |> Enum.take(n_columns) |> Enum.map(fn key -> case Map.fetch(constraints, key) do {:ok, {:literal, value}} -> column = Table.column(table, key) Value.apply_affinity(value, column.affinity) :error -> :missing end end) if :missing in values, do: :error, else: {:ok, values} end defp index_member_lookup_values(table, prefix) do prefix |> Enum.reduce_while({:ok, []}, fn {member, expr}, {:ok, acc} -> case index_member_lookup_value(table, member, expr) do {:ok, value} -> {:cont, {:ok, [value | acc]}} :error -> {:halt, :error} end end) |> case do {:ok, values} -> {:ok, Enum.reverse(values)} :error -> :error end end defp index_member_lookup_value(table, {:column, key}, {:literal, value}) do column = Table.column(table, key) {:ok, Value.apply_affinity(value, column.affinity)} end defp index_member_lookup_value(_table, {:expr, _expr}, {:literal, value}), do: {:ok, value} defp index_member_lookup_value(_table, _member, _expr), do: :error defp index_member_range_lookup_bounds(table, member, bounds) do bounds |> Enum.reduce_while({:ok, []}, fn {op, expr}, {:ok, acc} -> case index_member_lookup_value(table, member, expr) do {:ok, value} -> {:cont, {:ok, [{op, value} | acc]}} :error -> {:halt, :error} end end) |> case do {:ok, bounds} -> {:ok, Enum.reverse(bounds)} :error -> :error end end defp index_member_in_lookup_values(table, member, exprs) do exprs |> Enum.reduce_while({:ok, []}, fn expr, {:ok, acc} -> case index_member_lookup_value(table, member, expr) do {:ok, value} -> {:cont, {:ok, [value | acc]}} :error -> {:halt, :error} end end) |> case do {:ok, values} -> {:ok, Enum.reverse(values)} :error -> :error end end defp expression_index_in_lookup_values(exprs) do exprs |> Enum.reduce_while({:ok, []}, fn {:literal, value}, {:ok, acc} -> {:cont, {:ok, [value | acc]}} _expr, _acc -> {:halt, :error} end) |> case do {:ok, values} -> {:ok, Enum.reverse(values)} :error -> :error end end defp expression_index_range_lookup_bounds(bounds) do bounds |> Enum.reduce_while({:ok, []}, fn {op, {:literal, value}}, {:ok, acc} -> {:cont, {:ok, [{op, value} | acc]}} _bound, _acc -> {:halt, :error} end) |> case do {:ok, bounds} -> {:ok, Enum.reverse(bounds)} :error -> :error end end defp equality_constraints(table, where) do where |> where_conjuncts() |> Enum.reduce(%{}, fn {:binary, :eq, left, right}, acc -> case equality_constraint(table, left, right) do nil -> acc {key, expr} -> Map.put(acc, key, expr) end _other, acc -> acc end) end defp index_range_lookup_bounds(table, index, bounds) do bounds |> Enum.reduce_while({:ok, []}, fn {op, expr}, {:ok, acc} -> case index_range_lookup_value(table, index, expr) do {:ok, value} -> {:cont, {:ok, [{op, value} | acc]}} :error -> {:halt, :error} end end) |> case do {:ok, bounds} -> {:ok, Enum.reverse(bounds)} :error -> :error end end defp index_range_lookup_value(table, index, {:literal, value}) do key = List.first(index.columns) column = Table.column(table, key) {:ok, Value.apply_affinity(value, column.affinity)} end defp index_range_lookup_value(_table, _index, _expr), do: :error defp index_in_lookup_values(table, index, exprs) do exprs |> Enum.reduce_while({:ok, []}, fn expr, {:ok, acc} -> case index_range_lookup_value(table, index, expr) do {:ok, value} -> {:cont, {:ok, [value | acc]}} :error -> {:halt, :error} end end) |> case do {:ok, values} -> {:ok, Enum.reverse(values)} :error -> :error end end defp index_lookup_rowids(db, index, {:eq, values}) do prefix_count = length(values) collations = Map.get(index, :collations, []) |> Enum.take(prefix_count) if direct_index_lookup?(index, values, collations) do index |> Map.get(:entries, %{}) |> Map.get(List.to_tuple(values), []) |> Enum.sort() else index |> Map.get(:entries, %{}) |> Enum.flat_map(fn {stored_values, rowids} -> stored_prefix = stored_values |> Tuple.to_list() |> Enum.take(prefix_count) if values_equal_with_collations?(db, stored_prefix, values, collations) do rowids else [] end end) |> Enum.uniq() |> Enum.sort() end end defp index_lookup_rowids(db, index, {:in, values}) do values |> Enum.flat_map(&index_lookup_rowids(db, index, {:eq, [&1]})) |> Enum.uniq() |> Enum.sort() end defp index_lookup_rowids(db, index, {:range, bounds}) do case ordered_binary_range_rowids(index, bounds, 0) do {:ok, rowids} -> rowids :error -> collation = List.first(Map.get(index, :collations) || []) || :binary index |> Map.get(:entries, %{}) |> Enum.flat_map(fn {stored_values, rowids} -> stored_value = stored_values |> Tuple.to_list() |> List.first() if index_range_match?(db, stored_value, bounds, collation) do rowids else [] end end) |> Enum.uniq() |> Enum.sort() end end defp index_lookup_rowids(db, index, {:member_range, prefix_values, bounds}) do prefix_count = length(prefix_values) collations = Map.get(index, :collations, []) prefix_collations = Enum.take(collations, prefix_count) range_collation = Enum.at(collations, prefix_count) || :binary index |> Map.get(:entries, %{}) |> Enum.flat_map(fn {stored_values, rowids} -> stored_values = Tuple.to_list(stored_values) stored_prefix = Enum.take(stored_values, prefix_count) stored_range_value = Enum.at(stored_values, prefix_count) if values_equal_with_collations?(db, stored_prefix, prefix_values, prefix_collations) and index_range_match?(db, stored_range_value, bounds, range_collation) do rowids else [] end end) |> Enum.uniq() |> Enum.sort() end defp index_lookup_rowids(db, index, {:member_in, prefix_values, values}) do values |> Enum.flat_map(fn value -> index_lookup_rowids(db, index, {:eq, prefix_values ++ [value]}) end) |> Enum.uniq() |> Enum.sort() end defp ordered_binary_range_rowids(index, bounds, member_index) do with true <- binary_collation_index?(index), ordered when is_tuple(ordered) <- Map.get(index, :ordered_entries), {:ok, lower, upper} <- index_range_limits(bounds) do start = ordered_range_start(ordered, lower, member_index) ordered |> collect_ordered_range(start, tuple_size(ordered), upper, member_index, []) |> Enum.uniq() |> Enum.sort() |> then(&{:ok, &1}) else _other -> :error end end defp index_range_limits(bounds) do Enum.reduce_while(bounds, {:ok, nil, nil}, fn {_op, nil}, _acc -> {:halt, :error} {op, value}, {:ok, lower, upper} when op in [:gt, :ge] -> {:cont, {:ok, strongest_lower_bound(lower, {value, op}), upper}} {op, value}, {:ok, lower, upper} when op in [:lt, :le] -> {:cont, {:ok, lower, strongest_upper_bound(upper, {value, op})}} _other, _acc -> {:halt, :error} end) end defp strongest_lower_bound(nil, bound), do: bound defp strongest_lower_bound({value, op} = current, {new_value, new_op} = new) do case Value.compare(new_value, value) do :gt -> new :eq when new_op == :gt and op == :ge -> new _other -> current end end defp strongest_upper_bound(nil, bound), do: bound defp strongest_upper_bound({value, op} = current, {new_value, new_op} = new) do case Value.compare(new_value, value) do :lt -> new :eq when new_op == :lt and op == :le -> new _other -> current end end defp ordered_range_start(_ordered, nil, _member_index), do: 0 defp ordered_range_start(ordered, {value, op}, member_index) do first_ordered_index(ordered, value, op == :ge, member_index, 0, tuple_size(ordered)) end defp first_ordered_index(_ordered, _value, _include_equal?, _member_index, low, low), do: low defp first_ordered_index(ordered, value, include_equal?, member_index, low, high) do mid = div(low + high, 2) {stored_values, _rowids} = elem(ordered, mid) stored_value = elem(stored_values, member_index) before_start? = case Value.compare(stored_value, value) do :lt -> true :eq -> not include_equal? :gt -> false end if before_start? do first_ordered_index(ordered, value, include_equal?, member_index, mid + 1, high) else first_ordered_index(ordered, value, include_equal?, member_index, low, mid) end end defp collect_ordered_range(_ordered, index, size, _upper, _member_index, acc) when index >= size do acc end defp collect_ordered_range(ordered, index, size, upper, member_index, acc) do {stored_values, rowids} = elem(ordered, index) stored_value = elem(stored_values, member_index) cond do is_nil(stored_value) -> collect_ordered_range(ordered, index + 1, size, upper, member_index, acc) upper_bound_exceeded?(stored_value, upper) -> acc true -> collect_ordered_range(ordered, index + 1, size, upper, member_index, rowids ++ acc) end end defp upper_bound_exceeded?(_stored_value, nil), do: false defp upper_bound_exceeded?(stored_value, {value, op}) do case Value.compare(stored_value, value) do :gt -> true :eq -> op == :lt :lt -> false end end defp direct_index_lookup?(index, values, collations) do length(values) == length(index_members(index)) and Enum.all?(collations, &binary_collation_name?/1) end defp binary_collation_name?(nil), do: true defp binary_collation_name?(:binary), do: true defp binary_collation_name?(name) when is_binary(name), do: String.downcase(name) == "binary" defp binary_collation_name?(_name), do: false defp index_range_match?(db, stored_value, bounds, collation) do collation = normalize_collation!(collation, %{db: db}) Enum.all?(bounds, fn {op, value} -> Value.compare_op(op, stored_value, value, collation) == true end) end # SQLite prefers a unique index, then the longest usable equality prefix. defp best_equality_index(table, eq_keys, where) do lookup_indexes(table) |> Enum.filter(fn index -> prefix = Enum.take_while(index.columns, &MapSet.member?(eq_keys, &1)) prefix != [] and index_lookup_usable?(table, index, length(prefix), where) end) |> Enum.sort_by(fn index -> prefix = Enum.take_while(index.columns, &MapSet.member?(eq_keys, &1)) {if(index.unique, do: 0, else: 1), -length(prefix)} end) |> List.first() end defp relation(_db, nil, _outer), do: {[], [[]]} defp relation(db, {:table, {:schema, schema, name}, alias_name}, outer) do ensure_table_schema!(db, schema, name) key = Table.key(name) if key in ["sqlite_schema", "sqlite_master"] do relation_from_result( ["type", "name", "tbl_name", "rootpage", "sql"], sqlite_schema_rows(db, schema), [:text, :text, :text, :integer, :text], alias_name || name ) else relation_named_table(db, Database.table_storage_key(schema, name), name, alias_name, outer) end end defp relation(db, {:table, name, alias_name}, outer) do key = Table.key(name) table_key = relation_unqualified_table_key(db, name) # Check materialized CTEs first, then pending (lazy), then tables, then views. relation_named_table(db, key, table_key, name, alias_name, outer) end defp relation(db, {:subquery, select, alias_name}, outer) do result = query_result(db, select, outer) {keys, affinities} = subquery_column_meta(result.columns, result.affinities) template_name = if alias_name, do: Table.key(alias_name), else: "exsql_subquery_#{:erlang.phash2({keys, result.columns}) |> Integer.to_string()}" columns = Enum.zip([keys, result.columns, affinities]) tmpl = %{ name: template_name, source_name: alias_name, columns: columns, columns_by_key: index_columns(columns), hidden: MapSet.new(), row: %{}, rowid: nil, has_rowid: false } rows = for row <- result.rows, do: [%{tmpl | row: Map.new(Enum.zip(keys, row))}] {[tmpl], rows} end defp relation(db, {:grouped, source, alias_name}, outer) do case relation(db, source, outer) do {[tmpl], rows} when alias_name != nil -> {[%{tmpl | name: Table.key(alias_name), source_name: alias_name}], rows} relation -> relation end end # A table-valued function on the right of a join is lateral: its # arguments may reference columns of the rows to its left, so it is # re-evaluated per left row (`FROM t, json_each(t.doc)`). defp relation(db, {:join, type, left, {:table_function, _, _, _} = tf, constraint}, outer) do {ltmpls, lrows} = relation(db, left, outer) {[rtmpl], _norows} = table_function_relation(db, tf, %{db: db, frames: [], group: nil, outer: outer}, true) rows = Enum.flat_map(lrows, fn lframes -> env = %{db: db, frames: lframes, group: nil, outer: outer} {[_tmpl], rrows} = table_function_relation(db, tf, env, false) matched = for [rframe] <- rrows, join_match?(db, constraint, [], lframes, rframe, outer), do: lframes ++ [rframe] case matched do [] when type.left -> [lframes ++ [null_frame(rtmpl)]] matched -> matched end end) {ltmpls ++ [rtmpl], rows} end defp relation(db, {:table_function, _, _, _} = tf, outer) do table_function_relation(db, tf, %{db: db, frames: [], group: nil, outer: outer}, false) end # Predicate pushdown: a base table wrapped with the WHERE conjuncts that # reference only that table, so the nested-loop join sees a pre-filtered # relation instead of the full table. The outer WHERE is re-applied later, # so this only removes rows that could never have survived. defp relation(db, {:prefiltered, src, preds}, outer) do {tmpls, rows} = relation(db, src, outer) filtered = Enum.filter(rows, fn frames -> env = %{db: db, frames: frames, group: nil, outer: outer} Enum.all?(preds, &matches_where?(&1, env)) end) {tmpls, filtered} end # Inner hash join: build a multimap of right rows keyed by the equi-join key, # then probe once per left row. Replaces the O(n*m) nested loop for equality # joins. Any extra (non-key) constraint is still checked per candidate, and # the outer WHERE is re-applied downstream. defp relation(db, {:hashjoin, type, left, right, constraint, equi}, outer) do {ltmpls, lrows} = relation(db, left, outer) {rtmpls, rrows} = relation(db, right, outer) constraint = if constraint == nil, do: {:on, {:literal, true}}, else: constraint using = using_columns(type, constraint, ltmpls, rtmpls) rtmpls = Enum.map(rtmpls, &%{&1 | hidden: MapSet.union(&1.hidden, MapSet.new(using))}) rframes = Enum.map(rrows, fn frames -> frames |> Enum.with_index() |> Enum.map(fn {frame, index} -> %{frame | hidden: Enum.fetch!(rtmpls, index).hidden} end) end) {lexprs, rexprs} = Enum.unzip(equi) build = Enum.reduce(rframes, %{}, fn rframe, acc -> case hash_join_key(rexprs, %{db: db, frames: rframe, group: nil, outer: outer}) do :null -> acc key -> Map.update(acc, key, [rframe], &[rframe | &1]) end end) # A NULL probe key means the equi-condition (part of the ON) cannot be # true: an inner join drops the row, a LEFT join null-extends it — the same # outcome an unmatched (or fully filtered) probe gets below. null_right_rows = if type.left, do: Enum.map(rtmpls, &null_frame/1) rows = Enum.flat_map(lrows, fn lframes -> matches = case hash_join_key(lexprs, %{db: db, frames: lframes, group: nil, outer: outer}) do :null -> [] key -> build |> Map.get(key, []) |> Enum.reverse() |> Enum.filter(&join_match?(db, constraint, using, lframes, &1, outer)) |> Enum.map(&(lframes ++ &1)) end case matches do [] when type.left -> [lframes ++ null_right_rows] matches -> matches end end) {ltmpls ++ rtmpls, rows} end # Pure cartesian (comma join with no ON/USING and no outer side): the cross # product directly, skipping the per-pair predicate check, with_index, and # matched-right bookkeeping the general path carries. The WHERE that selects # rows is applied downstream. defp relation( db, {:join, %{left: false, right: false, natural: false}, left, right, nil}, outer ) do {ltmpls, lrows} = relation(db, left, outer) {rtmpls, rrows} = relation(db, right, outer) rows = for lframes <- lrows, rframe <- rrows, do: lframes ++ rframe {ltmpls ++ rtmpls, rows} end defp relation(db, {:join, type, left, right, constraint}, outer) do {ltmpls, lrows} = relation(db, left, outer) {rtmpls, rrows} = relation(db, right, outer) constraint = if constraint == nil, do: {:on, {:literal, true}}, else: constraint using = using_columns(type, constraint, ltmpls, rtmpls) rtmpls = Enum.map(rtmpls, &%{&1 | hidden: MapSet.union(&1.hidden, MapSet.new(using))}) rframes = Enum.map(rrows, fn frames -> Enum.with_index(frames) |> Enum.map(fn {frame, index} -> %{frame | hidden: Enum.fetch!(rtmpls, index).hidden} end) end) null_right_rows = Enum.map(rtmpls, &null_frame/1) {rows, matched_rights} = Enum.map_reduce(lrows, MapSet.new(), fn lframes, matched_rights -> matched = for {rframe, right_index} <- Enum.with_index(rframes), join_match?(db, constraint, using, lframes, rframe, outer), do: {lframes ++ rframe, right_index} rows = case matched do [] when type.left -> [lframes ++ null_right_rows] [] -> [] matches -> Enum.map(matches, &elem(&1, 0)) end matched_rights = Enum.reduce(matched, matched_rights, fn {_row, right_index}, matched_rights -> MapSet.put(matched_rights, right_index) end) {rows, matched_rights} end) |> then(fn {rows, matched_rights} -> {Enum.flat_map(rows, & &1), matched_rights} end) right_rows = if type.right do for {rframe, right_index} <- Enum.with_index(rframes), not MapSet.member?(matched_rights, right_index) do right_unmatched_row(ltmpls, rframe, using) end else [] end {ltmpls ++ rtmpls, rows ++ right_rows} end defp subquery_column_meta(columns, affinities) do duplicate = %{} {keys, _} = columns |> Enum.with_index() |> Enum.map_reduce(duplicate, fn {name, index}, seen -> base_key = Table.key(name) n = Map.get(seen, base_key, 0) + 1 suffix = if n == 1, do: "", else: "__exsql#{n - 1}" key = base_key <> suffix affinity = Enum.at(affinities, index) || :blob {{key, affinity}, Map.put(seen, base_key, n)} end) {Enum.map(keys, &elem(&1, 0)), Enum.map(keys, &elem(&1, 1))} end defp right_unmatched_row(ltmpls, rframes, using) when is_list(rframes) do left_frames = Enum.map(ltmpls, fn tmpl -> frame = null_frame(tmpl) row = Enum.reduce(using, frame.row, fn key, row -> if visible?(frame, key) do Map.put(row, key, resolve_right_row_value(rframes, key)) else row end end) %{frame | row: row} end) left_frames ++ rframes end defp right_unmatched_row(ltmpls, rframe, using) when is_map(rframe), do: right_unmatched_row(ltmpls, [rframe], using) defp relation_named_table(db, key, name, alias_name, outer), do: relation_named_table(db, key, key, name, alias_name, outer) defp relation_named_table(db, key, table_key, name, alias_name, outer) do cond do Map.has_key?(db.ctes, key) -> cte = Map.fetch!(db.ctes, key) if Map.get(cte, :actual_count) != nil and cte.actual_count != length(cte.columns) do fail("table #{name} has #{cte.actual_count} values for #{length(cte.columns)} columns") end relation_from_result(cte.columns, cte.rows, cte.affinities, alias_name || name) key in ["sqlite_schema", "sqlite_master"] -> relation_from_result( ["type", "name", "tbl_name", "rootpage", "sql"], sqlite_schema_rows(db), [:text, :text, :text, :integer, :text], alias_name || name ) key == "sqlite_sequence" and sqlite_sequence_exists?(db) -> relation_from_result( ["name", "seq"], sqlite_sequence_rows(db), [:text, :integer], alias_name || name ) Map.has_key?(db.tables, table_key) -> table = Map.fetch!(db.tables, table_key) tmpl = table_frame(table, alias_name) rows = for {rowid, row} <- Table.scan_positional(table), do: [%{tmpl | row: row, rowid: rowid}] {[tmpl], rows} Map.has_key?(db.views, table_key) -> view = Map.fetch!(db.views, table_key) result = query_result(db, view.query, outer) {columns, affinities} = case view.columns do nil -> {result.columns, result.affinities} col_names -> if length(col_names) != length(result.columns) do fail( "expected #{length(col_names)} columns for '#{name}' but got #{length(result.columns)}" ) end affs = result.affinities ++ List.duplicate(:blob, length(col_names)) {col_names, Enum.take(affs, length(col_names))} end relation_from_result(columns, result.rows, affinities, alias_name || name) true -> fail("no such table: #{name}") end end defp relation_unqualified_table_key(db, name) do Enum.find_value(table_lookup_order(db), Table.key(name), fn schema -> key = Database.table_storage_key(schema, name) if Map.has_key?(db.tables, key) or Map.has_key?(db.views, key) do key else nil end end) end defp table_frame(table, alias_name) do columns = Table.frame_columns(table) %{ name: Table.key(alias_name || table.name), source_name: table.name, columns: columns, columns_by_key: index_columns(columns), col_index: Table.column_index(table), hidden: MapSet.new(), row: %{}, rowid: nil, has_rowid: not table.without_rowid } end # A key => column-tuple map for O(1) `has_column?`/`frame_column`, instead of a # linear `List.keymember?` scan of the columns list on every column access. defp index_columns(columns), do: Map.new(columns, fn col -> {column_key(col), col} end) defp null_frame(tmpl) do %{tmpl | row: Map.new(tmpl.columns, fn column -> {column_key(column), nil} end), rowid: nil} end # Build frames from a pre-computed result set (used for views and CTEs). # -- table-valued functions ------------------------------------------------------ @json_each_columns ~w(key value type atom id parent fullkey path) defp table_function_relation( _db, {:table_function, name, args, alias_name}, env, template_only? ) do columns = table_function_columns!(name) rows = if template_only?, do: [], else: table_function_rows(name, args, env) relation_from_result( columns, rows, List.duplicate(:blob, length(columns)), alias_name || name ) end defp table_function_columns!(name) when name in ["json_each", "json_tree"], do: @json_each_columns defp table_function_columns!("pragma_table_info"), do: ~w(cid name type notnull dflt_value pk) defp table_function_columns!("pragma_table_xinfo"), do: ~w(cid name type notnull dflt_value pk hidden) defp table_function_columns!("pragma_index_list"), do: ~w(seq name unique origin partial) defp table_function_columns!("pragma_index_info"), do: ~w(seqno cid name) defp table_function_columns!("pragma_foreign_key_list"), do: ~w(id seq table from to on_update on_delete match) defp table_function_columns!(name), do: fail("no such table: #{name}") # `pragma_(table)` table-valued functions reuse the corresponding # PRAGMA's row builder, so they stay in sync with the statement form. defp table_function_rows("pragma_table_info", args, env), do: pragma_table_fn_rows(args, env, &table_info_rows(&1, false)) defp table_function_rows("pragma_table_xinfo", args, env), do: pragma_table_fn_rows(args, env, &table_info_rows(&1, true)) defp table_function_rows("pragma_index_list", args, env), do: pragma_table_fn_rows(args, env, &index_list_rows/1) defp table_function_rows("pragma_foreign_key_list", args, env), do: pragma_table_fn_rows(args, env, &foreign_key_list_rows/1) defp table_function_rows("pragma_index_info", [arg | _], env) do case pragma_find_index_owner(env.db, eval(arg, env)) do {table, index} -> index_info_rows(table, index) nil -> [] end end defp table_function_rows(name, args, env) do {doc, path} = case args do [doc_expr] -> {eval(doc_expr, env), "$"} [doc_expr, path_expr] -> {eval(doc_expr, env), eval(path_expr, env) || "$"} _ -> fail("#{name}() requires 1 or 2 arguments") end if doc == nil do [] else jv = json_parse!(doc) steps = json_path!(path) case Json.get(jv, steps) do :missing -> [] {:ok, root} -> case name do "json_each" -> json_each_rows(root, path) "json_tree" -> json_tree_rows(root, nil, path, path, nil, 1) |> elem(0) end end end end defp pragma_table_fn_rows([arg | _], env, builder) do case pragma_fetch_table(env.db, eval(arg, env)) do {:ok, table} -> builder.(table) :error -> [] end end defp pragma_table_fn_rows([], _env, _builder), do: [] defp json_each_rows({:object, pairs}, path) do pairs |> Enum.with_index(1) |> Enum.map(fn {{key, jv}, id} -> key = Json.object_key_text(key) json_member_row(key, jv, id, nil, json_key_accessor(path, key), path) end) end defp json_each_rows({:array, items}, path) do items |> Enum.with_index() |> Enum.map(fn {jv, index} -> json_member_row(index, jv, index + 1, nil, "#{path}[#{index}]", path) end) end defp json_each_rows(scalar, path) do [json_member_row(nil, scalar, 1, nil, path, path)] end # json_tree emits the value itself, then its descendants. The id column is # an arbitrary unique integer, as documented for the SQLite originals. defp json_tree_rows(jv, key, fullkey, path, parent_id, next_id) do row = json_member_row(key, jv, next_id, parent_id, fullkey, path) id = next_id {child_rows, next_id} = case jv do {:object, pairs} -> Enum.reduce(pairs, {[], next_id + 1}, fn {k, v}, {acc, n} -> k = Json.object_key_text(k) {rows, n} = json_tree_rows(v, k, json_key_accessor(fullkey, k), fullkey, id, n) {acc ++ rows, n} end) {:array, items} -> items |> Enum.with_index() |> Enum.reduce({[], next_id + 1}, fn {v, index}, {acc, n} -> {rows, n} = json_tree_rows(v, index, "#{fullkey}[#{index}]", fullkey, id, n) {acc ++ rows, n} end) _scalar -> {[], next_id + 1} end {[row | child_rows], next_id} end defp json_member_row(key, jv, id, parent_id, fullkey, path) do atom = case jv do {:array, _} -> nil {:object, _} -> nil scalar -> Json.to_sql(scalar) end [key, Json.to_sql(jv), Json.type_name(jv), atom, id, parent_id, fullkey, path] end defp json_key_accessor(path, key) do if String.match?(key, ~r/^[A-Za-z_][A-Za-z0-9_]*$/) do "#{path}.#{key}" else ~s(#{path}."#{key}") end end defp relation_from_result(columns, rows, affinities, alias_name) do keys = Enum.map(columns, &Table.key/1) affinities = affinities ++ List.duplicate(:blob, length(keys)) frame_columns = Enum.zip([keys, columns, Enum.take(affinities, length(keys))]) tmpl = %{ name: Table.key(alias_name), source_name: alias_name, columns: frame_columns, columns_by_key: index_columns(frame_columns), hidden: MapSet.new(), row: %{}, rowid: nil, has_rowid: false } rows = for row <- rows, do: [%{tmpl | row: Map.new(Enum.zip(keys, row))}] {[tmpl], rows} end defp sqlite_schema_rows(db), do: sqlite_schema_rows(db, nil) defp sqlite_schema_rows(db, schema) do table_rows = db.tables |> Map.values() |> Enum.filter(&sqlite_schema_matches?(&1.schema, schema)) |> Enum.sort_by(&Table.key(&1.name)) |> Enum.with_index(1) |> Enum.flat_map(fn {table, rootpage} -> table_row = ["table", table.name, table.name, rootpage, create_table_sql(table)] autoindex_rows = table.autoindexes |> Enum.with_index(rootpage + 500) |> Enum.map(fn {index, index_rootpage} -> ["index", index.name, table.name, index_rootpage, nil] end) index_rows = table.indexes |> Enum.with_index(rootpage + 1000) |> Enum.map(fn {index, index_rootpage} -> ["index", index.name, table.name, index_rootpage, create_index_sql(table, index)] end) [table_row | autoindex_rows ++ index_rows] end) sequence_row = if main_schema?(schema) and sqlite_sequence_exists?(db) do [ [ "table", "sqlite_sequence", "sqlite_sequence", 0, "CREATE TABLE sqlite_sequence(name,seq)" ] ] else [] end view_rows = db.views |> Map.values() |> Enum.filter(&sqlite_schema_matches?(&1.schema, schema)) |> Enum.sort_by(&Table.key(&1.name)) |> Enum.map(fn view -> ["view", view.name, view.name, 0, create_view_sql(view)] end) trigger_rows = db.triggers |> Map.values() |> Enum.filter(&sqlite_schema_matches?(&1.schema, schema)) |> Enum.sort_by(& &1.seq) |> Enum.map(fn trigger -> ["trigger", trigger.name, trigger.table_name, 0, create_trigger_sql(trigger)] end) table_rows ++ sequence_row ++ view_rows ++ trigger_rows end defp sqlite_schema_matches?(object_schema, schema) do Table.key(object_schema || "main") == Table.key(schema || "main") end # The stored SQL for a trigger is reconstructed from the parsed definition; # body statements are not round-tripped to SQL text yet. defp create_trigger_sql(trigger) do timing = case trigger.timing do :before -> "BEFORE" :after -> "AFTER" :instead_of -> "INSTEAD OF" end event = case {trigger.event, trigger.update_columns} do {:update, columns} when is_list(columns) -> "UPDATE OF #{Enum.join(columns, ", ")}" {event, _} -> event |> Atom.to_string() |> String.upcase() end "CREATE TRIGGER #{trigger.name} #{timing} #{event} ON #{trigger.table_name} " <> "FOR EACH ROW BEGIN ... END" end defp sqlite_sequence_exists?(db), do: Enum.any?(db.tables, fn {_key, table} -> table.autoincrement end) defp ensure_sqlite_sequence_exists!(db) do unless sqlite_sequence_exists?(db), do: fail("no such table: sqlite_sequence") end defp sqlite_sequence_table(db) do rows = db |> sqlite_sequence_rows() |> Enum.with_index(1) |> Map.new(fn {[name, seq], rowid} -> {rowid, {name, seq}} end) %Table{ name: "sqlite_sequence", columns: [ %ColumnDef{name: "name", affinity: :text}, %ColumnDef{name: "seq", affinity: :integer} ], rows: rows, next_rowid: map_size(rows) + 1 } end defp put_sqlite_sequence(db, name, sequence, visible?) do key = Table.key(name) case Map.fetch(db.tables, key) do {:ok, %{autoincrement: true} = table} -> put_table(db, %{table | sequence: sequence, sequence_row: visible?}) _other -> orphans = if visible? do Map.put(db.sqlite_sequence_orphans, key, {name, sequence}) else Map.delete(db.sqlite_sequence_orphans, key) end %{db | sqlite_sequence_orphans: orphans} end end defp sqlite_sequence_rows(db) do table_rows = db.tables |> Map.values() |> Enum.filter(&(&1.autoincrement and &1.sequence_row)) |> Enum.map(&{Table.key(&1.name), [&1.name, &1.sequence]}) orphan_rows = db.sqlite_sequence_orphans |> Enum.map(fn {key, {name, sequence}} -> {key, [name, sequence]} end) (table_rows ++ orphan_rows) |> Enum.sort_by(fn {key, _row} -> key end) |> Enum.map(fn {_key, row} -> row end) end defp create_table_sql(table) do definition_sql = table.columns |> Enum.map(&column_def_sql/1) |> Kernel.++(Enum.map(table.foreign_keys, &table_foreign_key_sql(table, &1))) |> Enum.join(", ") options = [ if(table.without_rowid, do: "WITHOUT ROWID"), if(table.strict, do: "STRICT") ] |> Enum.reject(&is_nil/1) suffix = if options == [], do: "", else: " " <> Enum.join(options, ", ") "CREATE TABLE #{table.name}(#{definition_sql})#{suffix}" end defp table_foreign_key_sql(table, {child_keys, parent_table, parent_keys, actions}) do child_columns = child_keys |> Enum.map_join(", ", &display_column_name(table, &1)) references = case parent_keys do [] -> parent_table keys -> "#{parent_table}(#{Enum.join(keys, ", ")})" end "FOREIGN KEY(#{child_columns}) REFERENCES #{references}#{fk_actions_sql(actions)}" end defp fk_actions_sql(actions) do [ if(actions.on_delete != :no_action, do: " ON DELETE #{fk_action_name(actions.on_delete)}" ), if(actions.on_update != :no_action, do: " ON UPDATE #{fk_action_name(actions.on_update)}" ), if(actions.deferred, do: " DEFERRABLE INITIALLY DEFERRED") ] |> Enum.reject(&is_nil/1) |> Enum.join() end defp fk_action_name(:no_action), do: "NO ACTION" defp fk_action_name(:restrict), do: "RESTRICT" defp fk_action_name(:set_null), do: "SET NULL" defp fk_action_name(:set_default), do: "SET DEFAULT" defp fk_action_name(:cascade), do: "CASCADE" defp column_def_sql(column) do [ column.name, column.declared_type, generated_sql(column), if(column.primary_key, do: "PRIMARY KEY"), if(column.autoincrement, do: "AUTOINCREMENT"), if(column.not_null, do: "NOT NULL"), if(column.unique, do: "UNIQUE"), column.default && "DEFAULT #{pragma_default(column.default)}", column.collate && "COLLATE #{column.collate}", references_sql(column) ] |> Enum.reject(&is_nil/1) |> Enum.join(" ") end defp generated_sql(%{generated: {kind, expr}}) do "GENERATED ALWAYS AS (#{expr_name(expr)}) #{kind |> Atom.to_string() |> String.upcase()}" end defp generated_sql(_column), do: nil defp references_sql(%{references: {table, [], actions}}), do: "REFERENCES #{table}#{fk_actions_sql(actions)}" defp references_sql(%{references: {table, columns, actions}}), do: "REFERENCES #{table}(#{Enum.join(columns, ", ")})#{fk_actions_sql(actions)}" defp references_sql(_column), do: nil defp create_index_sql(table, index) do unique = if index.unique, do: "UNIQUE ", else: "" columns = index |> index_members() |> Enum.map_join(", ", fn {:column, key} -> display_column_name(table, key) {:expr, expr} -> expr_name(expr) end) where = if index.where, do: " WHERE #{expr_name(index.where)}", else: "" "CREATE #{unique}INDEX #{index.name} ON #{table.name}(#{columns})#{where}" end defp create_view_sql(view), do: "CREATE VIEW #{view.name} AS SELECT" defp using_columns(%{natural: true}, _constraint, ltmpls, rtmpls) when is_list(rtmpls) do rtmpls |> Enum.flat_map(& &1.columns) |> Enum.map(&column_key/1) |> Enum.filter(fn key -> Enum.any?(rtmpls, fn rtmpl -> visible?(rtmpl, key) and not MapSet.member?(rtmpl.hidden, key) end) and Enum.any?(ltmpls, &visible?(&1, key)) end) |> Enum.uniq() end defp using_columns(_type, {:using, names}, ltmpls, rtmpls) when is_list(rtmpls) do Enum.map(names, fn name -> key = Table.key(name) right_visible = Enum.count(rtmpls, &visible?(&1, key)) if right_visible > 1 do fail("ambiguous column name: #{name}") end if right_visible == 1 and Enum.any?(ltmpls, &visible?(&1, key)) do key else fail("cannot join using column #{name} - column not present in both tables") end end) end defp using_columns(type, constraint, ltmpls, rtmpl) when is_map(rtmpl), do: using_columns(type, constraint, ltmpls, [rtmpl]) defp using_columns(_type, _constraint, _ltmpls, _rtmpl), do: [] defp join_match?(db, constraint, using, lframes, rframe, outer) when is_map(rframe), do: join_match?(db, constraint, using, lframes, [rframe], outer) defp join_match?(db, constraint, using, lframes, rframes, outer) when is_list(rframes) do case {constraint, using} do {{:on, expr}, []} -> env = %{db: db, frames: lframes ++ rframes, group: nil, outer: outer} truth(expr, env) == true {_, []} -> true {_, using} -> left_env = %{db: db, frames: lframes, group: nil, outer: nil} Enum.all?(using, fn key -> {a, b} = Value.comparison_coerce( resolve_column(left_env, nil, key), column_affinity(left_env, nil, key), resolve_right_row_value(rframes, key), resolve_right_frame_affinity(rframes, key) ) Value.compare_op(:eq, a, b, column_collation(left_env, nil, key)) == true end) end end defp resolve_right_row_value([], _key), do: nil defp resolve_right_row_value([frame | rest], key) do if has_column?(frame, key) do frame_cell(frame, key) else resolve_right_row_value(rest, key) end end defp resolve_right_frame_affinity([], _key), do: nil defp resolve_right_frame_affinity([frame | rest], key) do if has_column?(frame, key) do frame_affinity(frame, key) else resolve_right_frame_affinity(rest, key) end end # Targets carry the folded column key (`{column, key}` | `:rowid`) so the # per-row `insert_values/2` reuses it instead of re-folding `Table.key/1` for # every column on every inserted row. defp insert_targets(table, %Insert{columns: nil}) do # Reuse the cached, pre-folded `frame_columns` keys (parallel to `columns`) # rather than re-folding `Table.key(&1.name)` for every column on every insert. table |> Table.frame_columns() |> Enum.zip(table.columns) |> Enum.reject(fn {_fc, column} -> column.generated end) |> Enum.map(fn {{key, _name, _aff, _coll}, column} -> {column, key} end) end defp insert_targets(table, %Insert{columns: names}) do Enum.map(names, fn name -> case Table.column(table, name) do %{} = column -> if column.generated do fail("cannot INSERT into generated column \"#{column.name}\"") end {column, Table.key(column.name)} nil -> if Table.key(name) in @rowid_names and not table.without_rowid do :rowid else fail("table #{table.name} has no column named #{name}") end end end) end defp expand_columns(columns, templates) do Enum.flat_map(columns, fn :star -> if templates == [], do: fail("no tables specified") Enum.flat_map(templates, fn tmpl -> for column <- tmpl.columns, key = column_key(column), display = column_display(column), not MapSet.member?(tmpl.hidden, key) do alias_name = if(Table.key(display) == key, do: nil, else: display) {{:column, tmpl.name, key}, alias_name} end end) {:qualified_star, table} -> qkey = Table.key(table) tmpl = Enum.find(templates, &(&1.name == qkey)) || fail("no such table: #{table}") for column <- tmpl.columns, key = column_key(column), display = column_display(column) do alias_name = if(Table.key(display) == key, do: nil, else: display) {{:column, tmpl.name, key}, alias_name} end {expr, alias_name} -> [{expr, alias_name}] end) end # -- GROUP BY / HAVING ---------------------------------------------------------- defp grouped_envs(db, stmt, columns, templates, envs, outer) do group_exprs = stmt.group_by |> Enum.with_index(1) |> Enum.map(fn {expr, index} -> resolve_group_term(expr, index, columns, templates) end) having = stmt.having && substitute_aliases(stmt.having, columns, templates) template_env = %{db: db, frames: templates, group: nil, outer: outer} group_collations = Enum.map(group_exprs, &expr_collation(&1, template_env)) grouped_raw = group_members(group_exprs, group_collations, envs) grouped = grouped_raw |> Enum.map(fn members -> frames = case members do [first | _] -> first.frames [] -> [] end %{db: db, frames: frames, group: members, outer: outer} end) |> Enum.filter(fn genv -> having == nil or truth(having, genv) == true end) grouped end defp group_members([], _collations, envs) do [envs] end defp group_members(group_exprs, group_collations, envs) do envs |> Enum.map(fn env -> {Enum.map(group_exprs, &eval(&1, env)), env} end) |> Enum.sort(fn {a, _}, {b, _} -> compare_key_values(a, b, group_collations) != :gt end) |> Enum.reduce([], fn {key, env}, [{prev_key, members} | done] = acc -> if compare_key_values(key, prev_key, group_collations) == :eq do [{prev_key, [env | members]} | done] else [{key, [env]} | acc] end {key, env}, [] -> [{key, [env]}] end) |> Enum.reverse() |> Enum.map(fn {_key, members} -> Enum.reverse(members) end) end defp compare_key_values([], [], _collations), do: :eq defp compare_key_values([], _right, _collations), do: :lt defp compare_key_values(_left, [], _collations), do: :gt defp compare_key_values([left | left_tail], [right | right_tail], [collation | collations]) do case Value.compare(left, right, collation) do :eq -> compare_key_values(left_tail, right_tail, collations) other -> other end end defp compare_key_values([left | left_tail], [right | right_tail], []) do case Value.compare(left, right, :binary) do :eq -> compare_key_values(left_tail, right_tail, []) other -> other end end defp compare_keys([], []), do: :eq defp compare_keys([a | rest_a], [b | rest_b]) do case Value.compare(a, b) do :eq -> compare_keys(rest_a, rest_b) other -> other end end # A GROUP BY term may be a 1-based output column position, an output alias, # or an expression over the source rows. defp resolve_group_term({:literal, n}, index, columns, _templates) when is_integer(n) do if n < 1 or n > length(columns) do fail( "#{ordinal(index)} GROUP BY term out of range - should be between 1 and #{length(columns)}" ) end columns |> Enum.at(n - 1) |> elem(0) end defp resolve_group_term(expr, _index, columns, templates), do: substitute_aliases(expr, columns, templates) defp ordinal(1), do: "1st" defp ordinal(2), do: "2nd" defp ordinal(3), do: "3rd" defp ordinal(n), do: "#{n}th" # Replaces references to output aliases (`GROUP BY x`, `HAVING y>=4`) with # their expressions, unless the name is a real source column, which wins. defp substitute_aliases({:column, nil, name} = expr, columns, templates) do key = Table.key(name) if Enum.any?(templates, &has_column?(&1, key)) do expr else case Enum.find(columns, fn {_e, alias_name} -> alias_name != nil and Table.key(alias_name) == key end) do {aliased, _} -> aliased nil -> expr end end end defp substitute_aliases(expr, columns, templates) when is_tuple(expr) do expr |> Tuple.to_list() |> Enum.map(fn element when is_tuple(element) -> substitute_aliases(element, columns, templates) elements when is_list(elements) -> Enum.map(elements, &substitute_aliases(&1, columns, templates)) other -> other end) |> List.to_tuple() end defp substitute_aliases(expr, _columns, _templates), do: expr defp resolve_window_refs(columns, windows) do Enum.map(columns, fn {expr, alias_name} -> {resolve_window_refs_expr(expr, windows), alias_name} end) end defp resolve_window_refs_expr({:window, name, args, {:ref, window_name}, filter}, windows) do case Map.fetch(windows, window_name) do {:ok, spec} -> {:window, name, args, spec, filter} :error -> fail("no such window: #{window_name}") end end defp resolve_window_refs_expr(expr, windows) when is_tuple(expr) do expr |> Tuple.to_list() |> Enum.map(fn element when is_tuple(element) -> resolve_window_refs_expr(element, windows) elements when is_list(elements) -> Enum.map(elements, &resolve_window_refs_expr(&1, windows)) other -> other end) |> List.to_tuple() end defp resolve_window_refs_expr(expr, _windows), do: expr # -- DISTINCT / ORDER BY / LIMIT --------------------------------------------- defp distinct(projected, false, _columns, _env), do: projected defp distinct(projected, true, columns, env) do if columns == [] do projected else collations = Enum.map(columns, &expr_collation(elem(&1, 0), env)) # When every column dedups under the default binary collation (the common # case), a canonical `row_key/1` (the same key INTERSECT/EXCEPT use) gives # O(1) membership — O(n) overall instead of the O(n²) all-pairs scan, which # dominated DISTINCT over large result sets. Non-binary collations # (NOCASE/RTRIM) keep the exact linear comparison path. if Enum.all?(collations, &(&1 == :binary)) do distinct_rows_hashed(projected, MapSet.new(), []) else distinct_rows(projected, collations, []) end end end defp distinct_rows_hashed([], _seen, acc), do: Enum.reverse(acc) defp distinct_rows_hashed([{_env, row} = item | rest], seen, acc) do key = row_key(row) if MapSet.member?(seen, key) do distinct_rows_hashed(rest, seen, acc) else distinct_rows_hashed(rest, MapSet.put(seen, key), [item | acc]) end end defp distinct_rows([], _collations, acc), do: Enum.reverse(acc) defp distinct_rows([{env, row} | rest], collations, acc) do if Enum.any?(acc, fn {_env_row, keep_row} -> compare_key_values(row, keep_row, collations) == :eq end) do distinct_rows(rest, collations, acc) else distinct_rows(rest, collations, [{env, row} | acc]) end end defp order(projected, [], _columns, _names), do: projected defp order([], _order_by, _columns, _names), do: [] # Decorate-sort-undecorate: evaluate each row's sort keys once (O(n)) instead # of re-evaluating them inside every comparison (O(n log n)). Direction and # collation are per-term and row-independent (collation is schema-derived), so # they're resolved once up front. Enum.sort stays stable, preserving the # original tie order. defp order([{env0, _row0} | _] = projected, order_by, columns, names) do term_meta = Enum.map(order_by, fn {expr, direction} -> {direction, order_collation(expr, env0, columns, names)} end) # Resolve each term to a per-row key extractor once (position → index, or # output-column name → index, or fall back to evaluating the expression), # rather than re-resolving the name (with a `downcase` + names search) for # every row. extractors = Enum.map(order_by, fn {expr, _direction} -> order_key_extractor(expr, columns, names) end) projected |> Enum.map(fn {env, row} = item -> {Enum.map(extractors, & &1.(env, row)), item} end) |> Enum.sort(&order_keys_before?(elem(&1, 0), elem(&2, 0), term_meta)) |> Enum.map(&elem(&1, 1)) end defp order_keys_before?([a | as], [b | bs], [{direction, collation} | rest]) do case Value.compare(a, b, collation) do :eq -> order_keys_before?(as, bs, rest) :lt -> direction == :asc :gt -> direction == :desc end end defp order_keys_before?([], [], []), do: true defp sort_envs_by_terms(envs, []), do: envs defp sort_envs_by_terms(envs, terms) do Enum.sort(envs, fn env_a, env_b -> compare_term_values(terms, env_a, env_b) end) end defp sort_indexed_envs_by_terms(indexed_envs, []), do: indexed_envs defp sort_indexed_envs_by_terms(indexed_envs, terms) do Enum.sort(indexed_envs, fn {env_a, _}, {env_b, _} -> compare_term_values(terms, env_a, env_b) end) end defp compare_term_values(terms, env_a, env_b) do terms |> Enum.reduce_while(true, fn {expr, direction}, _ -> case Value.compare(eval(expr, env_a), eval(expr, env_b), expr_collation(expr, env_a)) do :eq -> {:cont, true} :lt -> {:halt, direction == :asc} :gt -> {:halt, direction == :desc} end end) end # ORDER BY terms may be 1-based output column positions, output aliases, or # arbitrary expressions over the source row (including aggregates, in # grouped queries). defp order_key_extractor({:collate, expr, _name}, columns, names), do: order_key_extractor(expr, columns, names) defp order_key_extractor({:literal, n}, columns, _names) when is_integer(n) do if n < 1 or n > length(columns), do: fail("ORDER BY term out of range: #{n}") fn _env, row -> Enum.at(row, n - 1) end end defp order_key_extractor({:column, nil, name} = expr, _columns, names) do lowered = Table.key(name) case Enum.find_index(names, &(Table.key(&1) == lowered)) do nil -> fn env, _row -> eval(expr, env) end index -> fn _env, row -> Enum.at(row, index) end end end defp order_key_extractor(expr, _columns, _names), do: fn env, _row -> eval(expr, env) end defp order_collation({:collate, {:literal, _n}, name}, env, _columns, _names), do: normalize_collation!(name, env) defp order_collation({:literal, n}, env, columns, _names) when is_integer(n) do case Enum.at(columns, n - 1) do {expr, _alias} -> expr_collation(expr, env) nil -> :binary end end defp order_collation({:column, nil, name} = expr, env, _columns, names) do lowered = String.downcase(name) case Enum.find_index(names, &(String.downcase(&1) == lowered)) do nil -> expr_collation(expr, env) _index -> expr_collation(expr, env) end end defp order_collation(expr, env, _columns, _names), do: expr_collation(expr, env) defp clamp(rows, _db, nil, _offset), do: rows defp clamp(rows, db, limit_expr, offset_expr) do limit = int_clause(limit_expr, db, "LIMIT") offset = if offset_expr, do: int_clause(offset_expr, db, "OFFSET"), else: 0 rows = Enum.drop(rows, max(offset, 0)) if limit < 0, do: rows, else: Enum.take(rows, limit) end defp int_clause(expr, db, clause) do case eval(expr, constant_env(db)) do n when is_integer(n) -> n _ -> fail("#{clause} must be an integer") end end # -- environments ------------------------------------------------------------------ defp constant_env(db), do: %{db: db, frames: [], group: nil, outer: nil} defp table_env(db, table, rowid, row) do frame = %{table_frame(table, nil) | row: row, rowid: rowid} %{db: db, frames: [frame], group: nil, outer: nil} end defp matches_where?(nil, _env), do: true defp matches_where?(expr, env), do: truth(expr, env) == true # Pre-resolves plain column references in a single-table scan WHERE to # `{:fast_column, key, affinity, collation}` (see the eval/expr_affinity/ # expr_collation clauses). Done once per query so the per-row filter skips the # `String.downcase` + frame visibility search and the affinity/collation # re-resolution that `resolve_column/3` and `comparison_operands/3` otherwise # repeat for every row. Only single-frame scans, only columns that # unambiguously resolve to that frame, and only through row-local predicate # nodes — never into subqueries (different scope) or function args (kept # simple); anything not rewritten still evaluates correctly via the normal path. defp precompile_scan_where(nil, _db, _templates, _outer), do: nil defp precompile_scan_where(where, db, [template] = templates, outer) do visible = template.columns |> Enum.map(&elem(&1, 0)) |> MapSet.new() |> MapSet.difference(template.hidden) env = %{db: db, frames: templates, group: nil, outer: outer} rewritten = where |> rewrite_scan_columns(template.name, visible, env) |> rewrite_outer_frame_columns(outer_frame_lookup(db, outer)) if expr_node_count(rewritten, 0) >= 12 do simplify_where_filter(rewritten) else rewritten end end defp precompile_scan_where(where, db, templates, outer) when is_list(templates) do where |> rewrite_frame_columns(frame_column_lookup(db, templates)) |> rewrite_outer_frame_columns(outer_frame_lookup(db, outer)) end defp precompile_scan_where(where, _db, _templates, _outer), do: where # Pre-resolves projection/hash-key column references against the already # planned frame layout. This is separate from precompile_scan_where/3 because # projection expressions must not receive WHERE-only boolean simplifications. defp precompile_scan_columns(columns, db, templates) do lookup = frame_column_lookup(db, templates) Enum.map(columns, fn {expr, alias} -> {rewrite_frame_columns(expr, lookup), alias} end) end defp frame_column_lookup(db, templates) do templates |> Enum.with_index() |> Enum.reduce(%{db: db, unqualified: %{}, qualified: %{}}, fn {template, index}, acc -> template.columns |> Enum.reject(fn column -> MapSet.member?(template.hidden, column_key(column)) end) |> Enum.reduce(acc, fn column, acc -> key = column_key(column) affinity = column_affinity_meta(column) collation = column_collation_meta(column) |> normalize_collation!(%{db: db}) compiled = {:fast_frame_column, index, key, affinity, collation} qualified_key = {template.name, key} %{ acc | unqualified: put_unqualified_column(acc.unqualified, key, compiled), qualified: Map.put(acc.qualified, qualified_key, compiled) } end) end) end defp put_unqualified_column(columns, key, compiled) do case Map.fetch(columns, key) do :error -> Map.put(columns, key, compiled) {:ok, _existing} -> Map.put(columns, key, :ambiguous) end end defp outer_frame_lookup(_db, nil), do: nil defp outer_frame_lookup(_db, %{group: group}) when group != nil, do: nil defp outer_frame_lookup(db, %{frames: frames}), do: frame_column_lookup(db, frames) # Compiles a (already column-pre-resolved) scan predicate into a closure that # avoids the per-row AST re-dispatch and the per-comparison affinity/collation # function calls — those are baked in at compile time. Only the common # row-local boolean/comparison/IN shapes over static-affinity, binary-collation # operands are compiled; anything else (bare columns, non-binary collation, # subqueries, …) falls back to the tree walker, so results are identical. # Returns `(env -> bool)` matching `matches_where?/2`, or nil for no filter. # A WHERE built entirely from literals and pure operators (no column/row, no # subquery, no function) evaluates to the same boolean for every row, so the # scan can fold it once. Returns `true`/`false`/`nil` for a constant predicate, # `:dynamic` otherwise. Conservative allowlist — anything outside it (column # refs, subqueries, functions like `random()`) is treated as dynamic. defp constant_filter_value(where, db) when not is_nil(where) do if const_predicate?(where), do: Value.truthy(eval(where, constant_env(db))), else: :dynamic end defp constant_filter_value(_where, _db), do: :dynamic defp const_predicate?({:literal, _value}), do: true defp const_predicate?({:not, e}), do: const_predicate?(e) defp const_predicate?({:negate, e}), do: const_predicate?(e) defp const_predicate?({:bitnot, e}), do: const_predicate?(e) defp const_predicate?({:collate, e, _name}), do: const_predicate?(e) defp const_predicate?({:cast, e, _aff}), do: const_predicate?(e) defp const_predicate?({:is, l, r}), do: const_predicate?(l) and const_predicate?(r) defp const_predicate?({:is_not, l, r}), do: const_predicate?(l) and const_predicate?(r) defp const_predicate?({:between, e, lo, hi, _neg}), do: const_predicate?(e) and const_predicate?(lo) and const_predicate?(hi) defp const_predicate?({:binary, op, l, r}) when op in [ :eq, :ne, :lt, :le, :gt, :ge, :and, :or, :add, :sub, :mul, :div, :mod, :bitand, :bitor, :shl, :shr, :concat ], do: const_predicate?(l) and const_predicate?(r) defp const_predicate?(_other), do: false defp compile_scan_filter(nil), do: nil defp compile_scan_filter(where) do # Closures only pay off on large predicates; on a small one the extra call # indirection costs more than the per-row AST dispatch it removes. Below the # threshold, fall back to evaluating the (already column-pre-resolved) tree. if System.get_env("EXSQL_NO_CLOSURE") == nil and expr_node_count(where, 0) >= 2 do pred = compile_bool(where) fn env -> pred.(env) == true end else fn env -> matches_where?(where, env) end end end # A precompiled predicate is "row-local" when evaluating it touches only the # current frame's columns — no `env.db`, `env.outer`, or other frames — so it # can be filtered against a minimal `%{group:, frames:}` map. Conservative: an # allowlist of node types, anything else (functions, LIKE/GLOB which read # `db.case_sensitive_like`, subqueries, unresolved or cross-frame columns) # falls back to the full-env path. defp row_local?({:fast_column, _key, _aff, _coll}), do: true defp row_local?({:literal, _value}), do: true defp row_local?({:not, e}), do: row_local?(e) defp row_local?({:negate, e}), do: row_local?(e) defp row_local?({:bitnot, e}), do: row_local?(e) defp row_local?({:cast, e, _aff}), do: row_local?(e) defp row_local?({:collate, e, _name}), do: row_local?(e) defp row_local?({:is, l, r}), do: row_local?(l) and row_local?(r) defp row_local?({:is_not, l, r}), do: row_local?(l) and row_local?(r) defp row_local?({:between, e, lo, hi, _neg}), do: row_local?(e) and row_local?(lo) and row_local?(hi) defp row_local?({:in, e, list, _neg}) when is_list(list), do: row_local?(e) and Enum.all?(list, &row_local?/1) defp row_local?({:in_cached, e, _members, _aff, _neg}), do: row_local?(e) defp row_local?({:in_membership, e, _membership}), do: row_local?(e) defp row_local?({:binary, op, l, r}) when op in [ :eq, :ne, :lt, :le, :gt, :ge, :and, :or, :add, :sub, :mul, :div, :mod, :bitand, :bitor, :shl, :shr, :concat ], do: row_local?(l) and row_local?(r) defp row_local?(_other), do: false # Boolean context (the WHERE filter and AND/OR/NOT operands): builds a closure # returning `boolean()|nil` directly, skipping the `bool_value/1` (→ 0/1) wrap # and `truthy/1` unwrap that the value-context `compile_pred/1` would pay for # every row. `compiled_comparison/5` already yields a raw boolean. defp compile_bool({:binary, :and, left, right}) do l = compile_bool(left) r = compile_bool(right) fn env -> Value.sql_and(l.(env), r.(env)) end end defp compile_bool({:binary, :or, left, right} = node) do # `col = v1 OR col = v2 OR …` (same column, all literal RHS) is exactly # `col IN (v1, v2, …)` — route it through the IN membership compiler (O(1) # set probe per row) instead of evaluating a chain of comparisons. Big win # for an unindexed column (the indexed case already takes the index via # or_access_path; this only changes the residual per-row filter). case or_chain_to_in(node) do {:ok, col_node, literals} -> bool_fallback({:in, col_node, literals, false}) :no -> l = compile_bool(left) r = compile_bool(right) fn env -> Value.sql_or(l.(env), r.(env)) end end end defp compile_bool({:not, expr}) do c = compile_bool(expr) fn env -> Value.sql_not(c.(env)) end end defp compile_bool({:binary, op, left, right} = node) when op in [:eq, :ne, :lt, :le, :gt, :ge] do aff_l = static_affinity(left) aff_r = static_affinity(right) if aff_l && aff_r && binary_collation?(left) && binary_collation?(right) do compiled_comparison(op, left, aff_l, right, aff_r) else bool_fallback(node) end end defp compile_bool({:between, expr, low, high, negated} = node) do aff_e = static_affinity(expr) aff_lo = static_affinity(low) aff_hi = static_affinity(high) if aff_e && aff_lo && aff_hi && binary_collation?(expr) && binary_collation?(low) && binary_collation?(high) do compiled_between(expr, aff_e, low, aff_lo, high, aff_hi, negated) else bool_fallback(node) end end defp compile_bool({:is, left, right} = node), do: compiled_is(:eq, left, right, node) defp compile_bool({:is_not, left, right} = node), do: compiled_is(:ne, left, right, node) defp compile_bool(node), do: bool_fallback(node) # Recognizes `col = lit OR col = lit OR …` (≥2 disjuncts, all the same fast # column against literals) and returns `{:ok, col_node, [literal_nodes]}` so it # can be compiled as `col IN (…)`. `:no` for any other OR shape (mixed columns, # non-literal RHS, nested non-equality), which keeps the plain OR path. defp or_chain_to_in({:binary, :or, _left, _right} = node) do leaves = or_leaves(node) with [_, _ | _] <- leaves, parsed when is_list(parsed) <- parse_eq_col_literals(leaves), [{key, col_node, _lit} | _] <- parsed, true <- Enum.all?(parsed, fn {k, _, _} -> k == key end) do {:ok, col_node, Enum.map(parsed, fn {_, _, lit} -> lit end)} else _ -> :no end end defp or_leaves({:binary, :or, left, right}), do: or_leaves(left) ++ or_leaves(right) defp or_leaves(other), do: [other] defp parse_eq_col_literals(leaves) do Enum.reduce_while(leaves, [], fn leaf, acc -> case eq_col_literal(leaf) do {_key, _col, _lit} = parsed -> {:cont, [parsed | acc]} :no -> {:halt, :no} end end) |> case do :no -> :no list -> Enum.reverse(list) end end defp eq_col_literal( {:binary, :eq, {:fast_column, key, _aff, _coll} = col, {:literal, _} = lit} ), do: {key, col, lit} defp eq_col_literal( {:binary, :eq, {:literal, _} = lit, {:fast_column, key, _aff, _coll} = col} ), do: {key, col, lit} defp eq_col_literal(_other), do: :no # `IS` / `IS NOT` (incl. `IS NULL`) was falling through to the per-row `eval` # path (`comparison_operands` → per-row `static_affinity`/`comparison_coerce`). # When affinities and collations are statically resolvable — the same gate the # `=`/`<` comparisons use — precompile operands once and compare per row, # mirroring `comparison_operands`' fast branch exactly (binary collation, # affinity coercion folded in) but with `IS` semantics (`compare == :eq`, # never NULL). defp compiled_is(kind, left, right, node) do aff_l = static_affinity(left) aff_r = static_affinity(right) if aff_l && aff_r && binary_collation?(left) && binary_collation?(right) do {coerce_l, coerce_r} = coercion_pair(aff_l, aff_r) build_is(kind, operand(left, coerce_l), operand(right, coerce_r)) else bool_fallback(node) end end defp build_is(kind, {:const, a}, {:const, b}) do result = is_result(kind, a, b) fn _env -> result end end defp build_is(kind, {:col, lk}, {:const, b}), do: fn env -> is_result(kind, col_value(env, lk), b) end defp build_is(kind, {:const, a}, {:col, rk}), do: fn env -> is_result(kind, a, col_value(env, rk)) end defp build_is(kind, {:col, lk}, {:col, rk}), do: fn env -> is_result(kind, col_value(env, lk), col_value(env, rk)) end defp build_is(kind, {:col, lk}, {:fun, rf}), do: fn env -> is_result(kind, col_value(env, lk), rf.(env)) end defp build_is(kind, {:fun, lf}, {:col, rk}), do: fn env -> is_result(kind, lf.(env), col_value(env, rk)) end defp build_is(kind, {:fun, lf}, {:const, b}), do: fn env -> is_result(kind, lf.(env), b) end defp build_is(kind, {:const, a}, {:fun, rf}), do: fn env -> is_result(kind, a, rf.(env)) end defp build_is(kind, {:fun, lf}, {:fun, rf}), do: fn env -> is_result(kind, lf.(env), rf.(env)) end defp is_result(:eq, a, b), do: Value.compare(a, b, :binary) == :eq defp is_result(:ne, a, b), do: Value.compare(a, b, :binary) != :eq # `col BETWEEN lo AND hi` desugars to `col >= lo AND col <= hi`. When `col` is a # bare fast column (no per-operand coercion) and the bounds reduce to constants # — the dominant shape — read the column *once* per row rather than once for # each comparison. Anything else falls back to the two-comparison form. defp compiled_between(expr, aff_e, low, aff_lo, high, aff_hi, negated) do {ce_lo, c_lo} = coercion_pair(aff_e, aff_lo) {ce_hi, c_hi} = coercion_pair(aff_e, aff_hi) e_lo = operand(expr, ce_lo) e_hi = operand(expr, ce_hi) lo = operand(low, c_lo) hi = operand(high, c_hi) case {e_lo, e_hi, lo, hi} do {{:col, key}, {:col, key}, {:const, a}, {:const, b}} -> base = fn env -> v = col_value(env, key) Value.sql_and( Value.compare_op(:ge, v, a, :binary), Value.compare_op(:le, v, b, :binary) ) end if negated, do: fn env -> Value.sql_not(base.(env)) end, else: base _ -> ge = build_compare(:ge, e_lo, lo) le = build_compare(:le, e_hi, hi) if negated do fn env -> Value.sql_not(Value.sql_and(ge.(env), le.(env))) end else fn env -> Value.sql_and(ge.(env), le.(env)) end end end end # Non-boolean or non-fast nodes: evaluate as a value, then reduce to a boolean # exactly as the old `truthy(pred) == true` filter did. defp bool_fallback(node) do c = compile_pred(node) fn env -> Value.truthy(c.(env)) end end defp expr_node_count(_expr, acc) when acc >= 12, do: acc defp expr_node_count(tuple, acc) when is_tuple(tuple) do Enum.reduce(Tuple.to_list(tuple), acc + 1, &expr_node_count/2) end defp expr_node_count(list, acc) when is_list(list) do Enum.reduce(list, acc, &expr_node_count/2) end defp expr_node_count(_other, acc), do: acc defp simplify_where_filter({:binary, :and, _left, _right} = expr) do terms = expr |> where_conjuncts() |> Enum.map(&simplify_where_filter/1) cond do Enum.any?(terms, &false_filter?/1) -> false_filter() contradictory_conjuncts?(terms) -> false_filter() true -> terms |> Enum.reject(&true_filter?/1) |> then(&rebuild_boolean_filter(:and, &1)) end end defp simplify_where_filter({:binary, :or, _left, _right} = expr) do terms = expr |> where_disjuncts() |> Enum.map(&simplify_where_filter/1) cond do Enum.any?(terms, &true_filter?/1) -> true_filter() true -> terms |> Enum.reject(&false_filter?/1) |> then(&rebuild_boolean_filter(:or, &1)) end end defp simplify_where_filter({:between, _expr, {:literal, low}, {:literal, high}, false} = expr) when is_number(low) and is_number(high) do if Value.compare(low, high) == :gt, do: false_filter(), else: expr end defp simplify_where_filter(other), do: other defp false_filter, do: {:literal, 0} defp true_filter, do: {:literal, 1} defp false_filter?({:literal, value}), do: Value.truthy(value) == false defp false_filter?(_expr), do: false defp true_filter?({:literal, value}), do: Value.truthy(value) == true defp true_filter?(_expr), do: false defp rebuild_boolean_filter(:and, []), do: true_filter() defp rebuild_boolean_filter(:or, []), do: false_filter() defp rebuild_boolean_filter(_op, [term]), do: term defp rebuild_boolean_filter(op, [term | terms]), do: Enum.reduce(terms, term, &{:binary, op, &2, &1}) defp contradictory_conjuncts?(terms) do terms |> Enum.flat_map(&where_column_constraints/1) |> Enum.group_by(fn {key, _constraint} -> key end, fn {_key, constraint} -> constraint end) |> Enum.any?(fn {_key, constraints} -> contradictory_column_constraints?(constraints) end) end defp where_column_constraints({:binary, op, left, right}) when op in [:eq, :lt, :le, :gt, :ge] do cond do column_numeric_literal?(left, right) -> [{fast_column_key(left), comparison_constraint(op, literal_value(right))}] column_numeric_literal?(right, left) -> [{fast_column_key(right), comparison_constraint(flip_range_op(op), literal_value(left))}] true -> [] end end defp where_column_constraints({:is, column, {:literal, nil}}) do if fast_column?(column), do: [{fast_column_key(column), :null}], else: [] end defp where_column_constraints({:is_not, column, {:literal, nil}}) do if fast_column?(column), do: [{fast_column_key(column), :not_null}], else: [] end defp where_column_constraints({:between, column, {:literal, low}, {:literal, high}, false}) when is_number(low) and is_number(high) do if fast_column?(column) do [ {fast_column_key(column), {:range, :ge, low}}, {fast_column_key(column), {:range, :le, high}} ] else [] end end defp where_column_constraints(_term), do: [] defp column_numeric_literal?(column, {:literal, value}), do: fast_column?(column) and is_number(value) defp column_numeric_literal?(_column, _literal), do: false defp fast_column?({:fast_column, _key, _affinity, _collation}), do: true defp fast_column?(_expr), do: false defp fast_column_key({:fast_column, key, _affinity, _collation}), do: key defp literal_value({:literal, value}), do: value defp comparison_constraint(:eq, value), do: {:eq, value} defp comparison_constraint(op, value), do: {:range, op, value} defp contradictory_column_constraints?(constraints) do null? = :null in constraints not_null? = :not_null in constraints comparisons? = Enum.any?(constraints, &match?({:eq, _value}, &1)) or Enum.any?(constraints, &match?({:range, _op, _value}, &1)) cond do null? and (not_null? or comparisons?) -> true contradictory_equalities?(constraints) -> true equality_outside_range?(constraints) -> true contradictory_ranges?(constraints) -> true true -> false end end defp contradictory_equalities?(constraints) do constraints |> Enum.flat_map(fn {:eq, value} -> [value] _constraint -> [] end) |> case do [] -> false [first | rest] -> Enum.any?(rest, &(Value.compare(&1, first) != :eq)) end end defp equality_outside_range?(constraints) do equalities = Enum.flat_map(constraints, fn {:eq, value} -> [value] _constraint -> [] end) ranges = Enum.flat_map(constraints, fn {:range, op, value} -> [{op, value}] _constraint -> [] end) Enum.any?(equalities, fn equality -> Enum.any?(ranges, fn {op, value} -> Value.compare_op(op, equality, value) != true end) end) end defp contradictory_ranges?(constraints) do constraints |> Enum.reduce({nil, nil}, fn {:range, op, value}, {lower, upper} when op in [:gt, :ge] -> {strongest_lower_bound(lower, {value, op}), upper} {:range, op, value}, {lower, upper} when op in [:lt, :le] -> {lower, strongest_upper_bound(upper, {value, op})} _constraint, acc -> acc end) |> incompatible_range_bounds?() end defp incompatible_range_bounds?({nil, _upper}), do: false defp incompatible_range_bounds?({_lower, nil}), do: false defp incompatible_range_bounds?({{lower_value, lower_op}, {upper_value, upper_op}}) do case Value.compare(lower_value, upper_value) do :gt -> true :eq -> lower_op == :gt or upper_op == :lt :lt -> false end end defp compile_pred({:binary, :and, left, right}) do l = compile_pred(left) r = compile_pred(right) fn env -> bool_value(Value.sql_and(Value.truthy(l.(env)), Value.truthy(r.(env)))) end end defp compile_pred({:binary, :or, left, right}) do l = compile_pred(left) r = compile_pred(right) fn env -> bool_value(Value.sql_or(Value.truthy(l.(env)), Value.truthy(r.(env)))) end end defp compile_pred({:not, expr}) do c = compile_pred(expr) fn env -> bool_value(Value.sql_not(Value.truthy(c.(env)))) end end defp compile_pred({:binary, op, left, right} = node) when op in [:eq, :ne, :lt, :le, :gt, :ge] do aff_l = static_affinity(left) aff_r = static_affinity(right) if aff_l && aff_r && binary_collation?(left) && binary_collation?(right) do cmp = compiled_comparison(op, left, aff_l, right, aff_r) fn env -> bool_value(cmp.(env)) end else fn env -> eval(node, env) end end end # `expr BETWEEN low AND high` lowers to two comparisons. Compiling it (rather # than falling through to the generic per-row `eval`) lets each comparison # bake its affinity coercion in once — the hot path for the `index/between/*` # corpus files. defp compile_pred({:between, expr, low, high, negated} = node) do aff_e = static_affinity(expr) aff_lo = static_affinity(low) aff_hi = static_affinity(high) if aff_e && aff_lo && aff_hi && binary_collation?(expr) && binary_collation?(low) && binary_collation?(high) do ge = compiled_comparison(:ge, expr, aff_e, low, aff_lo) le = compiled_comparison(:le, expr, aff_e, high, aff_hi) if negated do fn env -> bool_value(Value.sql_not(Value.sql_and(ge.(env), le.(env)))) end else fn env -> bool_value(Value.sql_and(ge.(env), le.(env))) end end else fn env -> eval(node, env) end end end defp compile_pred({:in, expr, list, negated} = node) when is_list(list) do aff = static_affinity(expr) if aff && binary_collation?(expr) do c = compile_pred(expr) case literal_values(list) do {:ok, values} -> membership = compile_in_membership(values, aff, :binary, :blob, negated) fn env -> compiled_in_membership(c.(env), membership) end :error -> members = Enum.map(list, &compile_pred/1) fn env -> in_membership(c.(env), aff, :binary, Enum.map(members, & &1.(env)), :blob, negated) end end else fn env -> eval(node, env) end end end defp compile_pred({:in_cached, expr, members, rhs_affinity, negated} = node) do aff = static_affinity(expr) if aff && binary_collation?(expr) do c = compile_pred(expr) membership = compile_in_membership(members, aff, :binary, rhs_affinity, negated) fn env -> compiled_in_membership(c.(env), membership) end else fn env -> eval(node, env) end end end defp compile_pred({:in_membership, expr, membership}) do c = compile_pred(expr) fn env -> compiled_in_membership(c.(env), membership) end end defp compile_pred({:fast_column, key, _affinity, _collation}) do fn %{group: nil, frames: [frame]} -> frame_cell(frame, key) env -> resolve_column(env, nil, key) end end defp compile_pred({:fast_frame_column, index, key, _affinity, _collation}) do fn env -> fast_frame_column(env, index, key) end end defp compile_pred({:fast_outer_frame_column, index, key, _affinity, _collation}) do fn env -> fast_outer_frame_column(env, index, key) end end defp compile_pred({:literal, value}), do: fn _env -> value end defp compile_pred(expr), do: fn env -> eval(expr, env) end # Builds a closure computing `compare_op(op, left, right, :binary)` with §4.2 # comparison affinity applied. The affinities are static, so the coercion # branch is decided **once** here instead of per row (as # `Value.comparison_coerce/4` would). Returns the raw `boolean()|nil` result # (callers wrap with `bool_value/1`). Mirrors `comparison_coerce/4` exactly. defp compiled_comparison(op, left, aff_l, right, aff_r) do {coerce_l, coerce_r} = coercion_pair(aff_l, aff_r) build_compare(op, operand(left, coerce_l), operand(right, coerce_r)) end # Splits the comparison-affinity rule into the per-operand coercion each side # needs (`nil` = none), so the left/right coercions can be applied independently # (e.g. fused once across both bounds of a BETWEEN). defp coercion_pair(aff_l, aff_r) do case coercion(aff_l, aff_r) do :none -> {nil, nil} :right_num -> {nil, :numeric} :left_num -> {:numeric, nil} :right_text -> {nil, :text} :left_text -> {:text, nil} end end # Classifies an operand as a compile-time `{:const, value}` (literals, with any # affinity coercion folded in once) or a `{:fun, closure}` evaluated per row. defp operand({:literal, value}, nil), do: {:const, value} defp operand({:literal, value}, affinity), do: {:const, Value.apply_affinity(value, affinity)} # An uncoerced column reads inline (`col_value/2`) in the comparison closure # below, saving the separate `fast_column` closure call per row. defp operand({:fast_column, key, _aff, _coll}, nil), do: {:col, key} defp operand(node, nil), do: {:fun, compile_pred(node)} defp operand(node, affinity) do f = compile_pred(node) {:fun, fn env -> Value.apply_affinity(f.(env), affinity) end} end # `fast_column` read, matching its eval clauses exactly. The hot single-frame # path inlines the row map lookup instead of calling `Map.get/2` (which itself # delegates to `Map.get/3`) — two fewer function calls on a per-row-per-column # hot path (tens of millions of calls in scan-heavy queries). defp col_value(%{group: nil, frames: [frame]}, key), do: frame_cell(frame, key) defp col_value(env, key), do: resolve_column(env, nil, key) # Reads a column from a frame, handling both representations: cold frames # (joins built from maps, views, null-frames) carry a `key => value` map row; # the hot single-table scan carries a positional tuple read via `col_index`. defp frame_cell(frame, key) do case frame.row do %{^key => value} -> value row when is_tuple(row) -> case frame.col_index do %{^key => pos} -> ExSQL.Table.cell(row, pos) _ -> nil end _ -> nil end end # Widen a frame's row to a full `key => value` map (for DML/returning paths # that consume whole rows). Map rows pass through; tuple rows are rebuilt. defp frame_row_map(%{row: row}) when is_map(row), do: row defp frame_row_map(%{row: row, col_index: ci}), do: Map.new(ci, fn {k, pos} -> {k, ExSQL.Table.cell(row, pos)} end) # Specialized per the shape of each side, so a constant operand (the common # `column literal`) is captured directly and a column read is inlined, # rather than each going through a closure every row. defp build_compare(op, {:const, a}, {:const, b}) do result = Value.compare_op(op, a, b, :binary) fn _env -> result end end defp build_compare(op, {:col, lk}, {:const, b}), do: fn env -> Value.compare_op(op, col_value(env, lk), b, :binary) end defp build_compare(op, {:const, a}, {:col, rk}), do: fn env -> Value.compare_op(op, a, col_value(env, rk), :binary) end defp build_compare(op, {:col, lk}, {:col, rk}), do: fn env -> Value.compare_op(op, col_value(env, lk), col_value(env, rk), :binary) end defp build_compare(op, {:col, lk}, {:fun, rf}), do: fn env -> Value.compare_op(op, col_value(env, lk), rf.(env), :binary) end defp build_compare(op, {:fun, lf}, {:col, rk}), do: fn env -> Value.compare_op(op, lf.(env), col_value(env, rk), :binary) end defp build_compare(op, {:fun, lf}, {:const, b}), do: fn env -> Value.compare_op(op, lf.(env), b, :binary) end defp build_compare(op, {:const, a}, {:fun, rf}), do: fn env -> Value.compare_op(op, a, rf.(env), :binary) end defp build_compare(op, {:fun, lf}, {:fun, rf}), do: fn env -> Value.compare_op(op, lf.(env), rf.(env), :binary) end defp coercion(aff_l, aff_r) do num_l = aff_l in [:integer, :real, :numeric] num_r = aff_r in [:integer, :real, :numeric] cond do num_l and not num_r -> :right_num num_r and not num_l -> :left_num aff_l == :text and aff_r != :text -> :right_text aff_r == :text and aff_l != :text -> :left_text true -> :none end end # Affinity computable without a row (matches expr_affinity/2): nil only for a # bare {:column,…} that was not pre-resolved, which forces the eval fallback. defp static_affinity({:fast_column, _key, affinity, _collation}), do: affinity defp static_affinity({:fast_frame_column, _index, _key, affinity, _collation}), do: affinity defp static_affinity({:fast_outer_frame_column, _index, _key, affinity, _collation}), do: affinity defp static_affinity({:cast, _expr, affinity}), do: affinity defp static_affinity({:collate, expr, _name}), do: static_affinity(expr) defp static_affinity({:column, _qualifier, _name}), do: nil defp static_affinity(_expr), do: :blob # True when an operand's comparison collation is provably binary (the default): # a pre-resolved column carrying :binary, a literal, or any expression with no # COLLATE and no bare column. Conservative — anything else returns false and # the comparison takes the eval fallback (which resolves collation per row). defp binary_collation?({:fast_column, _key, _affinity, collation}), do: collation == :binary defp binary_collation?({:fast_frame_column, _index, _key, _affinity, collation}), do: collation == :binary defp binary_collation?({:fast_outer_frame_column, _index, _key, _affinity, collation}), do: collation == :binary defp binary_collation?({:literal, _value}), do: true defp binary_collation?({:column, _qualifier, _name}), do: false defp binary_collation?({:collate, _expr, _name}), do: false defp binary_collation?({:cast, expr, _affinity}), do: binary_collation?(expr) defp binary_collation?(expr) when is_tuple(expr) do expr |> Tuple.to_list() |> Enum.all?(&binary_collation_part?/1) end defp binary_collation?(_expr), do: true defp binary_collation_part?(element) when is_tuple(element), do: binary_collation?(element) defp binary_collation_part?(elements) when is_list(elements), do: Enum.all?(elements, &binary_collation_part?/1) defp binary_collation_part?(_element), do: true defp rewrite_scan_columns({:column, qualifier, name} = col, frame_name, visible, env) do key = Table.key(name) if (qualifier == nil or Table.key(qualifier) == frame_name) and MapSet.member?(visible, key) do {:fast_column, key, expr_affinity(col, env), expr_collation(col, env)} else col end end defp rewrite_scan_columns({:binary, op, l, r}, fname, vis, env) do {:binary, op, rewrite_scan_columns(l, fname, vis, env), rewrite_scan_columns(r, fname, vis, env)} end defp rewrite_scan_columns({:not, e}, fname, vis, env), do: {:not, rewrite_scan_columns(e, fname, vis, env)} defp rewrite_scan_columns({:negate, e}, fname, vis, env), do: {:negate, rewrite_scan_columns(e, fname, vis, env)} defp rewrite_scan_columns({:bitnot, e}, fname, vis, env), do: {:bitnot, rewrite_scan_columns(e, fname, vis, env)} defp rewrite_scan_columns({:cast, e, affinity}, fname, vis, env), do: {:cast, rewrite_scan_columns(e, fname, vis, env), affinity} defp rewrite_scan_columns({:collate, e, n}, fname, vis, env), do: {:collate, rewrite_scan_columns(e, fname, vis, env), n} defp rewrite_scan_columns({:is, l, r}, fname, vis, env), do: {:is, rewrite_scan_columns(l, fname, vis, env), rewrite_scan_columns(r, fname, vis, env)} defp rewrite_scan_columns({:is_not, l, r}, fname, vis, env), do: {:is_not, rewrite_scan_columns(l, fname, vis, env), rewrite_scan_columns(r, fname, vis, env)} defp rewrite_scan_columns({:between, e, lo, hi, neg}, fname, vis, env) do {:between, rewrite_scan_columns(e, fname, vis, env), rewrite_scan_columns(lo, fname, vis, env), rewrite_scan_columns(hi, fname, vis, env), neg} end defp rewrite_scan_columns({:in, e, list, neg}, fname, vis, env) when is_list(list) do {:in, rewrite_scan_columns(e, fname, vis, env), Enum.map(list, &rewrite_scan_columns(&1, fname, vis, env)), neg} end # `IN (uncorrelated subquery)`: evaluate the subquery once here (compile time) # instead of for every scanned row. Only provably self-contained single-table # subqueries are hoisted — see hoist_in_subquery/2 — so correctness is # unaffected; a correlated or complex subquery is left as a normal IN. defp rewrite_scan_columns({:in, e, {:select, %Select{} = select}, neg}, fname, vis, env) do rewritten = rewrite_scan_columns(e, fname, vis, env) case hoist_in_subquery(select, env.db) do {:ok, members, rhs_affinity} -> in_membership_expr(rewritten, members, rhs_affinity, neg) :no -> {:in, rewritten, {:select, select}, neg} end end defp rewrite_scan_columns({:subquery, %Select{} = select} = expr, _fname, _vis, env) do select |> hoist_scalar_subquery(env.db, expr) |> compile_correlated_subquery(select, env.db, frame_column_lookup(env.db, env.frames)) end defp rewrite_scan_columns({:exists, %Select{} = select} = expr, _fname, _vis, env) do select |> hoist_exists_subquery(env.db, expr) |> compile_correlated_subquery(select, env.db, frame_column_lookup(env.db, env.frames)) end defp rewrite_scan_columns({:like, e, p, esc, neg}, fname, vis, env) do {:like, rewrite_scan_columns(e, fname, vis, env), rewrite_scan_columns(p, fname, vis, env), esc, neg} end defp rewrite_scan_columns({:glob, e, p, neg}, fname, vis, env), do: {:glob, rewrite_scan_columns(e, fname, vis, env), rewrite_scan_columns(p, fname, vis, env), neg} defp rewrite_scan_columns({:regexp, e, p, neg}, fname, vis, env) do {:regexp, rewrite_scan_columns(e, fname, vis, env), rewrite_scan_columns(p, fname, vis, env), neg} end defp rewrite_scan_columns(other, _fname, _vis, _env), do: other defp rewrite_frame_columns({:column, nil, name} = column, lookup) do case Map.get(lookup.unqualified, Table.key(name)) do {:fast_frame_column, _index, _key, _affinity, _collation} = compiled -> compiled _missing_or_ambiguous -> column end end defp rewrite_frame_columns({:column, qualifier, name} = column, lookup) do case Map.get(lookup.qualified, {Table.key(qualifier), Table.key(name)}) do {:fast_frame_column, _index, _key, _affinity, _collation} = compiled -> compiled nil -> column end end defp rewrite_frame_columns({:subquery, %Select{} = select} = expr, %{db: db} = lookup) do select |> hoist_scalar_subquery(db, expr) |> compile_correlated_subquery(select, db, lookup) end defp rewrite_frame_columns({:exists, %Select{} = select} = expr, %{db: db} = lookup) do select |> hoist_exists_subquery(db, expr) |> compile_correlated_subquery(select, db, lookup) end defp rewrite_frame_columns({:window, _name, _args, _spec, _filter} = expr, _lookup), do: expr defp rewrite_frame_columns(%_struct{} = term, _lookup), do: term defp rewrite_frame_columns(tuple, lookup) when is_tuple(tuple) do tuple |> Tuple.to_list() |> Enum.map(&rewrite_frame_columns(&1, lookup)) |> List.to_tuple() end defp rewrite_frame_columns(list, lookup) when is_list(list) do Enum.map(list, &rewrite_frame_columns(&1, lookup)) end defp rewrite_frame_columns(other, _lookup), do: other defp rewrite_outer_frame_columns(expr, nil), do: expr defp rewrite_outer_frame_columns({:column, nil, _name} = column, _lookup), do: column defp rewrite_outer_frame_columns({:column, qualifier, name} = column, lookup) do case Map.get(lookup.qualified, {Table.key(qualifier), Table.key(name)}) do {:fast_frame_column, index, key, affinity, collation} -> {:fast_outer_frame_column, index, key, affinity, collation} nil -> column end end defp rewrite_outer_frame_columns(tuple, lookup) when is_tuple(tuple) do tuple |> Tuple.to_list() |> Enum.map(&rewrite_outer_frame_columns(&1, lookup)) |> List.to_tuple() end defp rewrite_outer_frame_columns(list, lookup) when is_list(list) do Enum.map(list, &rewrite_outer_frame_columns(&1, lookup)) end defp rewrite_outer_frame_columns(other, _lookup), do: other # Evaluates an IN-subquery once if it is provably uncorrelated: a single plain # base table in FROM and every referenced column is an unqualified column of # that table (or a rowid name), with no nested subquery. Anything qualified, # multi-table, or containing a subquery is declined (`:no`) and left to run # per-row — conservative, so a correlated subquery is never wrongly cached. defp hoist_in_subquery(%Select{} = select, db) do with {:ok, local_columns} <- subquery_local_columns(select, db), true <- all_refs_local?(select, local_columns) do result = query_result(db, select, nil) {:ok, Enum.map(result.rows, &hd/1), List.first(result.affinities) || :blob} else _ -> :no end end defp in_membership_expr(expr, members, rhs_affinity, negated) do case static_affinity(expr) do nil -> {:in_cached, expr, members, rhs_affinity, negated} affinity -> if binary_collation?(expr) do {:in_membership, expr, compile_in_membership(members, affinity, :binary, rhs_affinity, negated)} else {:in_cached, expr, members, rhs_affinity, negated} end end end defp compile_correlated_subquery( {:subquery, %Select{}}, %Select{columns: [{{:function, "count", :star}, nil}]} = select, db, outer_lookup ) do case correlated_table_filter(select, db, outer_lookup) do {:ok, table_key, template, filter} -> {:correlated_count, table_key, template, filter} :no -> {:subquery, select} end end defp compile_correlated_subquery({:exists, %Select{}}, %Select{} = select, db, outer_lookup) do case correlated_table_filter(select, db, outer_lookup) do {:ok, table_key, template, filter} -> {:correlated_exists, table_key, template, filter} :no -> {:exists, select} end end defp compile_correlated_subquery(expr, _select, _db, _outer_lookup), do: expr defp correlated_table_filter( %Select{ from: {:table, name, alias_name}, group_by: [], having: nil, windows: windows, order_by: [], limit: nil, offset: nil, distinct: false } = select, db, outer_lookup ) when windows == %{} do table_key = table_source_key(name) case plain_table(db, table_key) do %Table{} = table -> template = table_frame(table, alias_name) filter = correlated_table_filter(select.where, db, template, outer_lookup) {:ok, table_key, template, filter} nil -> :no end end defp correlated_table_filter(_select, _db, _outer_lookup), do: :no defp correlated_table_filter(nil, _db, _template, _outer_lookup), do: nil defp correlated_table_filter(where, db, template, outer_lookup) do where |> precompile_scan_where(db, [template], nil) |> rewrite_outer_frame_columns(outer_lookup) |> compile_scan_filter() end defp hoist_scalar_subquery(%Select{} = select, db, fallback) do if uncorrelated_subquery?(select, db) do db |> query_result(select, nil) |> scalar_subquery_literal() else fallback end end defp scalar_subquery_literal(%Result{rows: [[value | _] | _]}), do: {:literal, value} defp scalar_subquery_literal(%Result{rows: []}), do: {:literal, nil} defp hoist_exists_subquery(%Select{} = select, db, fallback) do if uncorrelated_subquery?(select, db) do {:literal, bool_value(query_result(db, select, nil).rows != [])} else fallback end end defp uncorrelated_subquery?(%Select{} = select, db) do with {:ok, local_columns} <- subquery_local_columns(select, db) do all_refs_local?(select, local_columns) else _ -> false end end # Union of column keys of every plain single-table FROM anywhere in the # subquery tree (the subquery's own FROM plus any nested subqueries'). Bails on # a join/subquery/non-plain FROM (more complex scoping than we analyze here). defp subquery_local_columns(%Select{from: from} = select, db) do with {:ok, base} <- from_table_columns(from, db) do select |> nested_selects() |> Enum.reduce_while({:ok, base}, fn sub, {:ok, acc} -> case subquery_local_columns(sub, db) do {:ok, cols} -> {:cont, {:ok, MapSet.union(acc, cols)}} :error -> {:halt, :error} end end) end end defp from_table_columns({:table, name, _alias}, db) do case plain_table(db, table_source_key(name)) do %Table{} = table -> {:ok, MapSet.new(table.columns, &Table.key(&1.name))} _ -> :error end end defp from_table_columns(_other, _db), do: :error defp nested_selects(%Select{} = select) do select |> select_scope_exprs() |> Enum.flat_map(&collect_nested_selects/1) end defp collect_nested_selects({:select, %Select{} = s}), do: [s | nested_selects(s)] defp collect_nested_selects({:subquery, %Select{} = s}), do: [s | nested_selects(s)] defp collect_nested_selects({:subquery, %Select{} = s, _alias}), do: [s | nested_selects(s)] defp collect_nested_selects({:scalar_subquery, %Select{} = s}), do: [s | nested_selects(s)] defp collect_nested_selects({:exists, %Select{} = s}), do: [s | nested_selects(s)] defp collect_nested_selects(tuple) when is_tuple(tuple), do: tuple |> Tuple.to_list() |> Enum.flat_map(&collect_nested_selects/1) defp collect_nested_selects(list) when is_list(list), do: Enum.flat_map(list, &collect_nested_selects/1) defp collect_nested_selects(_other), do: [] defp select_scope_exprs(%Select{} = s) do column_exprs = Enum.flat_map(s.columns, fn {expr, _alias} -> [expr] _star -> [] end) order_exprs = Enum.map(s.order_by, &elem(&1, 0)) Enum.reject([s.where, s.having | column_exprs] ++ order_exprs ++ s.group_by, &is_nil/1) end # Every column the subquery (and its nested subqueries) references is an # unqualified column of one of the subquery-tree's own tables — i.e. it never # reaches the outer query, so the subquery is uncorrelated and safe to hoist. # A column whose name is absent from the union must resolve to the outer scope # (correlated) and forces `:no`; a qualified column is declined conservatively. defp all_refs_local?(%Select{} = select, union) do select |> select_scope_exprs() |> Enum.all?(&expr_refs_local?(&1, union)) end defp expr_refs_local?(%Select{} = sub, union), do: all_refs_local?(sub, union) defp expr_refs_local?({:column, nil, name}, union) do key = Table.key(name) MapSet.member?(union, key) or key in @rowid_names end defp expr_refs_local?({:column, _qualifier, _name}, _union), do: false defp expr_refs_local?(tuple, union) when is_tuple(tuple) do tuple |> Tuple.to_list() |> Enum.all?(&expr_refs_local?(&1, union)) end defp expr_refs_local?(list, union) when is_list(list) do Enum.all?(list, &expr_refs_local?(&1, union)) end defp expr_refs_local?(_other, _union), do: true # -- column resolution ---------------------------------------------------------- # # Unqualified names search the visible columns of every frame (USING/NATURAL # join columns are hidden on the right side, so they resolve to the left # table); qualified names pick a frame by alias. Either falls back to the # outer query's environment, which is what makes subqueries correlated. defp resolve_column(env, nil, name) do key = Table.key(name) case Enum.filter(env.frames, &visible?(&1, key)) do [frame] -> frame_cell(frame, key) [_ | _] -> fail("ambiguous column name: #{name}") [] -> rowid_frames = if key in @rowid_names, do: Enum.filter(env.frames, & &1.has_rowid), else: [] case rowid_frames do [frame] -> frame.rowid [_ | _] -> fail("ambiguous column name: #{name}") [] when env.outer != nil -> eval({:column, nil, name}, env.outer) [] -> fail("no such column: #{name}") end end end defp resolve_column(env, qualifier, name) do qkey = Table.key(qualifier) case Enum.find(env.frames, &(&1.name == qkey)) do nil -> if env.outer != nil do eval({:column, qualifier, name}, env.outer) else fail("no such column: #{qualifier}.#{name}") end frame -> key = Table.key(name) cond do has_column?(frame, key) -> frame_cell(frame, key) key in @rowid_names and frame.has_rowid -> frame.rowid true -> fail("no such column: #{qualifier}.#{name}") end end end defp visible?(frame, key), do: has_column?(frame, key) and not MapSet.member?(frame.hidden, key) defp has_column?(%{columns_by_key: columns}, key), do: Map.has_key?(columns, key) defp has_column?(frame, key), do: List.keymember?(frame.columns, key, 0) defp column_key({key, _display, _affinity}), do: key defp column_key({key, _display, _affinity, _collation}), do: key defp column_display({_key, display, _affinity}), do: display defp column_display({_key, display, _affinity, _collation}), do: display defp column_affinity_meta({_key, _display, affinity}), do: affinity defp column_affinity_meta({_key, _display, affinity, _collation}), do: affinity defp column_collation_meta({_key, _display, _affinity}), do: nil defp column_collation_meta({_key, _display, _affinity, collation}), do: collation # -- expression affinity ---------------------------------------------------------- # # Mirrors sqlite3ExprAffinity: column references carry their column's # declared affinity, CAST its target, rowid INTEGER, everything else none # (which BLOB also means). Used to apply comparison affinity (§4.2). defp expr_affinity({:column, qualifier, name}, env) do case env.group do [first | _] -> expr_affinity({:column, qualifier, name}, first) [] -> :blob nil -> column_affinity(env, qualifier, name) end end defp expr_affinity({:fast_column, _key, affinity, _collation}, _env), do: affinity defp expr_affinity({:fast_frame_column, _index, _key, affinity, _collation}, _env), do: affinity defp expr_affinity({:fast_outer_frame_column, _index, _key, affinity, _collation}, _env), do: affinity defp expr_affinity({:collate, expr, _name}, env), do: expr_affinity(expr, env) defp expr_affinity({:cast, _expr, affinity}, _env), do: affinity defp expr_affinity(_expr, _env), do: :blob defp column_affinity(env, nil, name) do key = Table.key(name) case Enum.filter(env.frames, &visible?(&1, key)) do [frame] -> frame_affinity(frame, key) [_ | _] -> :blob [] -> cond do key in @rowid_names and Enum.any?(env.frames, & &1.has_rowid) -> :integer env.outer != nil -> expr_affinity({:column, nil, name}, env.outer) true -> :blob end end end defp column_affinity(env, qualifier, name) do qkey = Table.key(qualifier) case Enum.find(env.frames, &(&1.name == qkey)) do nil -> if env.outer != nil, do: expr_affinity({:column, qualifier, name}, env.outer), else: :blob frame -> key = Table.key(name) cond do has_column?(frame, key) -> frame_affinity(frame, key) key in @rowid_names and frame.has_rowid -> :integer true -> :blob end end end defp frame_affinity(frame, key) do frame |> frame_column(key) |> column_affinity_meta() end defp expr_collation({:collate, _expr, name}, env), do: normalize_collation!(name, env) defp expr_collation({:fast_column, _key, _affinity, collation}, _env), do: collation defp expr_collation({:fast_frame_column, _index, _key, _affinity, collation}, _env), do: collation defp expr_collation({:fast_outer_frame_column, _index, _key, _affinity, collation}, _env), do: collation defp expr_collation({:column, qualifier, name}, env) do case env.group do [first | _] -> expr_collation({:column, qualifier, name}, first) [] -> :binary nil -> column_collation(env, qualifier, name) end end defp expr_collation(expr, env) when is_tuple(expr) do explicit_collation(expr, env) || inherited_collation(expr, env) || :binary end defp expr_collation(_expr, _env), do: :binary defp explicit_collation({:collate, _expr, name}, env), do: normalize_collation!(name, env) defp explicit_collation({:fast_column, _key, _affinity, _collation}, _env), do: nil defp explicit_collation({:fast_frame_column, _index, _key, _affinity, _collation}, _env), do: nil defp explicit_collation({:fast_outer_frame_column, _index, _key, _affinity, _collation}, _env), do: nil defp explicit_collation(expr, env) when is_tuple(expr) do expr |> Tuple.to_list() |> Enum.find_value(fn element when is_tuple(element) -> explicit_collation(element, env) elements when is_list(elements) -> Enum.find_value(elements, &explicit_collation(&1, env)) _ -> nil end) end defp explicit_collation(_expr, _env), do: nil defp inherited_collation({:cast, expr, _affinity}, env), do: expr_collation(expr, env) defp inherited_collation({:binary, :concat, left, _right}, env), do: expr_collation(left, env) defp inherited_collation(_expr, _env), do: nil defp column_collation(env, nil, name) do key = Table.key(name) case Enum.filter(env.frames, &visible?(&1, key)) do [frame] -> frame_collation(frame, key, env) [] when env.outer != nil -> expr_collation({:column, nil, name}, env.outer) _ -> :binary end end defp column_collation(env, qualifier, name) do qkey = Table.key(qualifier) case Enum.find(env.frames, &(&1.name == qkey)) do nil -> if env.outer != nil, do: expr_collation({:column, qualifier, name}, env.outer), else: :binary frame -> key = Table.key(name) if has_column?(frame, key), do: frame_collation(frame, key, env), else: :binary end end defp frame_collation(frame, key, env) do frame |> frame_column(key) |> column_collation_meta() |> normalize_collation!(env) end defp frame_column(%{columns_by_key: columns}, key), do: Map.fetch!(columns, key) defp frame_column(frame, key), do: List.keyfind(frame.columns, key, 0) defp normalize_collation!(nil, _env), do: :binary # Already-normalized built-in collations (the overwhelming majority, e.g. the # atom stored on every index) short-circuit the `to_string/1` + Unicode # `String.downcase/1` the general clause would otherwise pay — which on a # per-index-entry path (range scans) was a measurable hot spot. defp normalize_collation!(:binary, _env), do: :binary defp normalize_collation!(:nocase, _env), do: :nocase defp normalize_collation!(:rtrim, _env), do: :rtrim defp normalize_collation!(name, env) do case String.downcase(to_string(name)) do "binary" -> :binary "nocase" -> :nocase "rtrim" -> :rtrim other -> custom_collation!(env.db, other) end end defp custom_collation!(db, name) do case Database.fetch_collation(db, name) do {:ok, %{callback: callback}} -> {:custom, name, fn a, b -> call_collation(callback, name, a, b) end} :error -> fail("no such collation sequence: #{name}") end end defp call_collation(callback, name, a, b) do callback |> apply([a, b]) |> normalize_collation_result(name) rescue e in Error -> reraise e, __STACKTRACE__ e -> fail("user-defined collation #{name} raised: #{Exception.message(e)}") end defp normalize_collation_result({:ok, result}, name), do: normalize_collation_result(result, name) defp normalize_collation_result({:error, message}, name), do: fail("user-defined collation #{name} error: #{message}") defp normalize_collation_result(:lt, _name), do: :lt defp normalize_collation_result(:eq, _name), do: :eq defp normalize_collation_result(:gt, _name), do: :gt defp normalize_collation_result(result, _name) when is_integer(result) and result < 0, do: :lt defp normalize_collation_result(0, _name), do: :eq defp normalize_collation_result(result, _name) when is_integer(result) and result > 0, do: :gt defp normalize_collation_result(_result, name), do: fail("user-defined collation #{name} returned unsupported value") # Evaluates both sides of a comparison and applies comparison affinity. defp comparison_operands(left, right, env) do aff_l = static_affinity(left) aff_r = static_affinity(right) if aff_l && aff_r && binary_collation?(left) && binary_collation?(right) do {a, b} = Value.comparison_coerce(eval(left, env), aff_l, eval(right, env), aff_r) {a, b, :binary} else comparison_operands_dynamic(left, right, env) end end defp comparison_operands_dynamic(left, right, env) do {a, b} = Value.comparison_coerce( eval(left, env), expr_affinity(left, env), eval(right, env), expr_affinity(right, env) ) {a, b, comparison_collation(left, right, env)} end defp comparison_collation(left, right, env) do case explicit_collation(left, env) do nil -> case explicit_collation(right, env) do nil -> case expr_collation(left, env) do :binary -> expr_collation(right, env) collation -> collation end collation -> collation end collation -> collation end end defp ensure_raise_allowed!(env) do if env.db.active_triggers == [] do fail("RAISE() may only be used within a trigger-program") end end # -- expression evaluation ------------------------------------------------------------ # # Booleans are SQL values: comparisons yield 1/0/NULL, matching SQLite where # any expression result is a storage-class value. defp eval({:literal, value}, _env), do: value # A parameter that was never bound evaluates to NULL, as in the C API. defp eval({:param, _index, _raw}, _env), do: nil # RAISE() inside a trigger program: IGNORE abandons the row operation # (caught by the trigger runner); the other forms abort with the message. defp eval({:raise, :ignore}, env) do ensure_raise_allowed!(env) throw(:raise_ignore) end defp eval({:raise, _kind, message}, env) do ensure_raise_allowed!(env) fail(Value.to_text(eval(message, env))) end # `->` returns JSON text, `->>` the SQL value; the right side may be a # full `$..` path, a bare object key, or an array index. defp eval({:json_arrow, left, right}, env) do json_arrow_value(eval(left, env), eval(right, env), &json_subtype/1) end defp eval({:json_arrow_text, left, right}, env) do json_arrow_value(eval(left, env), eval(right, env), &Json.to_sql/1) end defp eval({:function, "changes", []}, env), do: env.db.changes defp eval({:function, "total_changes", []}, env), do: env.db.total_changes defp eval({:function, "last_insert_rowid", []}, env), do: env.db.last_insert_rowid defp eval({:column, qualifier, name}, env) do case env.group do # A bare column in an aggregate query takes its value from an # arbitrary row; SQLite uses the last visited, we use the first. [first | _] -> eval({:column, qualifier, name}, first) [] -> nil nil -> resolve_column(env, qualifier, name) end end # A `{:column, …}` pre-resolved by `precompile_scan_where/3` to a direct row # lookup of the single scan frame, carrying its precomputed affinity and # collation. Equivalent to the `[frame]` branch of `resolve_column/3`, but # skips the per-row `String.downcase` + frame visibility search. The second # head is a defensive fallback for any unexpected env shape. defp eval({:fast_column, key, _affinity, _collation}, %{group: nil, frames: [frame]}) do frame_cell(frame, key) end defp eval({:fast_column, key, _affinity, _collation}, env) do resolve_column(env, nil, key) end defp eval({:fast_frame_column, index, key, _affinity, _collation}, env) do fast_frame_column(env, index, key) end defp eval({:fast_outer_frame_column, index, key, _affinity, _collation}, env) do fast_outer_frame_column(env, index, key) end defp eval({:binary, :and, left, right}, env) do bool_value(Value.sql_and(truth(left, env), truth(right, env))) end defp eval({:binary, :or, left, right}, env) do bool_value(Value.sql_or(truth(left, env), truth(right, env))) end defp eval({:binary, op, left, right}, env) when op in [:eq, :ne, :lt, :le, :gt, :ge] do {a, b, collation} = comparison_operands(left, right, env) bool_value(Value.compare_op(op, a, b, collation)) end defp eval({:binary, op, left, right}, env) when op in [:add, :sub, :mul, :div, :mod] do Value.arithmetic(op, eval(left, env), eval(right, env)) end defp eval({:binary, op, left, right}, env) when op in [:bitand, :bitor, :shl, :shr] do Value.bitwise(op, eval(left, env), eval(right, env)) end defp eval({:binary, :concat, left, right}, env) do Value.concat(eval(left, env), eval(right, env)) end defp eval({:bitnot, expr}, env), do: Value.bitnot(eval(expr, env)) defp eval({:cast, expr, affinity}, env), do: Value.cast(eval(expr, env), affinity) defp eval({:collate, expr, _name}, env), do: eval(expr, env) defp eval({:not, expr}, env), do: bool_value(Value.sql_not(truth(expr, env))) defp eval({:negate, expr}, env) do case eval(expr, env) do nil -> nil n when is_integer(n) -> # Negating the smallest 64-bit integer overflows into REAL. if Value.out_of_int64_range?(-n), do: -n * 1.0, else: -n n when is_float(n) -> -n other -> Value.arithmetic(:sub, 0, other) end end # IS / IS NOT never return NULL: NULL IS NULL is true. defp eval({:is, left, right}, env) do {a, b, collation} = comparison_operands(left, right, env) bool_value(Value.compare(a, b, collation) == :eq) end defp eval({:is_not, left, right}, env) do {a, b, collation} = comparison_operands(left, right, env) bool_value(Value.compare(a, b, collation) != :eq) end # IN comparisons use the affinity of the left side and (for subqueries) # the result column; affinities of columns inside an expression list are # ignored, as in SQLite. defp eval({:in, expr, {:select, select}, negated}, env) do result = query_result(env.db, select, env) members = Enum.map(result.rows, &hd/1) rhs_affinity = List.first(result.affinities) || :blob in_membership( eval(expr, env), expr_affinity(expr, env), expr_collation(expr, env), members, rhs_affinity, negated ) end # An `IN (uncorrelated subquery)` whose member set + result affinity were # evaluated once by `hoist_in_subquery/2` (see `rewrite_scan_columns/4`); # equivalent to the clause above but without re-running the subquery per row. defp eval({:in_cached, expr, members, rhs_affinity, negated}, env) do in_membership( eval(expr, env), expr_affinity(expr, env), expr_collation(expr, env), members, rhs_affinity, negated ) end defp eval({:in, expr, list, negated}, env) when is_list(list) do members = Enum.map(list, &eval(&1, env)) in_membership( eval(expr, env), expr_affinity(expr, env), expr_collation(expr, env), members, :blob, negated ) end defp eval({:in_membership, expr, membership}, env) do compiled_in_membership(eval(expr, env), membership) end defp eval({:between, expr, low, high, negated}, env) do {value_low, low_value, low_collation} = comparison_operands(expr, low, env) {value_high, high_value, high_collation} = comparison_operands(expr, high, env) result = Value.sql_and( Value.compare_op(:ge, value_low, low_value, low_collation), Value.compare_op(:le, value_high, high_value, high_collation) ) bool_value(if negated, do: Value.sql_not(result), else: result) end defp eval({:like, expr, pattern, escape, negated}, env) do result = if escape == nil do Value.like(eval(expr, env), eval(pattern, env), env.db.case_sensitive_like) else case eval(escape, env) do nil -> nil escape_value -> Value.like( eval(expr, env), eval(pattern, env), like_escape(escape_value), env.db.case_sensitive_like ) end end bool_value(if negated, do: Value.sql_not(result), else: result) end defp eval({:glob, expr, pattern, negated}, env) do result = Value.glob(eval(expr, env), eval(pattern, env)) bool_value(if negated, do: Value.sql_not(result), else: result) end defp eval({:regexp, expr, pattern, negated}, env) do result = regexp_match(eval(expr, env), eval(pattern, env)) bool_value(if negated, do: Value.sql_not(result), else: result) end defp eval({:match, _expr, _pattern, _negated}, _env) do fail("unable to use function MATCH in the requested context") end # A scalar subquery's value is the first column of its first row; an empty # result is NULL. defp eval({:subquery, select}, env) do case query_result(env.db, select, env).rows do [[value | _] | _] -> value [] -> nil end end defp eval({:exists, select}, env) do bool_value(query_result(env.db, select, env).rows != []) end defp eval({:correlated_count, table_key, template, filter}, env) do env.db.tables |> Map.fetch!(table_key) |> Table.scan() |> Enum.count(fn {rowid, row} -> correlated_table_match?(env, template, filter, rowid, row) end) end defp eval({:correlated_exists, table_key, template, filter}, env) do exists? = env.db.tables |> Map.fetch!(table_key) |> Table.scan() |> Enum.any?(fn {rowid, row} -> correlated_table_match?(env, template, filter, rowid, row) end) bool_value(exists?) end defp eval({:case, nil, branches, else_expr}, env) do Enum.find_value(branches, fn {when_expr, then_expr} -> if truth(when_expr, env) == true, do: {:matched, eval(then_expr, env)} end) |> case do {:matched, value} -> value nil -> if else_expr, do: eval(else_expr, env) end end defp eval({:case, operand, branches, else_expr}, env) do Enum.find_value(branches, fn {when_expr, then_expr} -> {value, when_value, collation} = comparison_operands(operand, when_expr, env) if Value.compare_op(:eq, value, when_value, collation) == true, do: {:matched, eval(then_expr, env)} end) |> case do {:matched, result} -> result nil -> if else_expr, do: eval(else_expr, env) end end defp eval({:function, name, args}, env) do cond do name in @window_functions -> fail("misuse of window function #{name}()") aggregate_call?(env.db, name, args) -> case env.group do group when is_list(group) -> aggregate(name, args, group, env) nil -> fail("misuse of aggregate function #{name}()") end args == :star -> fail("wrong use of '*' with function #{name}()") true -> scalar(env, name, Enum.map(args, &eval(&1, env))) end end defp eval({:filter_function, name, args, filter}, env) do cond do aggregate_call?(env.db, name, args) -> case env.group do group when is_list(group) -> group = Enum.filter(group, &(truth(filter, &1) == true)) aggregate(name, args, group, env) nil -> fail("misuse of aggregate function #{name}()") end true -> fail("FILTER may not be used with non-aggregate #{name}()") end end defp eval({:window, name, _args, _spec, _filter} = expr, env) do case Map.fetch(Map.get(env, :windows, %{}), expr) do {:ok, value} -> value :error -> fail("misuse of window function #{name}()") end end defp correlated_table_match?(_outer, _template, nil, _rowid, _row), do: true defp correlated_table_match?(outer, template, filter, rowid, row) do filter.(%{ db: outer.db, frames: [%{template | row: row, rowid: rowid}], group: nil, outer: outer }) end defp in_membership(value, affinity, collation, members, rhs_affinity, negated) do match? = fn member -> {a, b} = Value.comparison_coerce(value, affinity, member, rhs_affinity) Value.compare_op(:eq, a, b, collation) == true end membership = cond do value == nil and members != [] -> nil Enum.any?(members, match?) -> true Enum.any?(members, &is_nil/1) -> nil true -> false end bool_value(if negated, do: Value.sql_not(membership), else: membership) end defp compile_in_membership(members, affinity, :binary, rhs_affinity, negated) do {member_keys, numeric_member_keys, has_null?} = Enum.reduce(members, {MapSet.new(), MapSet.new(), false}, fn nil, {member_keys, numeric_member_keys, _has_null?} -> {member_keys, numeric_member_keys, true} member, {member_keys, numeric_member_keys, has_null?} -> {_left, coerced_member} = Value.comparison_coerce(nil, affinity, member, rhs_affinity) numeric_member_keys = if number_value?(coerced_member), do: MapSet.put(numeric_member_keys, numeric_value_key(coerced_member)), else: numeric_member_keys {MapSet.put(member_keys, value_key(coerced_member)), numeric_member_keys, has_null?} end) %{ affinity: affinity, rhs_affinity: rhs_affinity, member_keys: member_keys, numeric_member_keys: numeric_member_keys, numeric_probe?: numeric_probe_fast?(affinity, rhs_affinity), has_members?: members != [], has_null?: has_null?, negated?: negated } end defp compiled_in_membership(value, membership) do result = cond do value == nil and membership.has_members? -> nil value == nil -> false membership.numeric_probe? and number_value?(value) and MapSet.member?(membership.numeric_member_keys, numeric_value_key(value)) -> true membership.numeric_probe? and number_value?(value) -> if membership.has_null?, do: nil, else: false MapSet.member?( membership.member_keys, value |> in_lookup_value(membership.affinity, membership.rhs_affinity) |> value_key() ) -> true membership.has_null? -> nil true -> false end bool_value(if membership.negated?, do: Value.sql_not(result), else: result) end defp numeric_probe_fast?(_affinity, :text), do: false defp numeric_probe_fast?(_affinity, _rhs_affinity), do: true defp number_value?(value), do: is_integer(value) or is_float(value) defp numeric_value_key(value) when is_integer(value), do: value defp numeric_value_key(value) when is_float(value) do truncated = trunc(value) if truncated == value, do: truncated, else: value end defp in_lookup_value(value, affinity, rhs_affinity) do {coerced_value, _member} = Value.comparison_coerce(value, affinity, nil, rhs_affinity) coerced_value end defp truth(expr, env), do: expr |> eval(env) |> Value.truthy() defp like_escape(value) do text = Value.to_text(value) if String.length(text) == 1 do text else fail("ESCAPE expression must be a single character") end end defp bool_value(nil), do: nil defp bool_value(true), do: 1 defp bool_value(false), do: 0 defp fast_frame_column(%{frames: frames}, index, key) do frame = :lists.nth(index + 1, frames) frame_cell(frame, key) end defp fast_outer_frame_column(%{outer: outer}, index, key) when is_map(outer) do fast_frame_column(outer, index, key) end # -- scalar functions -------------------------------------------------------------- defp scalar(env, name, args) do case Database.fetch_scalar_function(env.db, name, length(args)) do {:ok, function} -> call_scalar_function(function, Enum.map(args, &sql_value/1)) :error -> if (Database.scalar_function_exists?(env.db, name) or Database.aggregate_function_exists?(env.db, name)) and not (Map.has_key?(@scalar_arity, name) or name in @aggregate_functions) do fail("wrong number of arguments to function #{name}()") else builtin_scalar(env, name, args) end end end defp builtin_scalar(env, "like", [pattern, v]), do: bool_value(Value.like(v, pattern, env.db.case_sensitive_like)) defp builtin_scalar(_env, "like", [_pattern, _v, nil]), do: nil defp builtin_scalar(env, "like", [pattern, v, escape]), do: bool_value(Value.like(v, pattern, like_escape(escape), env.db.case_sensitive_like)) defp builtin_scalar(_env, name, args), do: scalar(name, args) defp call_scalar_function(%{name: name, callback: callback}, args) do callback |> apply(args) |> normalize_scalar_function_result(name) rescue e in Error -> reraise e, __STACKTRACE__ e -> fail("user-defined function #{name}() raised: #{Exception.message(e)}") end defp normalize_scalar_function_result({:ok, value}, name), do: normalize_scalar_function_result(value, name) defp normalize_scalar_function_result({:error, message}, name), do: fail("user-defined function #{name}() error: #{message}") defp normalize_scalar_function_result(nil, _name), do: nil defp normalize_scalar_function_result(value, _name) when is_integer(value), do: value defp normalize_scalar_function_result(value, _name) when is_float(value), do: value defp normalize_scalar_function_result(value, _name) when is_binary(value), do: value defp normalize_scalar_function_result({:blob, value}, _name) when is_binary(value), do: {:blob, value} defp normalize_scalar_function_result(_value, name), do: fail("user-defined function #{name}() returned unsupported value") # -- JSON functions (json1) --------------------------------------------------- @jsonb_magic "ExSQL.JSONB\0" defp scalar("json", [nil]), do: nil defp scalar("json", [v]), do: v |> json_parse!() |> json_subtype() defp scalar("jsonb", [nil]), do: nil defp scalar("jsonb", [v]), do: v |> json_parse!() |> jsonb_blob() defp scalar("json_valid", [nil]), do: nil defp scalar("json_valid", [{:blob, _}]), do: 0 defp scalar("json_valid", [v]) do if match?({:ok, _}, Json.parse(Value.to_text(v))), do: 1, else: 0 end defp scalar("json_valid", [v, flags]) do flags = json_valid_flags!(flags) cond do v == nil -> nil match?({:blob, _}, v) -> jsonb_valid(v, flags) Bitwise.band(flags, 0b0001) != 0 and match?({:ok, _}, Json.parse(Value.to_text(v))) -> 1 Bitwise.band(flags, 0b0010) != 0 -> if match?({:ok, _}, Json.parse_json5(Value.to_text(v))), do: 1, else: 0 Bitwise.band(flags, 0b0001) != 0 -> if match?({:ok, _}, Json.parse(Value.to_text(v))), do: 1, else: 0 true -> 0 end end defp scalar("json_quote", [v]), do: v |> json_from_sql!() |> Json.render() defp scalar("json_array", args), do: json_subtype({:array, Enum.map(args, &json_from_sql!/1)}) defp scalar("jsonb_array", args), do: jsonb_blob({:array, Enum.map(args, &json_from_sql!/1)}) defp scalar("json_object", args) do json_object(args) |> json_subtype() end defp scalar("jsonb_object", args) do args |> json_object() |> jsonb_blob() end defp scalar("json_extract", [nil | _paths]), do: nil defp scalar("json_extract", [v, path]) do case Json.get(json_parse!(v), json_path!(path)) do {:ok, found} -> json_to_sql(found) :missing -> nil end end defp scalar("json_extract", [v | paths]) do jv = json_parse!(v) items = Enum.map(paths, fn path -> case Json.get(jv, json_path!(path)) do {:ok, found} -> found :missing -> :null end end) json_subtype({:array, items}) end defp scalar("jsonb_extract", [nil | _paths]), do: nil defp scalar("jsonb_extract", [v, path]) do case Json.get(json_parse!(v), json_path!(path)) do {:ok, {:array, _items} = found} -> jsonb_blob(found) {:ok, {:object, _pairs} = found} -> jsonb_blob(found) {:ok, found} -> Json.to_sql(found) :missing -> nil end end defp scalar("jsonb_extract", [v | paths]) do jv = json_parse!(v) items = Enum.map(paths, fn path -> case Json.get(jv, json_path!(path)) do {:ok, found} -> found :missing -> :null end end) jsonb_blob({:array, items}) end defp scalar("json_type", [nil]), do: nil defp scalar("json_type", [v]), do: v |> json_parse!() |> Json.type_name() defp scalar("json_type", [nil, _path]), do: nil defp scalar("json_type", [v, path]) do case Json.get(json_parse!(v), json_path!(path)) do {:ok, found} -> Json.type_name(found) :missing -> nil end end defp scalar("json_array_length", [v]), do: scalar("json_array_length", [v, "$"]) defp scalar("json_array_length", [nil, _path]), do: nil defp scalar("json_array_length", [v, path]) do case Json.get(json_parse!(v), json_path!(path)) do {:ok, {:array, items}} -> length(items) {:ok, _other} -> 0 :missing -> nil end end defp scalar("json_insert", [v | pairs]), do: json_write_pairs(v, pairs, :insert) defp scalar("json_replace", [v | pairs]), do: json_write_pairs(v, pairs, :replace) defp scalar("json_set", [v | pairs]), do: json_write_pairs(v, pairs, :set) defp scalar("jsonb_insert", [v | pairs]), do: jsonb_write_pairs(v, pairs, :insert) defp scalar("jsonb_replace", [v | pairs]), do: jsonb_write_pairs(v, pairs, :replace) defp scalar("jsonb_set", [v | pairs]), do: jsonb_write_pairs(v, pairs, :set) defp scalar("json_patch", [target, patch]) do if target == nil or patch == nil do nil else json_subtype(Json.merge_patch(json_parse!(target), json_parse!(patch))) end end defp scalar("jsonb_patch", [target, patch]) do if target == nil or patch == nil do nil else jsonb_blob(Json.merge_patch(json_parse!(target), json_parse!(patch))) end end defp scalar("json_pretty", [v]), do: scalar("json_pretty", [v, nil]) defp scalar("json_pretty", [nil, _indent]), do: nil defp scalar("json_pretty", [v, indent]) do indent = if is_nil(indent), do: " ", else: Value.to_text(indent) v |> json_parse!() |> Json.pretty(indent) end defp scalar("json_remove", [nil | _paths]), do: nil defp scalar("json_remove", [v | paths]) do paths |> Enum.reduce(json_parse!(v), fn path, jv -> Json.remove(jv, json_path!(path)) end) |> json_subtype() end defp scalar("jsonb_remove", [nil | _paths]), do: nil defp scalar("jsonb_remove", [v | paths]) do paths |> Enum.reduce(json_parse!(v), fn path, jv -> Json.remove(jv, json_path!(path)) end) |> jsonb_blob() end defp scalar("abs", [nil]), do: nil defp scalar("abs", [-9_223_372_036_854_775_808]), do: fail("integer overflow") defp scalar("abs", [v]) when is_integer(v) or is_float(v), do: abs(v) defp scalar("abs", [v]), do: scalar("abs", [Value.apply_affinity(v, :numeric)]) defp scalar("lower", [nil]), do: nil defp scalar("lower", [v]), do: v |> Value.to_text() |> ascii_case(?A..?Z, 32) defp scalar("upper", [nil]), do: nil defp scalar("upper", [v]), do: v |> Value.to_text() |> ascii_case(?a..?z, -32) defp scalar("length", [nil]), do: nil defp scalar("length", [{:blob, b}]), do: byte_size(b) defp scalar("length", [v]), do: v |> Value.to_text() |> String.length() defp scalar("typeof", [v]), do: v |> Value.type_of() |> Atom.to_string() defp scalar("coalesce", args) when length(args) >= 2, do: Enum.find(args, &(not is_nil(&1))) defp scalar("ifnull", [a, b]), do: if(is_nil(a), do: b, else: a) defp scalar("nullif", [a, b]) do if Value.compare_op(:eq, a, b) == true, do: nil, else: a end defp scalar("round", [v]), do: scalar("round", [v, 0]) defp scalar("round", [nil, _]), do: nil defp scalar("round", [v, digits]) when is_integer(digits) do case Value.apply_affinity(v, :real) do f when is_float(f) -> Float.round(f, max(digits, 0)) _ -> 0.0 end end defp scalar("pi", []), do: :math.pi() defp scalar("ceil", [v]), do: math_rounding(v, &Float.ceil/1) defp scalar("ceiling", [v]), do: scalar("ceil", [v]) defp scalar("floor", [v]), do: math_rounding(v, &Float.floor/1) defp scalar("trunc", [v]), do: math_rounding(v, &trunc/1) defp scalar("mod", [a, b]), do: math_binary(a, b, &math_mod/2) defp scalar("pow", [a, b]), do: math_binary(a, b, &:math.pow/2) defp scalar("power", args), do: scalar("pow", args) defp scalar("sqrt", [v]), do: math_unary(v, fn x -> if x < 0.0, do: nil, else: :math.sqrt(x) end) defp scalar("exp", [v]), do: math_unary(v, &:math.exp/1) defp scalar("ln", [v]), do: math_unary(v, fn x -> positive_math(x, &:math.log/1) end) defp scalar("log10", [v]), do: math_unary(v, fn x -> positive_math(x, &:math.log10/1) end) defp scalar("log2", [v]), do: math_unary(v, fn x -> positive_math(x, fn x -> :math.log(x) / :math.log(2) end) end) defp scalar("log", [v]), do: scalar("log10", [v]) defp scalar("log", [base, v]) do math_binary(base, v, fn base, x -> if base <= 0.0 or base == 1.0 or x <= 0.0 do nil else :math.log(x) / :math.log(base) end end) end defp scalar("radians", [v]), do: math_unary(v, &(&1 * :math.pi() / 180.0)) defp scalar("degrees", [v]), do: math_unary(v, &(&1 * 180.0 / :math.pi())) defp scalar("sin", [v]), do: math_unary(v, &:math.sin/1) defp scalar("cos", [v]), do: math_unary(v, &:math.cos/1) defp scalar("tan", [v]), do: math_unary(v, &:math.tan/1) defp scalar("asin", [v]), do: math_unary(v, fn x -> range_math(x, -1.0, 1.0, &:math.asin/1) end) defp scalar("acos", [v]), do: math_unary(v, fn x -> range_math(x, -1.0, 1.0, &:math.acos/1) end) defp scalar("atan", [v]), do: math_unary(v, &:math.atan/1) defp scalar("atan2", [a, b]), do: math_binary(a, b, &:math.atan2/2) defp scalar("sinh", [v]), do: math_unary(v, &:math.sinh/1) defp scalar("cosh", [v]), do: math_unary(v, &:math.cosh/1) defp scalar("tanh", [v]), do: math_unary(v, &:math.tanh/1) defp scalar("asinh", [v]), do: math_unary(v, fn x -> :math.log(x + :math.sqrt(x * x + 1.0)) end) defp scalar("acosh", [v]), do: math_unary(v, fn x -> if x < 1.0, do: nil, else: :math.log(x + :math.sqrt(x * x - 1.0)) end) defp scalar("atanh", [v]), do: math_unary(v, fn x -> if x <= -1.0 or x >= 1.0, do: nil, else: 0.5 * :math.log((1.0 + x) / (1.0 - x)) end) defp scalar("substr", [v, start]), do: scalar("substr", [v, start, nil]) defp scalar("substr", [nil, _, _]), do: nil defp scalar("substr", [v, start, len]) when is_integer(start) do text = Value.to_text(v) size = String.length(text) explicit_len = if is_integer(len) and len >= 0, do: len, else: if(is_integer(len), do: 0) # SQLite is 1-based. A negative start counts back from the end; a start at or # before position 1 leaves "empty" leading positions that still consume the # requested length (`substr('hello',0,2)` is `'h'`, not `'he'`). {from, count} = cond do start > 0 -> {start - 1, explicit_len || size} start == 0 -> {0, (explicit_len && max(explicit_len - 1, 0)) || size} true -> p1 = size + start cond do p1 >= 0 -> {p1, explicit_len || size} is_integer(len) -> {0, max((explicit_len || 0) + p1, 0)} true -> {0, size} end end String.slice(text, from, count) end defp scalar("replace", [a, b, c]) do if is_nil(a) or is_nil(b) or is_nil(c) do nil else text = Value.to_text(a) pattern = Value.to_text(b) # SQLite leaves the string unchanged for an empty pattern. if pattern == "", do: text, else: String.replace(text, pattern, Value.to_text(c)) end end defp scalar(name, [v]) when name in ["trim", "ltrim", "rtrim"], do: scalar(name, [v, " "]) defp scalar(name, [v, chars]) when name in ["trim", "ltrim", "rtrim"] do if is_nil(v) or is_nil(chars) do nil else text = Value.to_text(v) set = chars |> Value.to_text() |> String.graphemes() |> MapSet.new() case name do "trim" -> text |> trim_chars(set) |> reverse_text() |> trim_chars(set) |> reverse_text() "ltrim" -> trim_chars(text, set) "rtrim" -> text |> reverse_text() |> trim_chars(set) |> reverse_text() end end end defp scalar("instr", [a, b]) do if is_nil(a) or is_nil(b) do nil else haystack = Value.to_text(a) needle = Value.to_text(b) cond do needle == "" -> 1 true -> case String.split(haystack, needle, parts: 2) do [prefix, _] -> String.length(prefix) + 1 [_] -> 0 end end end end defp scalar("hex", [nil]), do: "" defp scalar("hex", [{:blob, b}]), do: Base.encode16(b) defp scalar("hex", [v]), do: v |> Value.to_text() |> Base.encode16() defp scalar("unhex", [v]), do: scalar("unhex", [v, ""]) defp scalar("unhex", [nil, _ignore]), do: nil defp scalar("unhex", [_v, nil]), do: nil defp scalar("unhex", [v, ignore]) do ignored = ignore |> Value.to_text() |> String.graphemes() |> MapSet.new() hex = v |> Value.to_text() |> String.graphemes() |> Enum.reject(&MapSet.member?(ignored, &1)) |> Enum.join() with 0 <- rem(String.length(hex), 2), {:ok, bytes} <- Base.decode16(hex, case: :mixed) do {:blob, bytes} else _ -> nil end end defp scalar("quote", [nil]), do: "NULL" defp scalar("quote", [{:blob, b}]), do: "X'" <> Base.encode16(b) <> "'" defp scalar("quote", [v]) when is_binary(v), do: "'" <> String.replace(v, "'", "''") <> "'" defp scalar("quote", [v]), do: Value.to_text(v) defp scalar(name, [format | args]) when name in ["printf", "format"] do sqlite_format(format, args) end defp scalar("random", []) do <> = :crypto.strong_rand_bytes(8) value end defp scalar("randomblob", [n]) do size = case Value.cast(n, :integer) do n when is_integer(n) and n > 0 -> n _ -> 1 end {:blob, :crypto.strong_rand_bytes(size)} end defp scalar("sqlite_version", []), do: @sqlite_version defp scalar("char", args) do Enum.map_join(args, fn codepoint when is_integer(codepoint) and codepoint > 0 -> <> _ -> "" end) end defp scalar("unicode", [nil]), do: nil defp scalar("unicode", [v]) do case Value.to_text(v) do <> -> codepoint _ -> nil end end defp scalar("sign", [v]) do case Value.apply_affinity(v, :numeric) do n when is_integer(n) or is_float(n) -> cond do n > 0 -> 1 n < 0 -> -1 true -> 0 end _ -> nil end end defp scalar("iif", [a, b, c]), do: if(Value.truthy(a) == true, do: b, else: c) defp scalar("zeroblob", [n]) when is_integer(n), do: {:blob, :binary.copy(<<0>>, max(n, 0))} defp scalar("octet_length", [nil]), do: nil defp scalar("octet_length", [{:blob, b}]), do: byte_size(b) defp scalar("octet_length", [v]), do: v |> Value.to_text() |> byte_size() defp scalar("concat", args) when args != [], do: args |> Enum.reject(&is_nil/1) |> Enum.map_join(&Value.to_text/1) defp scalar("concat_ws", [nil | _rest]), do: nil defp scalar("concat_ws", [separator | rest]) when rest != [] do rest |> Enum.reject(&is_nil/1) |> Enum.map_join(Value.to_text(separator), &Value.to_text/1) end defp scalar("glob", [pattern, v]), do: bool_value(Value.glob(v, pattern)) defp scalar("regexp", [pattern, v]), do: bool_value(regexp_match(v, pattern)) defp scalar("match", [_pattern, _v]) do fail("unable to use function MATCH in the requested context") end defp scalar("substring", args), do: scalar("substr", args) # Multi-argument min/max are scalar functions; one-argument are aggregates. defp scalar(name, args) when name in ["min", "max"] and length(args) >= 2 do if Enum.any?(args, &is_nil/1) do nil else comparator = case name do "min" -> fn a, b -> Value.compare(a, b) != :gt end "max" -> fn a, b -> Value.compare(a, b) != :lt end end Enum.reduce(args, fn x, best -> if comparator.(x, best), do: x, else: best end) end end # Date/time functions delegate to ExSQL.DateTime (mirrors date.c). defp scalar("date", args), do: DateTime.date(args) defp scalar("time", args), do: DateTime.time(args) defp scalar("timediff", args), do: DateTime.timediff(args) defp scalar("datetime", args), do: DateTime.datetime(args) defp scalar("julianday", args), do: DateTime.julianday(args) defp scalar("unixepoch", args), do: DateTime.unixepoch(args) defp scalar("strftime", args), do: DateTime.strftime(args) defp scalar(name, _args) do if Map.has_key?(@scalar_arity, name) or name in @aggregate_functions do fail("wrong number of arguments to function #{name}()") else fail("no such function: #{name}") end end # -- JSON helpers --------------------------------------------------------------- defp json_subtype(jv), do: {:json, Json.render(jv)} defp jsonb_blob(jv), do: {:blob, Json.to_sqlite_jsonb(jv)} defp json_aggregate_result("json_group_array", jv), do: json_subtype(jv) defp json_aggregate_result("json_group_object", jv), do: json_subtype(jv) defp json_aggregate_result("jsonb_group_array", jv), do: jsonb_blob(jv) defp json_aggregate_result("jsonb_group_object", jv), do: jsonb_blob(jv) defp json_to_sql({:array, _items} = jv), do: json_subtype(jv) defp json_to_sql({:object, _pairs} = jv), do: json_subtype(jv) defp json_to_sql(jv), do: Json.to_sql(jv) defp json_parse!({:blob, @jsonb_magic <> text}), do: json_parse!(text) defp json_parse!({:blob, blob}) do case Json.parse_sqlite_jsonb(blob) do {:ok, jv} -> jv :error -> fail("malformed JSON") end end defp json_parse!({:json, text}), do: json_parse!(text) defp json_parse!(v) do case Json.parse_json5(Value.to_text(v)) do {:ok, jv} -> jv :error -> fail("malformed JSON") end end defp json_valid_flags!(flags) do flags = Value.cast(flags, :integer) if is_integer(flags) and flags in 1..15 do flags else fail("FLAGS parameter to json_valid() must be between 1 and 15") end end defp json_path!(path) when is_binary(path) do case Json.parse_path(path) do {:ok, steps} -> steps :error -> fail("bad JSON path: '#{path}'") end end defp json_path!(path), do: fail("bad JSON path: '#{Value.to_text(path)}'") # An SQL value used as a JSON ingredient (json_array, json_set values, ...). defp json_from_sql!(nil), do: :null defp json_from_sql!(n) when is_number(n), do: n defp json_from_sql!({:json, text}), do: json_parse!(text) defp json_from_sql!({:blob, @jsonb_magic <> text}), do: json_parse!(text) defp json_from_sql!({:blob, blob}) do case Json.parse_sqlite_jsonb(blob) do {:ok, jv} -> jv :error -> fail("JSON cannot hold BLOB values") end end defp json_from_sql!(s) when is_binary(s), do: s defp jsonb_valid({:blob, @jsonb_magic <> _text}, flags), do: bool_value(Bitwise.band(flags, 0b1100) != 0) defp jsonb_valid({:blob, blob}, flags) do strict? = Bitwise.band(flags, 0b1000) != 0 and Json.sqlite_jsonb_strict?(blob) superficial? = Bitwise.band(flags, 0b0100) != 0 and Json.sqlite_jsonb_superficial?(blob) bool_value(strict? or superficial?) end defp json_object(args) do if rem(length(args), 2) != 0 do fail("json_object() requires an even number of arguments") end pairs = args |> Enum.chunk_every(2) |> Enum.map(fn [key, value] -> unless is_binary(key), do: fail("json_object() labels must be TEXT") {key, json_from_sql!(value)} end) {:object, pairs} end defp json_write_pairs(nil, _pairs, _mode), do: nil defp json_write_pairs(v, pairs, mode) do if rem(length(pairs), 2) != 0 do fail("json_#{mode}() needs an odd number of arguments") end pairs |> Enum.chunk_every(2) |> Enum.reduce(json_parse!(v), fn [path, value], jv -> Json.write(jv, json_path!(path), json_from_sql!(value), mode) end) |> json_subtype() end defp jsonb_write_pairs(nil, _pairs, _mode), do: nil defp jsonb_write_pairs(v, pairs, mode) do if rem(length(pairs), 2) != 0 do fail("jsonb_#{mode}() needs an odd number of arguments") end pairs |> Enum.chunk_every(2) |> Enum.reduce(json_parse!(v), fn [path, value], jv -> Json.write(jv, json_path!(path), json_from_sql!(value), mode) end) |> jsonb_blob() end defp json_arrow_value(json, path, _render) when is_nil(json) or is_nil(path), do: nil defp json_arrow_value(json, path, render) do case Json.get(json_parse!(json), json_arrow_path!(path)) do {:ok, found} -> render.(found) :missing -> nil end end defp json_arrow_path!(path) do cond do is_integer(path) -> [{:index, path}] is_binary(path) and String.starts_with?(path, "$") -> json_path!(path) is_binary(path) -> [{:key, path}] true -> fail("bad JSON path: '#{Value.to_text(path)}'") end end defp math_unary(value, fun) do case math_number(value) do nil -> nil number -> safe_math(fn -> fun.(number * 1.0) end) end end defp math_binary(a, b, fun) do with a when is_number(a) <- math_number(a), b when is_number(b) <- math_number(b) do safe_math(fn -> fun.(a * 1.0, b * 1.0) end) else _ -> nil end end defp math_rounding(value, fun) do case math_number(value) do nil -> nil number when is_integer(number) -> number number -> safe_math(fn -> fun.(number * 1.0) end) end end defp math_number(nil), do: nil defp math_number(value) when is_integer(value) or is_float(value), do: value defp math_number(value) do case Value.apply_affinity(value, :numeric) do number when is_integer(number) or is_float(number) -> number _ -> nil end end defp positive_math(value, fun) do if value <= 0.0, do: nil, else: fun.(value) end defp range_math(value, min, max, fun) do if value < min or value > max, do: nil, else: fun.(value) end defp math_mod(_a, b) when b == 0.0, do: nil defp math_mod(a, b), do: a - trunc(a / b) * b defp regexp_match(nil, _pattern), do: nil defp regexp_match(_value, nil), do: nil defp regexp_match(value, pattern) do case Regex.compile(Value.to_text(pattern)) do {:ok, regex} -> Regex.match?(regex, Value.to_text(value)) {:error, _reason} -> nil end end defp safe_math(fun) do case fun.() do nil -> nil result when is_number(result) -> result _ -> nil end rescue ArithmeticError -> nil ErlangError -> nil end defp trim_chars(text, set) do text |> String.graphemes() |> Enum.drop_while(&MapSet.member?(set, &1)) |> Enum.join() end defp reverse_text(text), do: String.reverse(text) # SQLite's printf/format is its own formatter, not libc's. This covers the # SQL-visible core used by func.test: strings, SQL quoting, numeric bases, # floating output, width/precision, dynamic * width/precision, and %c repeats. defp sqlite_format(nil, _args), do: nil defp sqlite_format(format, args), do: format |> Value.to_text() |> format_chunks(args, []) defp format_chunks("", _args, acc), do: acc |> Enum.reverse() |> IO.iodata_to_binary() defp format_chunks(<<"%%", rest::binary>>, args, acc), do: format_chunks(rest, args, ["%" | acc]) defp format_chunks(<<"%", rest::binary>>, args, acc) do {spec, rest, args} = parse_format_spec(rest, args) {piece, args} = format_piece(spec, args) format_chunks(rest, args, [piece | acc]) end defp format_chunks(<>, args, acc), do: format_chunks(rest, args, [<> | acc]) defp parse_format_spec(text, args) do {flags, text} = take_format_flags(text, MapSet.new()) {width, flags, text, args} = take_format_width(text, flags, args) {precision, text, args} = take_format_precision(text, args) text = drop_format_length(text) case text do <> -> {%{flags: flags, width: width, precision: precision, conv: conv}, rest, args} "" -> {%{flags: flags, width: width, precision: precision, conv: ?%}, "", args} end end defp take_format_flags(<>, flags) when flag in [?-, ?+, ?\s, ?0, ?#, ?,, ?!] do take_format_flags(rest, MapSet.put(flags, flag)) end defp take_format_flags(text, flags), do: {flags, text} defp take_format_width(<<"*", rest::binary>>, flags, [arg | args]) do case format_int(arg) do width when width < 0 -> {abs(width), MapSet.put(flags, ?-), rest, args} width -> {width, flags, rest, args} end end defp take_format_width(<<"*", rest::binary>>, flags, []), do: {0, flags, rest, []} defp take_format_width(text, flags, args) do {digits, rest} = take_digits(text, "") width = if digits == "", do: nil, else: String.to_integer(digits) {width, flags, rest, args} end defp take_format_precision(<<".*", rest::binary>>, [arg | args]) do precision = format_int(arg) {if(precision < 0, do: nil, else: precision), rest, args} end defp take_format_precision(<<".*", rest::binary>>, []), do: {0, rest, []} defp take_format_precision(<<".", rest::binary>>, args) do {digits, rest} = take_digits(rest, "") {if(digits == "", do: 0, else: String.to_integer(digits)), rest, args} end defp take_format_precision(text, args), do: {nil, text, args} defp take_digits(<>, acc) when digit in ?0..?9, do: take_digits(rest, acc <> <>) defp take_digits(text, acc), do: {acc, text} defp drop_format_length(<<"ll", rest::binary>>), do: rest defp drop_format_length(<<"l", rest::binary>>), do: rest defp drop_format_length(text), do: text defp format_piece(%{conv: conv} = spec, args) when conv in [?s, ?z] do {arg, args} = next_format_arg(args) arg |> format_text() |> limit_format_text(spec) |> apply_format_width(spec) |> then(&{&1, args}) end defp format_piece(%{conv: ?q} = spec, args) do {arg, args} = next_format_arg(args) arg |> format_text() |> limit_format_text(spec) |> escape_sql_quote() |> apply_format_width(spec) |> then(&{&1, args}) end defp format_piece(%{conv: ?Q} = spec, args) do {arg, args} = next_format_arg(args) piece = if is_nil(arg) do "NULL" else "'" <> (arg |> format_text() |> limit_format_text(spec) |> escape_sql_quote()) <> "'" end {apply_format_width(piece, spec), args} end defp format_piece(%{conv: ?j} = spec, args) do {arg, args} = next_format_arg(args) piece = if is_nil(arg), do: "", else: arg |> format_text() |> json_format_escape(spec) {apply_format_width(piece, spec), args} end defp format_piece(%{conv: ?J} = spec, args) do {arg, args} = next_format_arg(args) piece = if is_nil(arg) do "null" else "\"" <> (arg |> format_text() |> json_format_escape(spec)) <> "\"" end {apply_format_width(piece, spec), args} end defp format_piece(%{conv: ?w} = spec, args) do {arg, args} = next_format_arg(args) piece = arg |> format_text() |> limit_format_text(spec) |> String.replace("\"", "\"\"") {apply_format_width(piece, spec), args} end defp format_piece(%{conv: conv} = spec, args) when conv in [?d, ?i, ?u, ?x, ?X, ?o] do {arg, args} = next_format_arg(args) value = format_int(arg) piece = case conv do ?d -> signed_integer_piece(value, 10, false, spec) ?i -> signed_integer_piece(value, 10, false, spec) ?u -> unsigned_integer_piece(value, 10, false, spec) ?x -> unsigned_integer_piece(value, 16, false, spec) ?X -> unsigned_integer_piece(value, 16, true, spec) ?o -> unsigned_integer_piece(value, 8, false, spec) end {piece, args} end defp format_piece(%{conv: conv} = spec, args) when conv in [?f, ?e, ?E, ?g, ?G] do {arg, args} = next_format_arg(args) precision = format_float_precision(spec) value = format_float(arg) piece = case conv do ?f -> :io_lib.format(format_control(".#{precision}f"), [value]) ?e -> value |> scientific_piece(precision) |> maybe_alternate_float(spec) |> String.downcase() ?E -> value |> scientific_piece(precision) |> maybe_alternate_float(spec) |> String.upcase() ?g -> general_piece(value, precision, spec, false) ?G -> general_piece(value, precision, spec, true) end |> IO.iodata_to_binary() {piece |> signed_float_piece(value, spec) |> apply_numeric_width(spec, true), args} end defp format_piece(%{conv: ?c} = spec, args) do {arg, args} = next_format_arg(args) char = case format_text(arg) do <> -> <> "" -> <<0>> end piece = String.duplicate(char, spec.precision || 1) {apply_format_width(piece, spec), args} end defp format_piece(%{conv: conv} = spec, args) do {arg, args} = next_format_arg(args) piece = "%" <> <> <> format_text(arg) {apply_format_width(piece, spec), args} end defp next_format_arg([arg | args]), do: {arg, args} defp next_format_arg([]), do: {nil, []} defp format_text(nil), do: "" defp format_text(value), do: Value.to_text(value) defp limit_format_text(text, %{precision: nil}), do: text defp limit_format_text(text, %{precision: precision, flags: flags}) do if MapSet.member?(flags, ?!) do limit_format_text_characters(text, precision) else limit_format_text_bytes(text, precision) end end defp limit_format_text_characters(text, precision) do text |> String.graphemes() |> Enum.take(max(precision, 0)) |> Enum.join() end defp limit_format_text_bytes(text, precision) do limit = min(byte_size(text), max(precision, 0)) valid_utf8_prefix(text, limit) end defp valid_utf8_prefix(_text, limit) when limit <= 0, do: "" defp valid_utf8_prefix(text, limit) do prefix = binary_part(text, 0, limit) if String.valid?(prefix) do prefix else valid_utf8_prefix(text, limit - 1) end end defp escape_sql_quote(text), do: String.replace(text, "'", "''") defp json_format_escape(text, spec) do text |> limit_format_text(spec) |> :unicode.characters_to_binary(:utf8, :utf8) |> json_escape_bytes([]) end defp json_escape_bytes(<<>>, acc), do: acc |> Enum.reverse() |> IO.iodata_to_binary() defp json_escape_bytes(<<"\"", rest::binary>>, acc), do: json_escape_bytes(rest, ["\\\"" | acc]) defp json_escape_bytes(<<"\\", rest::binary>>, acc), do: json_escape_bytes(rest, ["\\\\" | acc]) defp json_escape_bytes(<<"\b", rest::binary>>, acc), do: json_escape_bytes(rest, ["\\b" | acc]) defp json_escape_bytes(<<"\t", rest::binary>>, acc), do: json_escape_bytes(rest, ["\\t" | acc]) defp json_escape_bytes(<<"\n", rest::binary>>, acc), do: json_escape_bytes(rest, ["\\n" | acc]) defp json_escape_bytes(<<"\f", rest::binary>>, acc), do: json_escape_bytes(rest, ["\\f" | acc]) defp json_escape_bytes(<<"\r", rest::binary>>, acc), do: json_escape_bytes(rest, ["\\r" | acc]) defp json_escape_bytes(<>, acc) when byte <= 0x1F do escape = "\\u00" <> (byte |> Integer.to_string(16) |> String.downcase() |> String.pad_leading(2, "0")) json_escape_bytes(rest, [escape | acc]) end defp json_escape_bytes(<>, acc), do: json_escape_bytes(rest, [<> | acc]) # SQLite's upper()/lower() fold only ASCII letters; non-ASCII (and the bytes of # multi-byte UTF-8 sequences, all >= 0x80) pass through unchanged. defp ascii_case(text, range, delta) do for <>, into: <<>> do if byte in range, do: <>, else: <> end end defp format_int(nil), do: 0 defp format_int(value) do case Value.cast(value, :integer) do value when is_integer(value) -> value _ -> 0 end end defp format_float(nil), do: 0.0 defp format_float(value) do case Value.cast(value, :real) do value when is_integer(value) -> value * 1.0 value when is_float(value) -> value _ -> 0.0 end end defp format_float_precision(%{precision: nil}), do: 6 defp format_float_precision(%{precision: precision}), do: precision defp signed_integer_piece(value, base, uppercase?, spec) do sign = cond do value < 0 -> "-" MapSet.member?(spec.flags, ?+) -> "+" MapSet.member?(spec.flags, ?\s) -> " " true -> "" end digits = abs(value) |> Integer.to_string(base) |> maybe_upcase(uppercase?) digits = integer_precision(digits, spec.precision) digits = maybe_group_integer(digits, base, spec) apply_numeric_width(sign <> digits, spec) end defp unsigned_integer_piece(value, base, uppercase?, spec) do value = if value < 0, do: value + 18_446_744_073_709_551_616, else: value digits = value |> Integer.to_string(base) |> maybe_upcase(uppercase?) prefix = cond do not MapSet.member?(spec.flags, ?#) -> "" base == 16 and uppercase? -> "0X" base == 16 -> "0x" base == 8 -> "0" true -> "" end digits = integer_precision(digits, spec.precision) digits = maybe_group_integer(digits, base, spec) apply_numeric_width(prefix <> digits, spec) end defp integer_precision(digits, nil), do: digits defp integer_precision(digits, precision) do String.duplicate("0", max(precision - String.length(digits), 0)) <> digits end defp maybe_upcase(text, true), do: String.upcase(text) defp maybe_upcase(text, false), do: String.downcase(text) defp signed_float_piece(piece, value, spec) when value >= 0 do piece = cond do MapSet.member?(spec.flags, ?+) -> "+" <> piece MapSet.member?(spec.flags, ?\s) -> " " <> piece true -> piece end maybe_group_decimal(piece, spec) end defp signed_float_piece(piece, _value, spec), do: maybe_group_decimal(piece, spec) defp maybe_group_integer(digits, 10, spec) do if MapSet.member?(spec.flags, ?,), do: group_digits(digits), else: digits end defp maybe_group_integer(digits, _base, _spec), do: digits defp maybe_group_decimal(piece, spec) do if MapSet.member?(spec.flags, ?,) and spec.conv == ?f do {prefix, rest} = numeric_prefix(piece) [whole | tail] = String.split(rest, ".", parts: 2) prefix <> group_digits(whole) <> if(tail == [], do: "", else: "." <> hd(tail)) else piece end end defp group_digits(digits) do digits |> String.reverse() |> String.graphemes() |> Enum.chunk_every(3) |> Enum.map_join(",", &Enum.join/1) |> String.reverse() end defp general_piece(value, precision, spec, uppercase?) do safe_precision = if precision == 0, do: 1, else: precision piece = cond do MapSet.member?(spec.flags, ?#) -> hash_general_piece(value, precision, safe_precision) MapSet.member?(spec.flags, ?!) -> alternate_general_piece(value, max(precision, 1)) true -> format_control(".#{safe_precision}g") |> :io_lib.format([value]) |> IO.iodata_to_binary() |> normalize_general_piece(precision) end if uppercase?, do: String.upcase(piece), else: piece end defp hash_general_piece(value, precision, safe_precision) do format_control(".#{safe_precision}g") |> :io_lib.format([value]) |> IO.iodata_to_binary() |> normalize_general_piece(precision) |> apply_hash_general_precision(precision) end defp apply_hash_general_precision(text, precision) do if String.contains?(text, ["e", "E"]) do apply_hash_scientific_precision(text, precision) else apply_hash_fixed_precision(text, precision) end end defp apply_hash_scientific_precision(text, precision) do Regex.replace(~r/^(.+?)([eE][+-]\d+)$/, text, fn _match, mantissa, exponent -> cond do precision == 0 and String.contains?(mantissa, ".") -> [head, _ | _] = String.split(mantissa, ".", parts: 2) head <> "." <> exponent precision == 0 -> mantissa <> "." <> exponent String.contains?(mantissa, ".") -> mantissa <> exponent true -> mantissa <> ".0" <> exponent end end) end defp apply_hash_fixed_precision(text, precision) do {sign, rest} = case text do <<"-", value::binary>> -> {"-", value} <<"+", value::binary>> -> {"+", value} _ -> {"", text} end {int_part, frac_part} = case String.split(rest, ".", parts: 2) do [integer, fraction] -> {integer, fraction} [integer] -> {integer, ""} end significant = hash_significant_digits(int_part, frac_part) needed = max(precision - significant, 0) padded_frac = frac_part <> String.duplicate("0", needed) sign <> int_part <> "." <> padded_frac end defp hash_significant_digits(int_part, frac_part) do if int_part != "0" do String.length(int_part <> frac_part) else sig = String.trim_leading(frac_part, "0") if sig == "", do: 1, else: String.length(sig) end end defp scientific_piece(value, 0) do format_control(".2e") |> :io_lib.format([value]) |> IO.iodata_to_binary() |> String.replace(~r/\.0(e[+-]\d+)$/i, "\\1") |> pad_scientific_exponent() end defp scientific_piece(value, precision) do format_control(".#{precision + 1}e") |> :io_lib.format([value]) |> IO.iodata_to_binary() |> pad_scientific_exponent() end defp maybe_alternate_float(piece, %{flags: flags, precision: precision}) do cond do MapSet.member?(flags, ?!) -> if precision == 0 do force_scientific_exclamation(piece) else force_decimal_point(piece) end MapSet.member?(flags, ?#) -> force_scientific_hash(piece) true -> piece end end defp force_scientific_hash(piece) do case Regex.run(~r/^(.+?)([eE][+-]\d+)$/, piece) do [_, mantissa, exponent] -> if String.contains?(mantissa, ".") do [head, _ | _] = String.split(mantissa, ".", parts: 2) head <> "." <> exponent else mantissa <> "." <> exponent end nil -> piece end end defp force_scientific_exclamation(piece) do case Regex.run(~r/^(.+?)([eE][+-]\d+)$/, piece) do [_, mantissa, exponent] -> if String.contains?(mantissa, ".") do [head, _ | _] = String.split(mantissa, ".", parts: 2) head <> ".0" <> exponent else mantissa <> ".0" <> exponent end nil -> piece end end defp alternate_general_piece(value, precision) do value |> Float.to_string() |> normalize_general_piece(precision) |> force_decimal_point() end defp normalize_general_piece(piece, precision) do case Regex.run(~r/^([+-]?)(\d+(?:\.\d+)?)[eE]([+-]?\d+)$/, piece) do [_, sign, significand, exponent_text] -> exponent = String.to_integer(exponent_text) if exponent >= -4 and exponent < precision do sign <> expand_scientific(significand, exponent) else sign <> trim_general_mantissa(significand) <> "e" <> signed_exponent(exponent) end nil -> piece end end defp expand_scientific(significand, exponent) do {whole, fractional} = case String.split(significand, ".", parts: 2) do [whole, fractional] -> {whole, fractional} [whole] -> {whole, ""} end digits = whole <> fractional decimal_position = String.length(whole) + exponent cond do decimal_position <= 0 -> "0." <> String.duplicate("0", -decimal_position) <> digits decimal_position >= String.length(digits) -> digits <> String.duplicate("0", decimal_position - String.length(digits)) true -> {left, right} = String.split_at(digits, decimal_position) left <> "." <> right end |> trim_general_mantissa() end defp trim_general_mantissa(text) do if String.contains?(text, ".") do text |> String.trim_trailing("0") |> String.trim_trailing(".") else text end end defp force_decimal_point(piece) do case Regex.run(~r/^(.+?)([eE][+-]\d+)$/, piece) do [_, mantissa, exponent] -> force_decimal_point(mantissa) <> exponent nil -> if String.contains?(piece, "."), do: piece, else: piece <> ".0" end end defp signed_exponent(exponent) do sign = if exponent < 0, do: "-", else: "+" digits = exponent |> abs() |> Integer.to_string() |> String.pad_leading(2, "0") sign <> digits end defp pad_scientific_exponent(text) do Regex.replace(~r/e([+-])(\d)$/, text, fn _match, sign, digit -> "e#{sign}0#{digit}" end) end defp format_control(options), do: ("~" <> options) |> String.to_charlist() # The `0` flag zero-pads to the field width. For integer conversions an # explicit precision suppresses it (C printf: precision sets the minimum digit # count); for float conversions (`zero_with_precision?` = true) precision is # the fraction width, so the flag still applies — e.g. `%05.2f` of 3.14 is # `03.14`, not ` 3.14`. defp apply_numeric_width(piece, spec, zero_with_precision? \\ false) do if MapSet.member?(spec.flags, ?0) and not MapSet.member?(spec.flags, ?-) and is_integer(spec.width) and (spec.precision == nil or zero_with_precision?) do pad_numeric_zero(piece, spec.width) else apply_format_width(piece, spec) end end defp pad_numeric_zero(piece, width) do size = String.length(piece) if size >= width do piece else {head, rest} = numeric_prefix(piece) head <> String.duplicate("0", width - size) <> rest end end defp numeric_prefix("-" <> rest), do: {"-", rest} defp numeric_prefix("+" <> rest), do: {"+", rest} defp numeric_prefix(" " <> rest), do: {" ", rest} defp numeric_prefix("0x" <> rest), do: {"0x", rest} defp numeric_prefix("0X" <> rest), do: {"0X", rest} defp numeric_prefix(rest), do: {"", rest} defp apply_format_width(piece, %{width: width, flags: flags}) when is_integer(width) do size = if MapSet.member?(flags, ?!) do String.length(piece) else byte_size(piece) end if size >= width do piece else padding = String.duplicate(" ", width - size) if MapSet.member?(flags, ?-) do piece <> padding else padding <> piece end end end defp apply_format_width(piece, _spec), do: piece # -- aggregate functions -------------------------------------------------------------- defp aggregate_call?(_db, name, :star), do: name == "count" defp aggregate_call?(db, name, {:distinct, args}), do: aggregate_call?(db, name, args) defp aggregate_call?(db, name, args) do is_list(args) and (match?({:ok, _}, Database.fetch_aggregate_function(db, name, length(args))) or (name in @aggregate_functions and not (name in ["min", "max"] and length(args) != 1))) end defp contains_aggregate?({:window, _name, _args, _spec, _filter}, _db), do: false defp contains_aggregate?({:filter_function, name, args, filter}, db) do aggregate_call?(db, name, args) or contains_aggregate?(filter, db) or (is_list(args) and Enum.any?(args, &contains_aggregate?(&1, db))) end defp contains_aggregate?({:function, name, {:distinct, args}}, db) do aggregate_call?(db, name, {:distinct, args}) or Enum.any?(args, &contains_aggregate?(&1, db)) end defp contains_aggregate?({:function, name, args}, db) do aggregate_call?(db, name, args) or (is_list(args) and Enum.any?(args, &contains_aggregate?(&1, db))) end defp contains_aggregate?(expr, db) when is_tuple(expr) do expr |> Tuple.to_list() |> Enum.any?(fn # Subquery ASTs are structs (maps), so this walk never descends into # them — an aggregate inside a subquery belongs to the subquery. sub when is_tuple(sub) -> contains_aggregate?(sub, db) subs when is_list(subs) -> Enum.any?(subs, &(is_tuple(&1) and contains_aggregate?(&1, db))) _ -> false end) end defp contains_aggregate?(_expr, _db), do: false defp aggregate("count", :star, group, _env), do: length(group) defp aggregate(name, {:distinct, args}, group, env) do if length(args) != 1 do fail("DISTINCT aggregates must have exactly one argument") end [arg] = args distinct_group = group |> Enum.map(fn member -> {eval(arg, member), member} end) |> Enum.reject(fn {value, _member} -> is_nil(value) end) |> Enum.uniq_by(fn {value, _member} -> value end) |> Enum.map(fn {_value, member} -> member end) aggregate(name, args, distinct_group, env) end defp aggregate(name, args, group, env) when is_list(args) do case Database.fetch_aggregate_function(env.db, name, length(args)) do {:ok, %{kind: :incremental_window} = function} -> call_incremental_window_aggregate(function, aggregate_rows_with_nulls(args, group)) {:ok, function} -> call_aggregate_function(function, aggregate_rows(args, group)) :error -> built_in_aggregate(name, args, group, env) end end defp aggregate_rows(args, group) do group |> Enum.map(fn member -> Enum.map(args, &eval(&1, member)) end) |> Enum.reject(&Enum.any?(&1, fn value -> is_nil(value) end)) end defp aggregate_rows_with_nulls(args, group) do Enum.map(group, fn member -> Enum.map(args, &eval(&1, member)) end) end defp call_aggregate_function(%{name: name, callback: callback}, rows) do callback |> apply([rows]) |> normalize_aggregate_function_result(name) rescue e in Error -> reraise e, __STACKTRACE__ e -> fail("user-defined aggregate #{name}() raised: #{Exception.message(e)}") end defp normalize_aggregate_function_result({:ok, value}, name), do: normalize_aggregate_function_result(value, name) defp normalize_aggregate_function_result({:error, message}, name), do: fail("user-defined aggregate #{name}() error: #{message}") defp normalize_aggregate_function_result(nil, _name), do: nil defp normalize_aggregate_function_result(value, _name) when is_integer(value), do: value defp normalize_aggregate_function_result(value, _name) when is_float(value), do: value defp normalize_aggregate_function_result(value, _name) when is_binary(value), do: value defp normalize_aggregate_function_result({:blob, value}, _name) when is_binary(value), do: {:blob, value} defp normalize_aggregate_function_result(_value, name), do: fail("user-defined aggregate #{name}() returned unsupported value") defp call_incremental_window_aggregate(function, rows) do state = call_incremental_window_init(function) state = Enum.reduce(rows, state, fn args, state -> call_incremental_window_update(function, :step, state, args) end) call_incremental_window_final(function, state) end defp call_incremental_window_init(%{name: name, callback: %{init: init}}) do init |> apply([]) |> normalize_incremental_window_state(name) rescue e in Error -> reraise e, __STACKTRACE__ e -> fail("user-defined window function #{name}() raised: #{Exception.message(e)}") end defp call_incremental_window_update( %{name: name, callback: callbacks}, callback_name, state, args ) do callbacks |> Map.fetch!(callback_name) |> apply([state, args]) |> normalize_incremental_window_state(name) rescue e in Error -> reraise e, __STACKTRACE__ e -> fail("user-defined window function #{name}() raised: #{Exception.message(e)}") end defp call_incremental_window_value(%{name: name, callback: %{value: value}}, state) do value |> apply([state]) |> normalize_aggregate_function_result(name) rescue e in Error -> reraise e, __STACKTRACE__ e -> fail("user-defined window function #{name}() raised: #{Exception.message(e)}") end defp call_incremental_window_final(%{name: name, callback: %{final: final}}, state) do final |> apply([state]) |> normalize_aggregate_function_result(name) rescue e in Error -> reraise e, __STACKTRACE__ e -> fail("user-defined window function #{name}() raised: #{Exception.message(e)}") end defp normalize_incremental_window_state({:ok, state}, name), do: normalize_incremental_window_state(state, name) defp normalize_incremental_window_state({:error, message}, name), do: fail("user-defined window function #{name}() error: #{message}") defp normalize_incremental_window_state(state, _name), do: state defp built_in_aggregate("count", [], group, _env), do: length(group) defp built_in_aggregate(name, [arg | rest], group, env) do values = for member <- group, value = eval(arg, member), not is_nil(value), do: value case name do "count" -> length(values) "sum" -> if values == [], do: nil, else: numeric_sum(values) "total" -> 1.0 * (values |> Enum.map(&to_num/1) |> Enum.sum()) "avg" -> if values == [], do: nil, else: (values |> Enum.map(&to_num/1) |> Enum.sum()) / length(values) "min" -> Enum.min(values, fn a, b -> Value.compare(a, b) != :gt end, fn -> nil end) "max" -> Enum.max(values, fn a, b -> Value.compare(a, b) != :lt end, fn -> nil end) name when name in ["group_concat", "string_agg"] -> separator = case rest do [sep_expr] -> case eval(sep_expr, env) do nil -> "," value -> Value.to_text(value) end [] -> "," end if values == [], do: nil, else: Enum.map_join(values, separator, &Value.to_text/1) # JSON aggregates keep NULL members, unlike the filtered `values`. name when name in ["json_group_array", "jsonb_group_array"] -> members = for member <- group, do: member |> then(&eval(arg, &1)) |> json_from_sql!() json_aggregate_result(name, {:array, members}) name when name in ["json_group_object", "jsonb_group_object"] -> case rest do [value_expr] -> pairs = for member <- group do key = eval(arg, member) {Value.to_text(key || ""), json_from_sql!(eval(value_expr, member))} end json_aggregate_result(name, {:object, pairs}) _ -> fail("wrong number of arguments to function #{name}()") end end end defp built_in_aggregate(name, _args, _group, _env), do: fail("wrong number of arguments to function #{name}()") defp numeric_sum(values) do nums = Enum.map(values, &to_num/1) if Enum.all?(nums, &is_integer/1) do sum = Enum.sum(nums) # sum() over integers errors on 64-bit overflow rather than going REAL. if Value.out_of_int64_range?(sum), do: fail("integer overflow") sum else nums |> Enum.map(&(&1 * 1.0)) |> Enum.sum() end end # Numeric coercion for aggregation. Unlike NUMERIC affinity this must not # demote integral floats: sum() over a REAL column stays REAL (ticket #2251). defp to_num(v) when is_integer(v) or is_float(v), do: v defp to_num(v) do case Value.apply_affinity(v, :numeric) do n when is_integer(n) or is_float(n) -> n _ -> 0 end end # -- naming ------------------------------------------------------------------------- # Output column name for an expression without an alias. SQLite uses the # original SQL text; we render an equivalent form. defp result_column_name(_db, _templates, {_expr, alias_name}) when is_binary(alias_name), do: alias_name defp result_column_name( %{full_column_names: true}, templates, {{:column, qualifier, name}, nil} ) do case result_column_source(templates, qualifier, name) do nil -> expr_name({:column, qualifier, name}) source -> "#{source}.#{name}" end end defp result_column_name( %{short_column_names: true}, _templates, {{:column, _qualifier, name}, nil} ), do: name defp result_column_name(_db, _templates, {expr, nil}), do: expr_name(expr) defp result_column_source(templates, nil, name) do key = Table.key(name) case Enum.filter(templates, &visible?(&1, key)) do [frame] -> frame.source_name || frame.name _ -> nil end end defp result_column_source(templates, qualifier, name) do qkey = Table.key(qualifier) key = Table.key(name) case Enum.find(templates, &(&1.name == qkey)) do %{source_name: source_name} = frame when source_name != nil -> if has_column?(frame, key), do: source_name, else: nil frame when is_map(frame) -> if has_column?(frame, key), do: frame.name, else: nil nil -> nil end end defp expr_name({:param, _index, raw}), do: raw defp expr_name({:column, nil, name}), do: name defp expr_name({:column, table, name}), do: "#{table}.#{name}" defp expr_name({:literal, nil}), do: "NULL" defp expr_name({:literal, {:blob, b}}), do: "x'#{Base.encode16(b)}'" defp expr_name({:literal, v}) when is_binary(v), do: "'#{v}'" defp expr_name({:literal, v}), do: Value.to_text(v) defp expr_name({:function, name, :star}), do: "#{name}(*)" defp expr_name({:function, name, {:distinct, args}}), do: "#{name}(DISTINCT #{Enum.map_join(args, ", ", &expr_name/1)})" defp expr_name({:function, name, args}), do: "#{name}(#{Enum.map_join(args, ", ", &expr_name/1)})" defp expr_name({:window, name, :star, spec, nil}), do: "#{name}(*) OVER #{window_name(spec)}" defp expr_name({:window, name, args, spec, nil}), do: "#{name}(#{Enum.map_join(args, ", ", &expr_name/1)}) OVER #{window_name(spec)}" defp expr_name({:window, name, args, spec, filter}) do "#{name}(#{Enum.map_join(args, ", ", &expr_name/1)}) FILTER (WHERE #{expr_name(filter)}) OVER #{window_name(spec)}" end defp expr_name({:binary, op, left, right}), do: "#{expr_name(left)} #{op_text(op)} #{expr_name(right)}" defp expr_name({:collate, expr, name}), do: "#{expr_name(expr)} COLLATE #{name}" defp expr_name({:negate, expr}), do: "-#{expr_name(expr)}" defp expr_name({:not, expr}), do: "NOT #{expr_name(expr)}" defp expr_name(_expr), do: "expr" defp window_name({:ref, name}), do: name defp window_name(%{partition_by: [], order_by: []}), do: "()" defp window_name(%{partition_by: partition_by, order_by: order_by}) do parts = [] parts = if partition_by == [], do: parts, else: ["PARTITION BY #{Enum.map_join(partition_by, ", ", &expr_name/1)}" | parts] parts = if order_by == [], do: parts, else: ["ORDER BY #{Enum.map_join(order_by, ", ", &order_expr_name/1)}" | parts] "(" <> (parts |> Enum.reverse() |> Enum.join(" ")) <> ")" end defp order_expr_name({expr, :asc}), do: expr_name(expr) defp order_expr_name({expr, :desc}), do: "#{expr_name(expr)} DESC" # Renders an expression compactly (no spaces around operators) for CHECK # constraint error messages, matching SQLite's behavior of using the raw SQL. defp check_text({:column, nil, name}), do: name defp check_text({:column, table, name}), do: "#{table}.#{name}" defp check_text({:literal, nil}), do: "NULL" defp check_text({:literal, {:blob, b}}), do: "x'#{Base.encode16(b)}'" defp check_text({:literal, v}) when is_binary(v), do: "'#{v}'" defp check_text({:literal, v}), do: Value.to_text(v) defp check_text({:function, name, :star}), do: "#{name}(*)" defp check_text({:function, name, args}), do: "#{name}(#{Enum.map_join(args, ",", &check_text/1)})" defp check_text({:binary, op, left, right}), do: "#{check_text(left)}#{op_text(op)}#{check_text(right)}" defp check_text({:negate, expr}), do: "-#{check_text(expr)}" defp check_text({:not, expr}), do: "NOT #{check_text(expr)}" defp check_text(expr), do: expr_name(expr) defp op_text(op) do %{ eq: "=", ne: "<>", lt: "<", le: "<=", gt: ">", ge: ">=", add: "+", sub: "-", mul: "*", div: "/", mod: "%", concat: "||", and: "AND", or: "OR" }[op] end # -- helpers ------------------------------------------------------------------------- # -- CTE resolution ---------------------------------------------------------- # # Non-recursive: evaluate each CTE in order (materialized), shadowing tables # and prior CTEs. The results are stored in db.ctes. # # Recursive: the standard queue algorithm from https://sqlite.org/lang_with.html # — UNION ALL appends all new rows, UNION only appends rows not yet seen. # A runaway guard stops at 1_000_000 rows. @recursive_row_cap 1_000_000 defp resolve_ctes(db, cte_defs, _recursive, outer_limit) do # Validate for duplicate CTE names names = Enum.map(cte_defs, &Table.key(&1.name)) case Enum.find(Enum.zip(names, cte_defs), fn {key, _} -> Enum.count(names, &(&1 == key)) > 1 end) do {_, cte} -> fail("duplicate WITH table name: #{cte.name}") nil -> :ok end # Effective row cap: use outer LIMIT if given, otherwise the global cap. row_cap = outer_limit || @recursive_row_cap # Build an index of all CTE defs for forward reference resolution. defs_by_key = Map.new(cte_defs, &{Table.key(&1.name), &1}) # Evaluate each CTE in declaration order, supporting forward references. # Cycle detection uses a MapSet of keys currently being evaluated. {db, _} = Enum.reduce(cte_defs, {db, MapSet.new()}, fn cte_def, {db, in_progress} -> key = Table.key(cte_def.name) if Map.has_key?(db.ctes, key) do # Already evaluated (e.g. pulled in as a forward reference) {db, in_progress} else evaluate_cte(db, cte_def, key, defs_by_key, in_progress, row_cap) end end) db end # Evaluate a single CTE, resolving forward references as needed. defp evaluate_cte(db, cte_def, key, defs_by_key, in_progress, row_cap) do if MapSet.member?(in_progress, key) do fail("circular reference: #{cte_def.name}") end in_progress = MapSet.put(in_progress, key) if recursive_cte?(cte_def.query, key, db) do db = resolve_single_recursive_cte(db, cte_def, key, row_cap) {db, in_progress} else # Check for self-references via subqueries (IN, EXISTS, scalar) — these are # circular references, not recursion. if query_references_key?(cte_def.query, key) do fail("circular reference: #{cte_def.name}") end # Resolve any forward references in this CTE's query before evaluating. # When query_result runs into an undefined CTE name, it will raise "no such table". # We intercept by pre-loading forward references here. {db, in_progress} = resolve_forward_refs(db, cte_def.query, key, defs_by_key, in_progress, row_cap) # For a compound query with declared columns: evaluate only the seed first to check # column count against declared columns. This ensures the column count error takes # priority over the compound-width-mismatch error when both would apply. # (Non-compound or no declared columns: skip this pre-check.) case {cte_def.columns, cte_def.query} do {cols, %Compound{left: seed_query}} when cols != nil -> seed_result = query_result(db, seed_query, nil) if length(seed_result.columns) != length(cols) do fail( "table #{cte_def.name} has #{length(seed_result.columns)} values for #{length(cols)} columns" ) end _ -> :ok end result = query_result(db, cte_def.query, nil) {columns, affinities, actual_count} = apply_cte_columns(cte_def, result, result.columns) cte = %{ columns: columns, rows: result.rows, affinities: affinities, actual_count: actual_count } {%{db | ctes: Map.put(db.ctes, key, cte)}, in_progress} end end # Walk a query's FROM clauses and pre-evaluate any CTEs that are referenced # but not yet in db.ctes. defp resolve_forward_refs( db, %Compound{left: l, right: r}, self_key, defs, in_progress, row_cap ) do {db, in_progress} = resolve_forward_refs(db, l, self_key, defs, in_progress, row_cap) resolve_forward_refs(db, r, self_key, defs, in_progress, row_cap) end defp resolve_forward_refs(db, %Select{from: from}, self_key, defs, in_progress, row_cap) do resolve_forward_refs_from(db, from, self_key, defs, in_progress, row_cap) end defp resolve_forward_refs(db, _query, _self_key, _defs, in_progress, _row_cap), do: {db, in_progress} defp resolve_forward_refs_from(db, nil, _self_key, _defs, in_progress, _row_cap), do: {db, in_progress} defp resolve_forward_refs_from(db, {:table, name, _alias}, self_key, defs, in_progress, row_cap) do ref_key = table_source_key(name) cond do # Already resolved or is a table/view or the CTE itself Map.has_key?(db.ctes, ref_key) or Map.has_key?(db.tables, ref_key) -> {db, in_progress} ref_key == self_key -> # Self-reference is allowed only in recursive CTEs; if we're here, # it will naturally cause "no such table" at eval time. {db, in_progress} # Circular reference: self_key is in in_progress, meaning the CTE identified # by self_key references another CTE (ref_key) that is already being evaluated. # Report the current CTE (self_key) as the source of the circular reference. MapSet.member?(in_progress, ref_key) -> self_def = Map.get(defs, self_key) self_name = if self_def, do: self_def.name, else: name fail("circular reference: #{self_name}") Map.has_key?(defs, ref_key) -> evaluate_cte(db, Map.fetch!(defs, ref_key), ref_key, defs, in_progress, row_cap) true -> {db, in_progress} end end defp resolve_forward_refs_from(db, {:subquery, _, _}, _self_key, _defs, in_progress, _row_cap), do: {db, in_progress} defp resolve_forward_refs_from(db, {:join, _t, l, r, _c}, self_key, defs, in_progress, row_cap) do {db, in_progress} = resolve_forward_refs_from(db, l, self_key, defs, in_progress, row_cap) resolve_forward_refs_from(db, r, self_key, defs, in_progress, row_cap) end # Check if a query directly references the CTE key in FROM (not inside a subquery). # A self-reference via EXISTS/IN subquery is a circular reference error, not recursion. defp recursive_cte?(%Compound{left: left, right: right}, key, db) do recursive_cte?(left, key, db) or recursive_cte?(right, key, db) end defp recursive_cte?(%Select{from: from}, key, _db), do: from_references?(from, key) defp recursive_cte?(%Values{}, _key, _db), do: false defp recursive_cte?(%With{}, _key, _db), do: false defp from_references?(nil, _key), do: false defp from_references?({:table, name, _alias}, key), do: table_source_key(name) == key defp from_references?({:subquery, _, _}, _key), do: false defp from_references?({:join, _type, left, right, _constraint}, key) do from_references?(left, key) or from_references?(right, key) end # Check whether a query references `key` anywhere (including in subquery WHERE clauses). # Used to detect self-referential CTEs that aren't direct FROM references (e.g. IN subqueries). defp query_references_key?(%Compound{left: l, right: r}, key) do query_references_key?(l, key) or query_references_key?(r, key) end defp query_references_key?(%Select{from: from, where: where, columns: cols}, key) do from_references_deep?(from, key) or expr_references_key?(where, key) or Enum.any?(cols, &expr_references_key?(&1, key)) end defp query_references_key?(%Values{}, _key), do: false defp query_references_key?(%With{}, _key), do: false defp query_references_key?(nil, _key), do: false # Check FROM for key, including inside subqueries. defp from_references_deep?(nil, _key), do: false defp from_references_deep?({:table, name, _}, key), do: table_source_key(name) == key defp from_references_deep?({:subquery, q, _}, key), do: query_references_key?(q, key) defp from_references_deep?({:join, _, l, r, _}, key) do from_references_deep?(l, key) or from_references_deep?(r, key) end # Check expressions for subquery references to key. defp expr_references_key?(nil, _key), do: false defp expr_references_key?(list, key) when is_list(list), do: Enum.any?(list, &expr_references_key?(&1, key)) # IN with subquery: {:in, expr, {:select, query}, negated} defp expr_references_key?({:in, expr, {:select, q}, _negated}, key) do expr_references_key?(expr, key) or query_references_key?(q, key) end defp expr_references_key?({:in, expr, items, _negated}, key) when is_list(items) do expr_references_key?(expr, key) or Enum.any?(items, &expr_references_key?(&1, key)) end defp expr_references_key?({:exists, q}, key), do: query_references_key?(q, key) defp expr_references_key?({:subquery, q}, key), do: query_references_key?(q, key) defp expr_references_key?({op, l, r}, key) when is_atom(op) do expr_references_key?(l, key) or expr_references_key?(r, key) end defp expr_references_key?({op, arg}, key) when is_atom(op) do expr_references_key?(arg, key) end defp expr_references_key?(_other, _key), do: false defp resolve_single_recursive_cte(db, cte_def, key, row_cap) do # A recursive CTE is: initial UNION [ALL] recursive_part # We require the top-level compound to be UNION or UNION ALL. # If the CTE itself has ORDER BY/LIMIT, extract and apply those after expansion. if contains_window?(cte_def.query) do fail("cannot use window functions in recursive queries") end case cte_def.query do %Compound{ op: op, left: initial_query, right: recursive_query, order_by: order_by, limit: limit, offset: offset } when op in [:union, :union_all] -> # Step 1: evaluate the initial (seed) query initial_result = query_result(db, initial_query, nil) {col_names, affinities, actual_count} = apply_cte_columns(cte_def, initial_result, initial_result.columns) # Validate column count against initial result's declared column count if actual_count != nil and actual_count != length(col_names) do fail( "table #{cte_def.name} has #{actual_count} values for #{length(col_names)} columns" ) end # Validate column count match against the initial result seed_rows = validate_cte_rows(initial_result.rows, col_names, cte_def.name) # Set up the working set all_rows = seed_rows seen = if op == :union, do: MapSet.new(Enum.map(seed_rows, &:erlang.term_to_binary/1)), else: nil working = seed_rows # Use the smaller of row_cap and CTE-level LIMIT (if present) as the expansion cap. expansion_cap = if limit != nil do n = int_clause(limit, db, "LIMIT") min(row_cap, n) else row_cap end # Iteratively expand (stop when no new rows or row_cap/limit reached) {expanded_rows, _seen} = iterate_recursive_cte( db, key, col_names, affinities, cte_def, recursive_query, all_rows, seen, working, op, expansion_cap ) # Apply ORDER BY / LIMIT from the CTE body if present final_rows = if order_by != [] or limit != nil do expanded_rows |> compound_order(order_by, col_names, [col_names]) |> clamp(%{db | ctes: %{}}, limit, offset) else expanded_rows end cte = %{columns: col_names, rows: final_rows, affinities: affinities, actual_count: nil} %{db | ctes: Map.put(db.ctes, key, cte)} _ -> # No UNION/UNION ALL at top — treat as non-recursive result = query_result(db, cte_def.query, nil) {columns, affinities, actual_count} = apply_cte_columns(cte_def, result, result.columns) cte = %{ columns: columns, rows: result.rows, affinities: affinities, actual_count: actual_count } %{db | ctes: Map.put(db.ctes, key, cte)} end end defp iterate_recursive_cte( db, key, col_names, affinities, cte_def, recursive_query, all_rows, seen, working, op, row_cap ) do if working == [] or length(all_rows) >= row_cap do {all_rows, seen} else # Bind the working set as the current value of this CTE working_cte = %{columns: col_names, rows: working, affinities: affinities} db_step = %{db | ctes: Map.put(db.ctes, key, working_cte)} step_result = query_result(db_step, recursive_query, nil) new_rows = validate_cte_rows(step_result.rows, col_names, cte_def.name) # Accumulate new rows for the working set (forward order via reversal) # and append them to all_rows in forward order. {next_working_rev, next_seen, added_rev} = Enum.reduce(new_rows, {[], seen, []}, fn row, {working_acc, seen_acc, added_acc} -> case op do :union_all -> {[row | working_acc], seen_acc, [row | added_acc]} :union -> bin = :erlang.term_to_binary(row) if MapSet.member?(seen_acc, bin) do {working_acc, seen_acc, added_acc} else {[row | working_acc], MapSet.put(seen_acc, bin), [row | added_acc]} end end end) next_working = Enum.reverse(next_working_rev) next_all = all_rows ++ Enum.reverse(added_rev) iterate_recursive_cte( db, key, col_names, affinities, cte_def, recursive_query, next_all, next_seen, next_working, op, row_cap ) end end defp validate_cte_rows(rows, col_names, cte_name) do expected = length(col_names) Enum.each(rows, fn row -> got = length(row) if got != expected do fail("table #{cte_name} has #{got} values for #{expected} columns") end end) rows end defp apply_cte_columns(cte_def, result, result_columns) do case cte_def.columns do nil -> {result_columns, result.affinities, nil} col_names -> affs = result.affinities ++ List.duplicate(:blob, length(col_names)) # Return {declared_columns, affinities, actual_column_count} # Validation happens when the CTE is actually accessed via relation/3 {col_names, Enum.take(affs, length(col_names)), length(result_columns)} end end defp fetch_table!(db, name) do ensure_schema_table_not_modified!(nil, name) # DML against a view is an error case dml_view(db, nil, name) do {:ok, _view} -> fail("cannot modify #{name} because it is a view") :error -> case Database.lookup_table(db, name) do {:ok, table} -> table {:error, message} -> fail(message) end end end defp fetch_table!(db, nil, name), do: fetch_table!(db, name) defp fetch_table!(db, schema, name) do ensure_schema_table_not_modified!(schema, name) ensure_schema_exists!(db, schema) case Map.fetch(db.tables, Database.table_storage_key(schema, name)) do {:ok, table} -> table :error -> fail("no such table: #{name}") end end defp ensure_schema_table_not_modified!(schema, name) do key = Table.key(name) if key in ["sqlite_schema", "sqlite_master"] do fail("table #{schema_label(schema)} may not be modified") end end defp schema_label("temp"), do: "sqlite_temp_master" defp schema_label(_), do: "sqlite_master" defp dml_view(db, nil, name), do: Database.lookup_view(db, name) defp dml_view(db, schema, name), do: Database.fetch_view(db, schema, name) defp put_table(db, %Table{} = table) do Database.put_table(db, refresh_index_entries(db, table)) end defp refresh_index_entries(db, %Table{} = table) do indexes = Enum.map(table.indexes, fn index -> build_index_entries(db, table, index) end) autoindexes = Enum.map(table.autoindexes, fn index -> build_index_entries(db, table, index) end) %{table | indexes: indexes, autoindexes: autoindexes} end defp ensure_index_entries(db, %Table{} = table) do if Enum.all?(lookup_indexes(table), &Map.has_key?(&1, :entries)) do table else refresh_index_entries(db, table) end end # Re-fetches the ensured copy of an index found during planning (which may # have run against a table whose entries were not yet materialized). defp ensured_index(%Table{} = table, index) do Enum.find(lookup_indexes(table), &(&1.name == index.name)) || index end @doc """ Materializes index entries for every table that has indexes without them. Bulk loads (`ExSQL.FileFormat.read/1`) fill `table.rows` directly, bypassing the executor's per-row index maintenance. Building the entries once at load time means reads never pay a rebuild: a query cannot persist lazily-built entries back into the caller's database value, so without this every indexed read re-built (and re-discarded) all of the table's indexes. """ @spec materialize_index_entries(Database.t()) :: Database.t() def materialize_index_entries(%Database{} = db) do tables = Map.new(db.tables, fn {key, table} -> {key, ensure_index_entries(db, table)} end) %{db | tables: tables} end defp indexes_have_entries?(%Table{} = table) do Enum.all?(lookup_indexes(table), &Map.has_key?(&1, :entries)) end # Adds one freshly-inserted row's entry to each index when entries are already # materialized, keeping them current row-by-row; a no-op otherwise (the final # store rebuilds). Lets a bulk insert avoid the per-row full rebuild and lets # the unique-conflict check use the O(1) entry lookup. defp maybe_add_index_entries(db, %Table{} = table, rowid) do if indexes_have_entries?(table), do: add_index_entries(db, table, [rowid]), else: table end # Adds the given rowids' entries to every index, mirroring build_index_entries/3 # per-row logic (partial-index WHERE check, member values, sorted rowid lists), # but only for those rowids rather than rescanning the whole table. defp add_index_entries(db, %Table{} = table, rowids) do %{ table | indexes: Enum.map(table.indexes, &add_rowids_to_index(db, table, &1, rowids)), autoindexes: Enum.map(table.autoindexes, &add_rowids_to_index(db, table, &1, rowids)) } end defp add_rowids_to_index(db, table, index, rowids) do entries = Enum.reduce(rowids, Map.fetch!(index, :entries), fn rowid, acc -> # Read the raw stored tuple (no per-row widen to a map): `index_member_values` # reads only the index's member columns, positionally. Widening here ran # once per index per insert — O(columns) x N-indexes on every row. case Map.get(table.rows, rowid) do nil -> acc row -> if index.where == nil or row_matches_partial_index?(db, table, rowid, row, index.where) do key = List.to_tuple(index_member_values(db, table, rowid, row, index)) # Prepend (O(1)); the per-key rowid list is sorted lazily at read # (`index_lookup_rowids`) instead of re-sorting the whole list on # every insert — that re-sort was O(n²) for low-cardinality indexes # (each key accumulates O(n) rowids). Map.update(acc, key, [rowid], &[rowid | &1]) else acc end end end) index |> Map.put(:entries, entries) |> Map.delete(:ordered_entries) end # Removes the given deleted `{rowid, row}`s from each index's entries, when # entries are materialized — the delete counterpart of add_index_entries/3, so # a delete touches only the affected keys instead of rescanning the whole # table to rebuild every index. defp remove_index_entries(db, %Table{} = table, deleted_pairs) do if indexes_have_entries?(table) do %{ table | indexes: Enum.map(table.indexes, &remove_pairs_from_index(db, table, &1, deleted_pairs)), autoindexes: Enum.map(table.autoindexes, &remove_pairs_from_index(db, table, &1, deleted_pairs)) } else table end end defp remove_pairs_from_index(db, table, index, deleted_pairs) do # Group the removed rowids by their index key, then filter each affected # key's list once against a set (O(k) per key) instead of `rowids -- [rowid]` # per deleted pair (O(k²) when many rows share a low-cardinality key). removed_by_key = Enum.reduce(deleted_pairs, %{}, fn {rowid, row}, acc -> if index.where == nil or row_matches_partial_index?(db, table, rowid, row, index.where) do key = List.to_tuple(index_member_values(db, table, rowid, row, index)) Map.update(acc, key, [rowid], &[rowid | &1]) else acc end end) entries = Enum.reduce(removed_by_key, Map.fetch!(index, :entries), fn {key, removed}, acc -> case acc do %{^key => rowids} -> removed_set = MapSet.new(removed) case Enum.reject(rowids, &MapSet.member?(removed_set, &1)) do [] -> Map.delete(acc, key) kept -> Map.put(acc, key, kept) end _ -> acc end end) index |> Map.put(:entries, entries) |> Map.delete(:ordered_entries) end defp build_index_entries(db, table, index) do entries = table |> Table.scan() |> Enum.reduce(%{}, fn {rowid, row}, entries -> if index.where == nil or row_matches_partial_index?(db, table, rowid, row, index.where) do values = index_member_values(db, table, rowid, row, index) Map.update(entries, List.to_tuple(values), [rowid], fn rowids -> [rowid | rowids] end) else entries end end) |> Map.new(fn {values, rowids} -> {values, Enum.sort(rowids)} end) index |> Map.put(:entries, entries) |> maybe_put_ordered_entries(entries) end defp maybe_put_ordered_entries(index, entries) do if binary_collation_index?(index) do ordered = entries |> Enum.sort(fn {left, _left_rowids}, {right, _right_rowids} -> compare_index_keys(left, right) != :gt end) |> List.to_tuple() Map.put(index, :ordered_entries, ordered) else Map.delete(index, :ordered_entries) end end defp compare_index_keys(left, right) do left_values = Tuple.to_list(left) right_values = Tuple.to_list(right) left_values |> Enum.zip(right_values) |> Enum.reduce_while(:eq, fn {left_value, right_value}, :eq -> case Value.compare(left_value, right_value) do :eq -> {:cont, :eq} other -> {:halt, other} end end) end defp ensure_unique_names(%CreateTable{} = stmt) do if stmt.columns == [], do: fail("table #{stmt.name} must have at least one column") duplicate = stmt.columns |> Enum.frequencies_by(&Table.key(&1.name)) |> Enum.find(fn {_name, count} -> count > 1 end) case duplicate do {name, _} -> fail("duplicate column name: #{name}") nil -> :ok end # Count inline primary keys inline_pk_count = Enum.count(stmt.columns, & &1.primary_key) # Count table-level primary key constraints table_pk_count = Enum.count(stmt.constraints, &match?({:primary_key, _, _}, &1)) total_pk = inline_pk_count + table_pk_count case total_pk do n when n > 1 -> fail("table #{stmt.name} has more than one primary key") _ -> :ok end end defp ensure_valid_autoincrement!(%CreateTable{} = stmt) do case Enum.filter(stmt.columns, & &1.autoincrement) do [] -> :ok [_column] when stmt.without_rowid -> fail("AUTOINCREMENT not allowed on WITHOUT ROWID tables") [column] when column.primary_key and column.affinity == :integer -> :ok [_ | _] -> fail("AUTOINCREMENT is only allowed on an INTEGER PRIMARY KEY") end end defp ensure_without_rowid_primary_key!(%CreateTable{without_rowid: false}), do: :ok defp ensure_without_rowid_primary_key!(%CreateTable{} = stmt) do has_primary_key? = Enum.any?(stmt.columns, & &1.primary_key) or Enum.any?(stmt.constraints, &match?({:primary_key, _, _}, &1)) unless has_primary_key? do fail("PRIMARY KEY missing on table #{stmt.name}") end end defp ensure_valid_strict_types!(%CreateTable{strict: false}), do: :ok defp ensure_valid_strict_types!(%CreateTable{} = stmt) do Enum.each(stmt.columns, fn column -> case column.declared_type do nil -> fail("missing datatype for #{stmt.name}.#{column.name}") type -> unless String.upcase(type) in ["INT", "INTEGER", "REAL", "TEXT", "BLOB", "ANY"] do fail(~s(unknown datatype for #{stmt.name}.#{column.name}: "#{type}")) end end end) end # Raises if the existing table data already violates a new unique index. defp check_unique_index_data!(db, table, index) do rows = table |> Table.scan() |> Enum.filter(fn {rowid, row} -> index.where == nil or row_matches_partial_index?(db, table, rowid, row, index.where) end) # Build value tuples for each row, skipping rows with any NULL value_tuples = rows |> Enum.map(fn {rowid, row} -> index_member_values(db, table, rowid, row, index) end) |> Enum.reject(fn vals -> Enum.any?(vals, &is_nil/1) end) # Check for duplicates using the index collations for type-correct equality. Enum.reduce_while(value_tuples, [], fn tuple, seen -> duplicate? = Enum.any?(seen, fn existing -> index_values_equal?(db, index, existing, tuple) end) if duplicate? do fail(index_conflict_message(table, index)) else {:cont, [tuple | seen]} end end) end @spec fail(String.t()) :: no_return() defp fail(message), do: raise(Error, message: message) # Conflict-clause failure: ABORT (the default) discards the statement's # changes, FAIL keeps the rows already changed, ROLLBACK additionally # rolls back and closes the enclosing transaction (or acts as ABORT when # there is none), as in SQLite's conflict-resolution algorithms. defp conflict_fail!(db, table, on_conflict, message) do case on_conflict do :fail -> raise(Error, message: message, db: put_table(db, table)) :rollback -> case List.last(db.txn_stack) do nil -> fail(message) {_kind, snapshot} -> db = %{db | txn_stack: [], defer_foreign_keys: false} raise(Error, message: message, db: Database.restore_schema(db, snapshot)) end _abort -> fail(message) end end end