defmodule ExSQL.Json do @moduledoc """ The json1 function family's data layer: a JSON parser/renderer over an order-preserving representation, plus path navigation (`$.a.b[0]`, `$[#-1]`). JSON values map to: * `:null`, `true`, `false` * numbers (integer, float, or preserved real literal token) * strings (binaries) * `{:array, [value]}` * `{:object, [{key, value}]}` — pairs in source/insertion order, as SQLite preserves object member order in `json_object` and rewrites Hand-rolled rather than `JSON`/`:json` because object member order is observable in SQLite's output. """ @type object_key :: binary() | {:jsonb_path_key, binary()} | {:jsonb_integer_literal, 4, binary(), integer()} | {:jsonb_escaped_text, 9, binary(), binary()} | {:jsonb_escaped_key, 8 | 9, binary(), binary()} @type t :: :null | boolean() | number() | {:real_literal, binary()} | {:jsonb_integer_literal, 4, binary(), integer()} | {:jsonb_real_literal, 5 | 6, binary()} | {:jsonb_text, 10, binary()} | {:jsonb_escaped_text, 8 | 9, binary(), binary()} | binary() | {:array, [t()]} | {:object, [{object_key(), t()}]} @type path_step :: {:key, binary()} | {:index, non_neg_integer()} | {:last_offset, integer()} # -- parsing ----------------------------------------------------------------- @spec parse(binary()) :: {:ok, t()} | :error def parse(text), do: parse(text, :canonical) @spec parse_json5(binary()) :: {:ok, t()} | :error def parse_json5(text), do: parse(text, :json5) defp parse(text, mode) when is_binary(text) do case value(skip_ws(text, mode), mode) do {:ok, jv, rest} -> case skip_ws(rest, mode) do "" -> {:ok, jv} _trailing -> :error end :error -> :error end end @spec parse_sqlite_jsonb(binary()) :: {:ok, t()} | :error def parse_sqlite_jsonb(blob) when is_binary(blob) do case sqlite_jsonb_value(blob) do {:ok, jv, ""} -> {:ok, jv} _other -> :error end end @spec sqlite_jsonb_strict?(binary()) :: boolean() def sqlite_jsonb_strict?(blob) when is_binary(blob) do case sqlite_jsonb_payload(blob) do {:ok, type, payload, ""} -> strict_jsonb_payload?(type, payload) _other -> false end end @spec sqlite_jsonb_superficial?(binary()) :: boolean() def sqlite_jsonb_superficial?(<> = blob) do case sqlite_jsonb_payload(blob) do {:ok, type, payload, ""} -> superficial_jsonb_payload?(header, type, payload) _other -> false end end def sqlite_jsonb_superficial?(blob) when is_binary(blob), do: false defp superficial_jsonb_payload?(_header, type, payload) when type in [0, 1, 2], do: payload == "" defp superficial_jsonb_payload?(header, _type, payload) do if byte_size(payload) > 7 or not jsonb_text_json_ambiguous?(header) do true else case sqlite_jsonb_payload(Bitwise.band(header, 0x0F), payload, "") do {:ok, value, ""} -> strict_jsonb_value?(value) :error -> false end end end defp jsonb_text_json_ambiguous?(header), do: header in [?{, ?[] or header in ?0..?9 defp skip_ws(<>, mode) when c in [?\s, ?\t, ?\n, ?\r], do: skip_ws(rest, mode) defp skip_ws(<>, :json5) when c in [?\f, ?\v], do: skip_ws(rest, :json5) defp skip_ws(<<"\u00A0", rest::binary>>, :json5), do: skip_ws(rest, :json5) defp skip_ws(<<"//", rest::binary>>, :json5) do rest |> skip_line_comment() |> skip_ws(:json5) end defp skip_ws(<<"/*", rest::binary>>, :json5) do case skip_block_comment(rest) do {:ok, rest} -> skip_ws(rest, :json5) :error -> <<0>> end end defp skip_ws(text, _mode), do: text defp skip_line_comment(<<"\n", rest::binary>>), do: rest defp skip_line_comment(<<"\r", rest::binary>>), do: rest defp skip_line_comment(<<_c, rest::binary>>), do: skip_line_comment(rest) defp skip_line_comment(<<>>), do: "" defp skip_block_comment(<<"*/", rest::binary>>), do: {:ok, rest} defp skip_block_comment(<<_c, rest::binary>>), do: skip_block_comment(rest) defp skip_block_comment(<<>>), do: :error defp value(<<"null", rest::binary>>, _mode), do: {:ok, :null, rest} defp value(<<"true", rest::binary>>, _mode), do: {:ok, true, rest} defp value(<<"false", rest::binary>>, _mode), do: {:ok, false, rest} defp value(<>, mode), do: string(rest, ?", mode, []) defp value(<>, :json5), do: string(rest, ?', :json5, []) defp value(<>, mode), do: object(skip_ws(rest, mode), mode, []) defp value(<>, mode), do: array(skip_ws(rest, mode), mode, []) defp value(text, :json5) do case json5_nonfinite(text) do {:ok, value, rest} -> {:ok, value, rest} :error -> json5_value(text) end end defp value(<> = text, _mode) when c in [?-, ?+] or c in ?0..?9, do: number(text, :canonical) defp value(_, _mode), do: :error defp json5_value(<> = text) when c in [?-, ?+] or c in ?0..?9, do: number(text, :json5) defp json5_value(<> = text) do case rest do <> when c in ?0..?9 -> number(text, :json5) _ -> :error end end defp json5_value(_text), do: :error defp json5_nonfinite(text) do downcased = String.downcase(text) [ {"+infinity", :json_pos_inf}, {"-infinity", :json_neg_inf}, {"infinity", :json_pos_inf}, {"+inf", :json_pos_inf}, {"-inf", :json_neg_inf}, {"inf", :json_pos_inf}, {"qnan", :json_nan}, {"snan", :json_nan}, {"nan", :json_nan} ] |> Enum.find_value(:error, fn {token, value} -> size = byte_size(token) case downcased do <<^token::binary-size(^size), _::binary>> -> <<_matched::binary-size(^size), rest::binary>> = text if json5_token_boundary?(rest) do {:ok, value, rest} end _other -> nil end end) end defp json5_token_boundary?(<>) when c == ?_ or c == ?$ or c in ?A..?Z or c in ?a..?z or c in ?0..?9, do: false defp json5_token_boundary?(_rest), do: true defp sqlite_jsonb_value(<>) do with {:ok, type, payload, rest} <- sqlite_jsonb_payload(<>) do sqlite_jsonb_payload(type, payload, rest) end end defp sqlite_jsonb_value(_blob), do: :error defp sqlite_jsonb_payload(<>) do type = Bitwise.band(header, 0x0F) size_code = Bitwise.bsr(header, 4) with {:ok, size, rest} <- sqlite_jsonb_payload_size(size_code, rest), true <- byte_size(rest) >= size do <> = rest {:ok, type, payload, rest} else _other -> :error end end defp sqlite_jsonb_payload(_blob), do: :error defp sqlite_jsonb_payload_size(size, rest) when size <= 11, do: {:ok, size, rest} defp sqlite_jsonb_payload_size(12, <>), do: {:ok, size, rest} defp sqlite_jsonb_payload_size(13, <>), do: {:ok, size, rest} defp sqlite_jsonb_payload_size(14, <>), do: {:ok, size, rest} defp sqlite_jsonb_payload_size(15, <>), do: {:ok, size, rest} defp sqlite_jsonb_payload_size(_size, _rest), do: :error defp sqlite_jsonb_payload(0, "", rest), do: {:ok, :null, rest} defp sqlite_jsonb_payload(1, "", rest), do: {:ok, true, rest} defp sqlite_jsonb_payload(2, "", rest), do: {:ok, false, rest} defp sqlite_jsonb_payload(3, payload, rest) do case Integer.parse(payload) do {value, ""} -> {:ok, value, rest} _other -> :error end end defp sqlite_jsonb_payload(4, payload, rest) do case Integer.parse(payload) do {value, ""} -> {:ok, {:jsonb_integer_literal, 4, payload, value}, rest} _other -> :error end end defp sqlite_jsonb_payload(type, payload, rest) when type in [5, 6] do case parse_jsonb_real_literal(type, payload) do {:ok, _value} -> {:ok, {:jsonb_real_literal, type, payload}, rest} :error -> :error end end defp sqlite_jsonb_payload(7, payload, rest), do: {:ok, payload, rest} defp sqlite_jsonb_payload(10, payload, rest), do: {:ok, {:jsonb_text, 10, payload}, rest} defp sqlite_jsonb_payload(type, payload, rest) when type in [8, 9] do case sqlite_jsonb_decode_escaped_text(type, payload) do {:ok, value} -> {:ok, {:jsonb_escaped_text, type, payload, value}, rest} _other -> :error end end defp sqlite_jsonb_payload(11, payload, rest) do case sqlite_jsonb_items(payload, []) do {:ok, items} -> {:ok, {:array, items}, rest} :error -> :error end end defp sqlite_jsonb_payload(12, payload, rest) do case sqlite_jsonb_pairs(payload, []) do {:ok, pairs} -> {:ok, {:object, pairs}, rest} :error -> :error end end defp sqlite_jsonb_payload(_type, _payload, _rest), do: :error defp sqlite_jsonb_items("", acc), do: {:ok, Enum.reverse(acc)} defp sqlite_jsonb_items(payload, acc) do case sqlite_jsonb_value(payload) do {:ok, item, rest} -> sqlite_jsonb_items(rest, [item | acc]) :error -> :error end end defp sqlite_jsonb_pairs("", acc), do: {:ok, Enum.reverse(acc)} defp sqlite_jsonb_pairs(payload, acc) do with {:ok, key, rest} <- sqlite_jsonb_object_key(payload), {:ok, value, rest} <- sqlite_jsonb_value(rest) do sqlite_jsonb_pairs(rest, [{key, value} | acc]) else _other -> :error end end defp sqlite_jsonb_object_key(blob) do with {:ok, type, payload, rest} <- sqlite_jsonb_payload(blob) do sqlite_jsonb_object_key(type, payload, rest) end end defp sqlite_jsonb_object_key(type, payload, rest) when type in [7, 10], do: {:ok, if(type == 10, do: {:jsonb_path_key, payload}, else: payload), rest} defp sqlite_jsonb_object_key(type, payload, rest) when type in [8, 9] do case sqlite_jsonb_decode_escaped_text(type, payload) do {:ok, value} -> {:ok, {:jsonb_escaped_key, type, payload, value}, rest} _other -> :error end end defp sqlite_jsonb_object_key(_type, _payload, _rest), do: :error defp strict_jsonb_payload?(type, payload) when type in [0, 1, 2], do: payload == "" defp strict_jsonb_payload?(3, payload) do match?({_value, ""}, Integer.parse(payload)) end defp strict_jsonb_payload?(4, _payload), do: false defp strict_jsonb_payload?(5, payload), do: match?({:ok, _value}, parse_jsonb_real_literal(5, payload)) defp strict_jsonb_payload?(6, "+" <> _payload), do: false defp strict_jsonb_payload?(6, payload), do: match?({:ok, _value}, parse_jsonb_real_literal(6, payload)) defp strict_jsonb_payload?(7, payload), do: jsonb_text_payload_strict?(payload) defp strict_jsonb_payload?(8, payload), do: jsonb_textj_payload_strict?(payload) defp strict_jsonb_payload?(type, payload) when type in [9, 10] do match?({:ok, _value}, sqlite_jsonb_decode_escaped_text(type, payload)) end defp strict_jsonb_payload?(11, payload), do: strict_jsonb_items?(payload) defp strict_jsonb_payload?(12, payload), do: strict_jsonb_pairs?(payload) defp strict_jsonb_payload?(_type, _payload), do: false defp strict_jsonb_items?(""), do: true defp strict_jsonb_items?(payload) do with {:ok, type, item, rest} <- sqlite_jsonb_payload(payload), true <- strict_jsonb_payload?(type, item) do strict_jsonb_items?(rest) else _other -> false end end defp strict_jsonb_pairs?(""), do: true defp strict_jsonb_pairs?(payload) do with {:ok, type, key, rest} <- sqlite_jsonb_payload(payload), true <- type in 7..10, true <- strict_jsonb_payload?(type, key), {:ok, value_type, value, rest} <- sqlite_jsonb_payload(rest), true <- strict_jsonb_payload?(value_type, value) do strict_jsonb_pairs?(rest) else _other -> false end end defp jsonb_text_payload_strict?(payload) do payload |> :binary.bin_to_list() |> Enum.all?(fn byte -> byte > 0x1F and byte not in [?", ?\\] end) end defp jsonb_textj_payload_strict?(payload), do: jsonb_textj_payload_strict?(payload, true) defp jsonb_textj_payload_strict?(<<>>, ok?), do: ok? defp jsonb_textj_payload_strict?(<>, true) when byte > 0x1F and byte not in [?", ?\\], do: jsonb_textj_payload_strict?(rest, true) defp jsonb_textj_payload_strict?(<>, true) when escape in [?\", ?\\, ?/, ?b, ?f, ?n, ?r, ?t], do: jsonb_textj_payload_strict?(rest, true) defp jsonb_textj_payload_strict?(<>, true) do case Integer.parse(hex, 16) do {_code, ""} -> jsonb_textj_payload_strict?(rest, true) _other -> false end end defp jsonb_textj_payload_strict?(_payload, _ok?), do: false defp strict_jsonb_value?({:jsonb_integer_literal, 4, _payload, _value}), do: false defp strict_jsonb_value?({:jsonb_real_literal, 6, "+" <> _payload}), do: false defp strict_jsonb_value?({:array, items}), do: Enum.all?(items, &strict_jsonb_value?/1) defp strict_jsonb_value?({:object, pairs}) do Enum.all?(pairs, fn {key, value} -> strict_jsonb_value?(key) and strict_jsonb_value?(value) end) end defp strict_jsonb_value?(_jv), do: true defp sqlite_jsonb_decode_escaped_text(type, payload) do case parse("\"" <> payload <> "\"", if(type == 9, do: :json5, else: :canonical)) do {:ok, value} when is_binary(value) -> {:ok, value} {:ok, {:jsonb_escaped_text, _type, _payload, value}} -> {:ok, value} _other -> :error end end defp object(<>, _mode, pairs), do: {:ok, {:object, Enum.reverse(pairs)}, rest} defp object(text, mode, pairs) do with {:ok, key, rest} <- object_key(text, mode), <> <- skip_ws(rest, mode), {:ok, jv, rest} <- value(skip_ws(rest, mode), mode) do case skip_ws(rest, mode) do <> -> rest = skip_ws(rest, mode) case {rest, mode} do {<>, :json5} -> {:ok, {:object, Enum.reverse([{key, jv} | pairs])}, rest} _ -> object(rest, mode, [{key, jv} | pairs]) end <> -> {:ok, {:object, Enum.reverse([{key, jv} | pairs])}, rest} _ -> :error end else _ -> :error end end defp object_key(<>, mode), do: string(rest, ?", mode, []) defp object_key(<>, :json5), do: string(rest, ?', :json5, []) defp object_key(text, :json5), do: bare_key(text, []) defp object_key(_text, _mode), do: :error defp bare_key(<>, acc), do: bare_key_unicode_escape(rest, acc) defp bare_key(<>, acc) when c == ?_ or c == ?$ or c in ?A..?Z or c in ?a..?z, do: bare_key_rest(rest, [acc, c]) defp bare_key(<>, acc) when c >= 0x80, do: bare_key_rest(rest, [acc, <>]) defp bare_key(_text, _acc), do: :error defp bare_key_rest(<>, acc), do: bare_key_unicode_escape(rest, acc) defp bare_key_rest(<>, acc) when c == ?_ or c == ?$ or c in ?A..?Z or c in ?a..?z or c in ?0..?9, do: bare_key_rest(rest, [acc, c]) defp bare_key_rest(<>, acc) when c >= 0x80, do: bare_key_rest(rest, [acc, <>]) defp bare_key_rest(rest, acc), do: {:ok, finish_bare_key(acc), rest} defp bare_key_unicode_escape(<>, acc) do case Integer.parse(hex, 16) do {code, ""} when code in 0xD800..0xDBFF -> case rest do <> -> case Integer.parse(hex2, 16) do {low, ""} when low in 0xDC00..0xDFFF -> code = 0x10000 + (code - 0xD800) * 0x400 + (low - 0xDC00) payload = ["\\u", hex, "\\u", hex2] bare_key_rest(rest2, [acc, {:bare_key_unicode_escape, payload, <>}]) _other -> :error end _other -> :error end {code, ""} when code not in 0xDC00..0xDFFF -> bare_key_rest(rest, [acc, {:bare_key_unicode_escape, ["\\u", hex], <>}]) _other -> :error end end defp bare_key_unicode_escape(_rest, _acc), do: :error defp finish_bare_key(acc) do if bare_key_has_unicode_escape?(acc) do {payload, value} = bare_key_parts(List.wrap(acc), [], []) {:jsonb_escaped_key, 8, IO.iodata_to_binary(payload), IO.iodata_to_binary(value)} else IO.iodata_to_binary(acc) end end defp bare_key_has_unicode_escape?({:bare_key_unicode_escape, _payload, _value}), do: true defp bare_key_has_unicode_escape?(list) when is_list(list), do: Enum.any?(list, &bare_key_has_unicode_escape?/1) defp bare_key_has_unicode_escape?(_part), do: false defp bare_key_parts([], payload, value), do: {Enum.reverse(payload), Enum.reverse(value)} defp bare_key_parts([{:bare_key_unicode_escape, escape, decoded} | rest], payload, value) do bare_key_parts(rest, [escape | payload], [decoded | value]) end defp bare_key_parts([part | rest], payload, value) when is_list(part) do {part_payload, part_value} = bare_key_parts(part, [], []) bare_key_parts(rest, [part_payload | payload], [part_value | value]) end defp bare_key_parts([part | rest], payload, value) do bare_key_parts(rest, [part | payload], [part | value]) end defp array(<>, _mode, items), do: {:ok, {:array, Enum.reverse(items)}, rest} defp array(text, mode, items) do case value(text, mode) do {:ok, jv, rest} -> case skip_ws(rest, mode) do <> -> rest = skip_ws(rest, mode) case {rest, mode} do {<>, :json5} -> {:ok, {:array, Enum.reverse([jv | items])}, rest} _ -> array(rest, mode, [jv | items]) end <> -> {:ok, {:array, Enum.reverse([jv | items])}, rest} _ -> :error end :error -> :error end end defp string(<>, quote, _mode, acc), do: {:ok, finish_string(acc), rest} defp string(<>, quote, mode, acc) do case escape do ?" -> string(rest, quote, mode, [acc, ?"]) ?' when mode == :json5 -> json5_string_escape(rest, quote, mode, acc, "\\'", "'") ?' -> string(rest, quote, mode, [acc, ?']) ?\\ -> string(rest, quote, mode, [acc, ?\\]) ?/ -> string(rest, quote, mode, [acc, ?/]) ?0 when mode == :json5 -> json5_nul_escape(rest, quote, mode, acc) ?b -> string(rest, quote, mode, [acc, ?\b]) ?f -> string(rest, quote, mode, [acc, ?\f]) ?n -> string(rest, quote, mode, [acc, ?\n]) ?r -> string(rest, quote, mode, [acc, ?\r]) ?t -> string(rest, quote, mode, [acc, ?\t]) ?u -> unicode_escape(rest, quote, mode, acc) ?x when mode == :json5 -> json5_hex_escape(rest, quote, mode, acc) ?v when mode == :json5 -> json5_string_escape(rest, quote, mode, acc, "\\v", <>) 0xE2 when mode == :json5 -> json5_line_separator_escape(rest, quote, mode, acc) ?\n when mode == :json5 -> json5_string_escape(rest, quote, mode, acc, "\\\n", "") ?\r when mode == :json5 -> json5_cr_escape(rest, quote, mode, acc) _ -> :error end end defp string(<>, quote, mode, acc) when c != quote, do: string(rest, quote, mode, [acc, c]) defp string(<<>>, _quote, _mode, _acc), do: :error defp unicode_escape(<>, quote, mode, acc) do case Integer.parse(hex, 16) do {code, ""} when code in 0xD800..0xDBFF -> # A UTF-16 surrogate pair split across two \u escapes. case rest do <> -> case Integer.parse(hex2, 16) do {low, ""} when low in 0xDC00..0xDFFF -> code = 0x10000 + (code - 0xD800) * 0x400 + (low - 0xDC00) json_string_escape( rest2, quote, mode, acc, ["\\u", hex, "\\u", hex2], <> ) _ -> json_string_escape( rest, quote, mode, acc, ["\\u", hex], json_unicode_code_bytes(code) ) end _ -> json_string_escape( rest, quote, mode, acc, ["\\u", hex], json_unicode_code_bytes(code) ) end {code, ""} -> json_string_escape(rest, quote, mode, acc, ["\\u", hex], json_unicode_code_bytes(code)) _ -> :error end end defp unicode_escape(_, _quote, _mode, _acc), do: :error defp json_unicode_code_bytes(code) when code in 0xD800..0xDFFF do <<0xE0 + Bitwise.bsr(code, 12), 0x80 + Bitwise.band(Bitwise.bsr(code, 6), 0x3F), 0x80 + Bitwise.band(code, 0x3F)>> end defp json_unicode_code_bytes(code), do: <> defp json5_hex_escape(<>, quote, mode, acc) do case Integer.parse(hex, 16) do {code, ""} -> json5_string_escape(rest, quote, mode, acc, ["\\x", hex], <>) _other -> :error end end defp json5_hex_escape(_rest, _quote, _mode, _acc), do: :error defp json5_nul_escape(<>, _quote, _mode, _acc) when digit in ?0..?9, do: :error defp json5_nul_escape(rest, quote, mode, acc), do: json5_string_escape(rest, quote, mode, acc, "\\0", <<0>>) defp json5_cr_escape(<<"\n", rest::binary>>, quote, mode, acc), do: json5_string_escape(rest, quote, mode, acc, "\\\r\n", "") defp json5_cr_escape(rest, quote, mode, acc), do: json5_string_escape(rest, quote, mode, acc, "\\\r", "") defp json5_line_separator_escape(<<0x80, sep, rest::binary>>, quote, mode, acc) when sep in [0xA8, 0xA9], do: json5_string_escape(rest, quote, mode, acc, <>, "") defp json5_line_separator_escape(_rest, _quote, _mode, _acc), do: :error defp json_string_escape(rest, quote, mode, acc, payload, decoded), do: string(rest, quote, mode, [acc, {:json_string_escape, payload, decoded}]) defp json5_string_escape(rest, quote, mode, acc, payload, decoded), do: string(rest, quote, mode, [acc, {:json5_string_escape, payload, decoded}]) defp finish_string(acc) do cond do string_has_json5_escape?(acc) -> {payload, value} = json_string_parts(List.wrap(acc), [], []) {:jsonb_escaped_text, 9, IO.iodata_to_binary(payload), IO.iodata_to_binary(value)} string_has_json_escape?(acc) -> {payload, value} = json_string_parts(List.wrap(acc), [], []) {:jsonb_escaped_text, 8, IO.iodata_to_binary(payload), IO.iodata_to_binary(value)} true -> IO.iodata_to_binary(acc) end end defp string_has_json5_escape?({:json5_string_escape, _payload, _value}), do: true defp string_has_json5_escape?(list) when is_list(list), do: Enum.any?(list, &string_has_json5_escape?/1) defp string_has_json5_escape?(_part), do: false defp string_has_json_escape?({:json_string_escape, _payload, _value}), do: true defp string_has_json_escape?({:json5_string_escape, _payload, _value}), do: true defp string_has_json_escape?(list) when is_list(list), do: Enum.any?(list, &string_has_json_escape?/1) defp string_has_json_escape?(_part), do: false defp json_string_parts([], payload, value), do: {Enum.reverse(payload), Enum.reverse(value)} defp json_string_parts([{:json_string_escape, escape, decoded} | rest], payload, value) do json_string_parts(rest, [escape | payload], [decoded | value]) end defp json_string_parts([{:json5_string_escape, escape, decoded} | rest], payload, value) do json_string_parts(rest, [escape | payload], [decoded | value]) end defp json_string_parts([part | rest], payload, value) when is_list(part) do {part_payload, part_value} = json_string_parts(part, [], []) json_string_parts(rest, [part_payload | payload], [part_value | value]) end defp json_string_parts([part | rest], payload, value) do json_string_parts(rest, [part | payload], [part | value]) end defp number(text, mode) do {token, rest} = take_number(text, []) token = IO.iodata_to_binary(token) parse_number_token(token, rest, mode) end defp parse_number_token(token, rest, :canonical) do cond do String.match?(token, ~r/^-?\d+$/) -> {:ok, String.to_integer(token), rest} String.match?(token, ~r/^-?\d+(\.\d+)?([eE][+-]?\d+)?$/) -> case parse_real_literal(token) do {:ok, value} -> {:ok, value, rest} :error -> :error end true -> :error end end defp parse_number_token(token, rest, :json5) do cond do String.match?(token, ~r/^[+-]?0[xX][0-9a-fA-F]+$/) -> sign = if String.starts_with?(token, "-"), do: -1, else: 1 unsigned = token |> String.trim_leading("+") |> String.trim_leading("-") digits = binary_part(unsigned, 2, byte_size(unsigned) - 2) {value, ""} = Integer.parse(digits, 16) {:ok, sign * value, rest} String.match?(token, ~r/^[+-]?(?:0|[1-9]\d*)$/) -> {:ok, String.to_integer(token), rest} String.match?(token, ~r/^[+-]?(?:\d+\.\d*|\.\d+)(?:[eE][+-]?\d+)?$/) or String.match?(token, ~r/^[+-]?\d+[eE][+-]?\d+$/) -> case token |> normalize_json5_number() |> parse_real_literal() do {:ok, value} -> {:ok, value, rest} :error -> :error end true -> :error end end defp normalize_json5_number(<<"-.", rest::binary>>), do: "-0." <> rest defp normalize_json5_number(<<"+.", rest::binary>>), do: "0." <> rest defp normalize_json5_number(<>), do: "0." <> rest defp normalize_json5_number(<>), do: normalize_json5_number(rest) defp normalize_json5_number(token) do if String.ends_with?(token, ".") do token <> "0" else token end end defp parse_real_literal(token) do case Float.parse(token) do {_value, ""} -> {:ok, {:real_literal, token}} :error -> if String.match?(token, ~r/^[+-]?(?:\d+(?:\.\d*)?|\.\d+)[eE][+-]?\d+$/) do {:ok, {:real_literal, token}} else :error end _other -> :error end end defp parse_jsonb_real_literal(5, token), do: parse_real_literal(token) defp parse_jsonb_real_literal(6, "+" <> token) do if String.contains?(token, ".") do :error else parse_real_literal(token) end end defp parse_jsonb_real_literal(6, token) do token |> normalize_json5_number() |> parse_real_literal() end defp take_number(<>, acc) when c in ?0..?9 or c in [?-, ?+, ?., ?e, ?E, ?x, ?X] or c in ?a..?f or c in ?A..?F, do: take_number(rest, [acc, c]) defp take_number(rest, acc), do: {acc, rest} # -- rendering --------------------------------------------------------------- @spec render(t()) :: binary() def render(jv), do: jv |> encode() |> IO.iodata_to_binary() @spec to_sqlite_jsonb(t()) :: binary() def to_sqlite_jsonb(jv), do: jv |> sqlite_jsonb_encode() |> IO.iodata_to_binary() @spec object_key_text(object_key()) :: binary() def object_key_text({:jsonb_path_key, key}), do: key def object_key_text({:jsonb_escaped_text, _type, _payload, key}), do: key def object_key_text({:jsonb_escaped_key, _type, _payload, key}), do: key def object_key_text(key), do: key @spec pretty(t(), binary()) :: binary() def pretty(jv, indent \\ " "), do: jv |> pretty_encode(indent, 0) |> IO.iodata_to_binary() defp encode(:null), do: "null" defp encode(true), do: "true" defp encode(false), do: "false" defp encode(:json_pos_inf), do: "9e999" defp encode(:json_neg_inf), do: "-9e999" defp encode(:json_nan), do: "null" defp encode({:real_literal, token}), do: token defp encode({:jsonb_integer_literal, 4, payload, _value}), do: encode_jsonb_int5(payload) defp encode({:jsonb_real_literal, 6, token}), do: encode_jsonb_float5(token) defp encode({:jsonb_real_literal, _type, token}), do: token defp encode({:jsonb_text, 10, value}), do: [?", escape(value), ?"] defp encode({:jsonb_escaped_text, 9, payload, _value}), do: [?", canonical_json5_payload(payload), ?"] defp encode({:jsonb_escaped_text, _type, payload, _value}), do: [?", payload, ?"] defp encode(n) when is_integer(n), do: Integer.to_string(n) defp encode(n) when is_float(n), do: Float.to_string(n) defp encode(s) when is_binary(s), do: [?", escape(s), ?"] defp encode({:array, items}), do: [?[, Enum.map_intersperse(items, ?,, &encode/1), ?]] defp encode({:object, pairs}) do [ ?{, Enum.map_intersperse(pairs, ?,, fn {k, v} -> [object_key_encode(k), ?:, encode(v)] end), ?} ] end defp encode_jsonb_int5(<<"-", _a, _b, rest::binary>>), do: ["-", jsonb_int5_tail_to_decimal(rest)] defp encode_jsonb_int5(<<"+", _a, _b, rest::binary>>), do: jsonb_int5_tail_to_decimal(rest) defp encode_jsonb_int5(<<_a, _b, rest::binary>>), do: jsonb_int5_tail_to_decimal(rest) defp encode_jsonb_int5(_payload), do: "0" defp jsonb_int5_tail_to_decimal(tail) do tail |> :binary.bin_to_list() |> Enum.reduce_while({0, false}, fn byte, {value, overflow?} -> cond do not hex_digit?(byte) -> {:halt, {value, overflow?}} Bitwise.bsr(value, 60) != 0 -> {:cont, {value, true}} true -> {:cont, {value * 16 + hex_digit_value(byte), overflow?}} end end) |> case do {_value, true} -> "9.0e999" {value, false} -> Integer.to_string(value) end end defp hex_digit?(byte), do: hex_digit_value(byte) != :error defp hex_digit_value(byte) when byte in ?0..?9, do: byte - ?0 defp hex_digit_value(byte) when byte in ?a..?f, do: byte - ?a + 10 defp hex_digit_value(byte) when byte in ?A..?F, do: byte - ?A + 10 defp hex_digit_value(_byte), do: :error defp encode_jsonb_float5(<<"-", rest::binary>>), do: ["-", encode_jsonb_float5_unsigned(rest)] defp encode_jsonb_float5(token), do: encode_jsonb_float5_unsigned(token) defp encode_jsonb_float5_unsigned(<> = token), do: ["0", jsonb_float5_dot_tail(token)] defp encode_jsonb_float5_unsigned(token), do: jsonb_float5_dot_tail(token) defp jsonb_float5_dot_tail(token) do if String.ends_with?(token, ".") do [token, "0"] else token end end defp object_key_encode({:jsonb_escaped_text, 9, payload, _key}), do: [?", canonical_json5_payload(payload), ?"] defp object_key_encode({:jsonb_escaped_key, 9, payload, _key}), do: [?", canonical_json5_payload(payload), ?"] defp object_key_encode({:jsonb_escaped_key, _type, payload, _key}), do: [?", payload, ?"] defp object_key_encode(key), do: [?", key |> object_key_text() |> escape(), ?"] defp sqlite_jsonb_encode(:null), do: <<0x00>> defp sqlite_jsonb_encode(true), do: <<0x01>> defp sqlite_jsonb_encode(false), do: <<0x02>> defp sqlite_jsonb_encode(n) when is_integer(n), do: sqlite_jsonb_node(3, Integer.to_string(n)) defp sqlite_jsonb_encode(n) when is_float(n), do: sqlite_jsonb_node(5, Float.to_string(n)) defp sqlite_jsonb_encode({:real_literal, token}), do: sqlite_jsonb_node(5, token) defp sqlite_jsonb_encode({:jsonb_integer_literal, 4, payload, _value}), do: sqlite_jsonb_node(4, payload) defp sqlite_jsonb_encode({:jsonb_real_literal, type, token}), do: sqlite_jsonb_node(type, token) defp sqlite_jsonb_encode({:jsonb_escaped_text, type, payload, _value}), do: sqlite_jsonb_node(type, payload) defp sqlite_jsonb_encode({:jsonb_text, 10, value}), do: sqlite_jsonb_node(10, value) defp sqlite_jsonb_encode(s) when is_binary(s) do payload = escape(s) type = if payload == s, do: 7, else: 8 sqlite_jsonb_node(type, payload) end defp sqlite_jsonb_encode({:jsonb_path_key, key}), do: sqlite_jsonb_node(10, key) defp sqlite_jsonb_encode({:jsonb_escaped_key, type, payload, _key}), do: sqlite_jsonb_node(type, payload) defp sqlite_jsonb_encode({:array, items}) do payload = Enum.map(items, &sqlite_jsonb_encode/1) sqlite_jsonb_node(11, payload) end defp sqlite_jsonb_encode({:object, pairs}) do payload = Enum.map(pairs, fn {key, value} -> [sqlite_jsonb_encode(key), sqlite_jsonb_encode(value)] end) sqlite_jsonb_node(12, payload) end defp sqlite_jsonb_node(type, payload) do payload = IO.iodata_to_binary(payload) size = byte_size(payload) cond do size <= 11 -> <> size <= 0xFF -> <> size <= 0xFFFF -> <> size <= 0xFFFFFFFF -> <> true -> <> end end defp pretty_encode({:array, []}, _indent, _level), do: "[]" defp pretty_encode({:array, items}, indent, level) do child_prefix = pretty_indent(indent, level + 1) prefix = pretty_indent(indent, level) [ ?[, ?\n, Enum.map_intersperse(items, [?,, ?\n], fn item -> [child_prefix, pretty_encode(item, indent, level + 1)] end), ?\n, prefix, ?] ] end defp pretty_encode({:object, []}, _indent, _level), do: "{}" defp pretty_encode({:object, pairs}, indent, level) do child_prefix = pretty_indent(indent, level + 1) prefix = pretty_indent(indent, level) [ ?{, ?\n, Enum.map_intersperse(pairs, [?,, ?\n], fn {key, value} -> [ child_prefix, object_key_encode(key), ?:, ?\s, pretty_encode(value, indent, level + 1) ] end), ?\n, prefix, ?} ] end defp pretty_encode(jv, _indent, _level), do: encode(jv) defp pretty_indent(indent, level), do: String.duplicate(indent, level) defp escape(s) do for <>, into: "" do case c do ?" -> "\\\"" ?\\ -> "\\\\" ?\n -> "\\n" ?\r -> "\\r" ?\t -> "\\t" c when c < 0x20 -> "\\u" <> String.pad_leading(Integer.to_string(c, 16), 4, "0") c -> <> end end end defp canonical_json5_payload(<>) do ["\\u00", <

>, <

>, canonical_json5_payload(rest)] end defp canonical_json5_payload(<>), do: ["\\u0009", canonical_json5_payload(rest)] defp canonical_json5_payload(<>), do: ["\\u0000", canonical_json5_payload(rest)] defp canonical_json5_payload(<>), do: [?', canonical_json5_payload(rest)] defp canonical_json5_payload(<>), do: canonical_json5_payload(rest) defp canonical_json5_payload(<>), do: canonical_json5_payload(rest) defp canonical_json5_payload(<>), do: canonical_json5_payload(rest) defp canonical_json5_payload(<>) when sep in [0xA8, 0xA9], do: canonical_json5_payload(rest) defp canonical_json5_payload(<>), do: [c, canonical_json5_payload(rest)] defp canonical_json5_payload(<<>>), do: [] # -- JSON paths --------------------------------------------------------------- @doc ~S|Parses `$`, `$.key`, `$."quoted key"`, `$[2]`, `$[#-1]` paths.| @spec parse_path(binary()) :: {:ok, [path_step()]} | :error def parse_path(<>), do: path_steps(rest, []) def parse_path(_), do: :error defp path_steps("", acc), do: {:ok, Enum.reverse(acc)} defp path_steps(<>, acc) do case String.split(rest, "\"", parts: 2) do [key, rest] -> path_steps(rest, [{:key, key} | acc]) _ -> :error end end defp path_steps(<>, acc) do {key, rest} = take_path_key(rest, []) if key == "", do: :error, else: path_steps(rest, [{:key, key} | acc]) end # Bare `$[#]` is the one-past-last index: a no-op read, and the append slot # for json_insert/json_set (SQLite uses it as `# == array length`). defp path_steps(<>, acc) do path_steps(rest, [{:last_offset, 0} | acc]) end defp path_steps(<>, acc) do case Integer.parse(rest) do {offset, <>} when offset <= 0 -> path_steps(rest, [{:last_offset, offset} | acc]) _ -> :error end end defp path_steps(<>, acc) do case Integer.parse(rest) do {index, <>} when index >= 0 -> path_steps(rest, [{:index, index} | acc]) _ -> :error end end defp path_steps(_, _acc), do: :error defp take_path_key(<> = rest, acc) when c in [?., ?[], do: {IO.iodata_to_binary(acc), rest} defp take_path_key(<>, acc), do: take_path_key(rest, [acc, c]) defp take_path_key(<<>>, acc), do: {IO.iodata_to_binary(acc), ""} # -- navigation --------------------------------------------------------------- @spec get(t(), [path_step()]) :: {:ok, t()} | :missing def get(jv, []), do: {:ok, jv} def get({:object, pairs}, [{:key, key} | rest]) do case object_keyfind(pairs, key) do {_key, jv} -> get(jv, rest) nil -> :missing end end def get({:array, items}, [{:index, index} | rest]) do case Enum.at(items, index) do nil -> :missing jv -> get(jv, rest) end end def get({:array, items}, [{:last_offset, offset} | rest]) do case Enum.at(items, length(items) + offset) do nil -> :missing jv -> get(jv, rest) end end def get(_jv, _steps), do: :missing @doc """ Writes `value` at the path. `:insert` only creates missing leaves, `:replace` only overwrites existing ones, `:set` does both. Missing intermediate containers leave the document unchanged, as in SQLite. """ @spec write(t(), [path_step()], t(), :insert | :replace | :set) :: t() # The root always exists: replace/set overwrite it, insert is a no-op. def write(jv, [], _value, :insert), do: jv def write(_jv, [], value, _mode), do: value def write({:object, pairs}, [{:key, key}], value, mode) do case {object_keyfind(pairs, key), mode} do {nil, mode} when mode in [:insert, :set] -> {:object, pairs ++ [{{:jsonb_path_key, key}, value}]} {{existing_key, _v}, mode} when mode in [:replace, :set] -> {:object, object_keyreplace(pairs, existing_key, value)} _ -> {:object, pairs} end end def write({:array, items}, [{:index, index}], value, mode) do cond do index < length(items) and mode in [:replace, :set] -> {:array, List.replace_at(items, index, value)} index >= length(items) and mode in [:insert, :set] -> {:array, items ++ [value]} true -> {:array, items} end end def write({:array, items}, [{:last_offset, offset}], value, mode) do write({:array, items}, [{:index, length(items) + offset}], value, mode) end def write({:object, pairs}, [{:key, key} | rest], value, mode) do case object_keyfind(pairs, key) do {existing_key, jv} -> {:object, object_keyreplace(pairs, existing_key, write(jv, rest, value, mode))} nil -> {:object, pairs} end end def write({:array, items}, [{:index, index} | rest], value, mode) do case Enum.at(items, index) do nil -> {:array, items} jv -> {:array, List.replace_at(items, index, write(jv, rest, value, mode))} end end def write({:array, items}, [{:last_offset, offset} | rest], value, mode) do write({:array, items}, [{:index, length(items) + offset} | rest], value, mode) end def write(jv, _steps, _value, _mode), do: jv @spec remove(t(), [path_step()]) :: t() def remove({:object, pairs}, [{:key, key}]), do: {:object, object_keydelete(pairs, key)} def remove({:array, items}, [{:index, index}]) when index < length(items), do: {:array, List.delete_at(items, index)} def remove({:array, items}, [{:last_offset, offset}]), do: {:array, List.delete_at(items, length(items) + offset)} def remove({:object, pairs}, [{:key, key} | rest]) do case object_keyfind(pairs, key) do {existing_key, jv} -> {:object, object_keyreplace(pairs, existing_key, remove(jv, rest))} nil -> {:object, pairs} end end def remove({:array, items}, [{:index, index} | rest]) do case Enum.at(items, index) do nil -> {:array, items} jv -> {:array, List.replace_at(items, index, remove(jv, rest))} end end def remove({:array, items}, [{:last_offset, offset} | rest]), do: remove({:array, items}, [{:index, length(items) + offset} | rest]) def remove(jv, _steps), do: jv @doc """ RFC 7386 MergePatch, as `json_patch()` implements it: patch objects merge recursively (null members delete), anything else replaces the target. """ @spec merge_patch(t(), t()) :: t() def merge_patch(target, {:object, patch_pairs}) do base = case target do {:object, pairs} -> pairs _other -> [] end Enum.reduce(patch_pairs, {:object, base}, fn {key, value}, {:object, pairs} -> case {value, object_keyfind(pairs, object_key_text(key))} do {:null, _found} -> {:object, object_keydelete(pairs, object_key_text(key))} {value, nil} -> {:object, pairs ++ [{key, merge_patch(:null, value)}]} {value, {existing_key, existing}} -> {:object, object_keyreplace(pairs, existing_key, merge_patch(existing, value))} end end) end def merge_patch(_target, patch), do: patch defp object_keyfind(pairs, key) do Enum.find(pairs, fn {existing_key, _value} -> object_key_text(existing_key) == key end) end defp object_keyreplace(pairs, key, value) do Enum.map(pairs, fn {^key, _old_value} -> {key, value} other -> other end) end defp object_keydelete(pairs, key) do Enum.reject(pairs, fn {existing_key, _value} -> object_key_text(existing_key) == key end) end # -- SQL bridging -------------------------------------------------------------- @doc "The `json_type()` name for a JSON value." @spec type_name(t()) :: binary() def type_name(:null), do: "null" def type_name(true), do: "true" def type_name(false), do: "false" def type_name(:json_pos_inf), do: "real" def type_name(:json_neg_inf), do: "real" def type_name(:json_nan), do: "null" def type_name({:real_literal, _token}), do: "real" def type_name({:jsonb_integer_literal, 4, _payload, _value}), do: "integer" def type_name({:jsonb_real_literal, _type, _token}), do: "real" def type_name({:jsonb_text, 10, _value}), do: "text" def type_name({:jsonb_escaped_text, _type, _payload, _value}), do: "text" def type_name(n) when is_integer(n), do: "integer" def type_name(n) when is_float(n), do: "real" def type_name(s) when is_binary(s), do: "text" def type_name({:array, _}), do: "array" def type_name({:object, _}), do: "object" @doc "Converts a JSON value to its SQL representation (leaves stay scalar, containers render)." @spec to_sql(t()) :: ExSQL.Value.t() def to_sql(:null), do: nil def to_sql(true), do: 1 def to_sql(false), do: 0 def to_sql(:json_pos_inf), do: "9e999" def to_sql(:json_neg_inf), do: "-9e999" def to_sql(:json_nan), do: nil def to_sql({:real_literal, token}), do: real_literal_to_sql(token) def to_sql({:jsonb_integer_literal, 4, _payload, value}), do: value def to_sql({:jsonb_real_literal, 6, token}), do: token |> normalize_json5_number() |> real_literal_to_sql() def to_sql({:jsonb_real_literal, _type, token}), do: real_literal_to_sql(token) def to_sql({:jsonb_text, 10, value}), do: value def to_sql({:jsonb_escaped_text, _type, _payload, value}), do: value def to_sql(n) when is_number(n), do: n def to_sql(s) when is_binary(s), do: s def to_sql(container), do: render(container) defp real_literal_to_sql(token) do case Float.parse(token) do {value, ""} -> value :error -> real_literal_overflow_to_sql(token) _other -> real_literal_overflow_to_sql(token) end end defp real_literal_overflow_to_sql("-" <> _rest), do: "-9e999" defp real_literal_overflow_to_sql(_token), do: "9e999" end