defmodule ExToon.Encoder.Core do @moduledoc false # Main encoding pipeline. Converts a normalized value tree into a TOON document string. # # Design decisions: # - normalize/1 returns ordered [{String.t(), term()}] pairs for objects and [term()] for # arrays. The encoder distinguishes them by checking if the head element is a # {binary_key, _} tuple. # - encode_value/3 is the external entry point called by ExToon.encode/2. # - All internal helpers return String.t() directly (the iodata() wrapping is a thin # layer — the public API converts via IO.iodata_to_binary/1). # - Empty objects (empty pair-list) encode to "" (no output); empty arrays encode to # an inline form with zero length. alias ExToon.StringUtils alias ExToon.Encoder.{Normalize, Primitives, Replacer, Folding} @type opts :: keyword() # --------------------------------------------------------------------------- # Public entry point # --------------------------------------------------------------------------- @spec encode_value(term(), opts(), [String.t() | integer()]) :: iodata() def encode_value(input, opts, path) do replacer = Keyword.get(opts, :replacer) # Apply replacer to the root value (key is "" for root) before normalization # so the replacer receives the original Elixir term. replaced_input = case Replacer.apply(input, replacer, "", path) do {:keep, v} -> v # Skipping the root is a no-op — there's nothing to omit it from :skip -> input end # For plain maps: use lazy normalization so the replacer receives the original # Elixir values (not normalized pairs) and uses child_path (path including current key). case replaced_input do map when is_map(map) and not is_struct(map) -> if map_size(map) == 0 do "" else sorted_pairs = map |> Enum.sort_by(fn {k, _} -> raw_key_to_string(k) end) |> Enum.map(fn {k, v} -> {raw_key_to_string(k), v} end) encode_object_with_original_values(sorted_pairs, opts, path, 0) end _ -> final = Normalize.normalize(replaced_input) do_encode(final, opts, path, 0) end end # Convert atom or binary key to string (without recursively normalizing the value). defp raw_key_to_string(k) when is_atom(k), do: Atom.to_string(k) defp raw_key_to_string(k) when is_binary(k), do: k # Encode an object from a list of {string_key, original_elixir_value} pairs. # Unlike encode_object/4 (which expects normalized pairs), this function: # - passes the original (pre-normalization) value to the replacer # - uses child_path (path ++ [k]) for the replacer call # - detects key-folding collisions before folding # - normalizes each value AFTER the replacer has run defp encode_object_with_original_values(pairs, opts, path, depth) do indent_size = Keyword.get(opts, :indent, 2) key_folding = Keyword.get(opts, :key_folding, :off) flatten_depth = Keyword.get(opts, :flatten_depth, :infinity) replacer = Keyword.get(opts, :replacer) doc_delim = resolve_delimiter(Keyword.get(opts, :delimiter, :comma)) # Build the set of ALL original sibling keys for collision detection. all_sibling_keys = MapSet.new(pairs, fn {k, _} -> k end) pairs |> Enum.flat_map(fn {k, v_original} -> child_path = path ++ [k] # Normalize the value to allow Folding.fold (which requires normalized pairs lists). v_for_fold = Normalize.normalize(v_original) {folded_key, folded_value_norm} = Folding.fold(k, v_for_fold, key_folding, flatten_depth, 0) # Collision detection: skip folding if the resulting key conflicts with a sibling. collision? = folded_key != k and MapSet.member?(all_sibling_keys, folded_key) {use_key, use_opts, precomputed_norm} = if collision? do # Disable key folding in the subtree to prevent partial folding that # could recreate the same colliding path at a nested level. {k, Keyword.put(opts, :key_folding, :off), v_for_fold} else {folded_key, opts, folded_value_norm} end # Apply replacer with the ORIGINAL value and the CHILD PATH. case Replacer.apply(v_original, replacer, k, child_path) do :skip -> [] {:keep, final_v} -> # When the replacer returns the same object (identity), reuse the pre-computed # normalized/folded value to avoid re-normalizing the whole subtree. final_norm = if final_v === v_original do precomputed_norm else Normalize.normalize(final_v) end line = encode_kv(use_key, final_norm, use_opts, child_path, depth, doc_delim, indent_size) if line == "" do [] else [line] end end end) |> Enum.join("\n") end # encode_lines/2: returns a Stream of line binaries. # NOTE: unlike encode/2, encoding failures raise EncodeError during enumeration. @spec encode_lines(term(), opts()) :: Enumerable.t() def encode_lines(input, opts) do Stream.resource( fn -> result = encode_value(input, opts, []) binary = IO.iodata_to_binary(result) # An empty binary (e.g. empty map) must produce zero lines, not [""]. if binary == "", do: [], else: String.split(binary, "\n") end, fn [] -> {:halt, []} [line | rest] -> {[line], rest} end, fn _ -> :ok end ) end # --------------------------------------------------------------------------- # Core dispatch # --------------------------------------------------------------------------- # Primitives are delegated directly to Primitives module with the document delimiter defp do_encode(nil, _opts, _path, _depth), do: "null" defp do_encode(true, _opts, _path, _depth), do: "true" defp do_encode(false, _opts, _path, _depth), do: "false" defp do_encode(n, _opts, _path, _depth) when is_integer(n), do: Integer.to_string(n) defp do_encode(f, opts, _path, _depth) when is_float(f) do doc_delim = resolve_delimiter(Keyword.get(opts, :delimiter, :comma)) Primitives.encode_primitive(f, doc_delim) end defp do_encode(s, opts, _path, _depth) when is_binary(s) do doc_delim = resolve_delimiter(Keyword.get(opts, :delimiter, :comma)) Primitives.encode_primitive(s, doc_delim) end # Empty object sentinel — produced by normalize/1 for empty plain maps. # Encodes to "" (no output, invisible in the document). defp do_encode(:empty_object, _opts, _path, _depth), do: "" defp do_encode(pairs, opts, path, depth) when is_list(pairs) do case pairs do [] -> # Empty list (from Encodable.to_toon returning [] or normalize of empty keyword list). # This is treated as an empty ARRAY (encoded as [0]:). # Empty MAPS go through :empty_object sentinel instead — see normalize/1. doc_delim = resolve_delimiter(Keyword.get(opts, :delimiter, :comma)) prefix = String.duplicate(" ", depth * Keyword.get(opts, :indent, 2)) delim_sym = delimiter_symbol(doc_delim) "#{prefix}[0#{delim_sym}]:" [{k, _} | _] when is_binary(k) -> # Object: ordered key-value pairs encode_object(pairs, opts, path, depth) _ -> # Array active_delim = resolve_delimiter(Keyword.get(opts, :delimiter, :comma)) encode_array_standalone(pairs, opts, path, depth, active_delim) end end # --------------------------------------------------------------------------- # Object encoding # --------------------------------------------------------------------------- defp encode_object(pairs, opts, path, depth) do indent_size = Keyword.get(opts, :indent, 2) key_folding = Keyword.get(opts, :key_folding, :off) flatten_depth = Keyword.get(opts, :flatten_depth, :infinity) replacer = Keyword.get(opts, :replacer) doc_delim = resolve_delimiter(Keyword.get(opts, :delimiter, :comma)) # Build sibling key set for collision detection (same as in encode_object_with_original_values). all_sibling_keys = MapSet.new(pairs, fn {k, _} -> k end) pairs |> Enum.flat_map(fn {k, v} -> {folded_key, folded_value} = Folding.fold(k, v, key_folding, flatten_depth, depth) child_path = path ++ [k] # Collision: if folding produced a key that already exists as a sibling, skip fold # and disable folding for the subtree to prevent partial-fold collisions at deeper levels. {use_key, use_value, use_opts} = if folded_key != k and MapSet.member?(all_sibling_keys, folded_key) do {k, v, Keyword.put(opts, :key_folding, :off)} else {folded_key, folded_value, opts} end case Replacer.apply(use_value, replacer, k, path) do :skip -> [] {:keep, final_v} -> normalized_v = Normalize.normalize(final_v) line = encode_kv(use_key, normalized_v, use_opts, child_path, depth, doc_delim, indent_size) if line == "" do [] else [line] end end end) |> Enum.join("\n") end # --------------------------------------------------------------------------- # Key-value line encoding # --------------------------------------------------------------------------- defp encode_kv(key, value, opts, path, depth, doc_delim, indent_size) do prefix = String.duplicate(" ", depth * indent_size) encoded_key = encode_key(key) case value do nil -> "#{prefix}#{encoded_key}: null" true -> "#{prefix}#{encoded_key}: true" false -> "#{prefix}#{encoded_key}: false" n when is_integer(n) -> "#{prefix}#{encoded_key}: #{n}" f when is_float(f) -> "#{prefix}#{encoded_key}: #{Primitives.encode_primitive(f, doc_delim)}" s when is_binary(s) -> encoded_val = Primitives.encode_primitive(s, doc_delim) "#{prefix}#{encoded_key}: #{encoded_val}" :empty_object -> # Empty nested object — emit bare key with no value "#{prefix}#{encoded_key}:" [] -> # Empty array — use zero-length inline form delim_sym = delimiter_symbol(doc_delim) "#{prefix}#{encoded_key}[0#{delim_sym}]:" [{_, _} | _] = sub_pairs -> # Nested object: recurse with increased depth child_block = encode_object(sub_pairs, opts, path, depth + 1) if child_block == "" do "#{prefix}#{encoded_key}:" else "#{prefix}#{encoded_key}:\n#{child_block}" end list when is_list(list) -> encode_array_kv(encoded_key, list, opts, path, depth, doc_delim, indent_size, prefix) end end # --------------------------------------------------------------------------- # Array encoding (as a key-value entry) # --------------------------------------------------------------------------- defp encode_array_kv(encoded_key, list, opts, path, depth, doc_delim, indent_size, prefix) do replacer = Keyword.get(opts, :replacer) delim_sym = delimiter_symbol(doc_delim) # Apply replacer to each array element before deciding encoding format. filtered_list = list |> Enum.with_index() |> Enum.flat_map(fn {item, idx} -> key = Integer.to_string(idx) item_path = path ++ [idx] case Replacer.apply(item, replacer, key, item_path) do :skip -> [] {:keep, final_item} -> [Normalize.normalize(final_item)] end end) n = length(filtered_list) cond do all_primitives?(filtered_list) -> # Primitive array: key[N]: v1,v2,v3 values = Enum.map(filtered_list, &Primitives.encode_primitive(&1, doc_delim)) "#{prefix}#{encoded_key}[#{n}#{delim_sym}]: #{Enum.join(values, <>)}" # Tabular format is skipped when a replacer is present because tabular # encoding bypasses per-field replacer application. Fall through to the # expanded list format so each object's fields are individually processed. replacer == nil and tabular?(filtered_list) -> encode_tabular_kv( encoded_key, filtered_list, doc_delim, indent_size, prefix, depth, n, delim_sym ) true -> # Expanded list: key[N]:\n - item (replacer already applied above) encode_expanded_kv_normalized( encoded_key, filtered_list, opts, path, depth, doc_delim, indent_size, prefix, n, delim_sym ) end end defp encode_tabular_kv( encoded_key, list, doc_delim, indent_size, prefix, depth, n, delim_sym ) do # Extract field order from the first row [{_, _} | _] = first = hd(list) fields = Enum.map(first, fn {k, _} -> k end) encoded_fields = Enum.map(fields, &encode_key/1) fields_str = Enum.join(encoded_fields, <>) row_prefix = String.duplicate(" ", (depth + 1) * indent_size) rows = Enum.map(list, fn pairs -> values = Enum.map(fields, fn f -> v = find_value(pairs, f) Primitives.encode_primitive(v, doc_delim) end) "#{row_prefix}#{Enum.join(values, <>)}" end) "#{prefix}#{encoded_key}[#{n}#{delim_sym}]{#{fields_str}}:\n#{Enum.join(rows, "\n")}" end defp encode_expanded_kv( encoded_key, list, opts, path, depth, doc_delim, indent_size, prefix, n, delim_sym ) do # List is NOT yet replacer-filtered — apply replacer and normalize here. replacer = Keyword.get(opts, :replacer) item_prefix = String.duplicate(" ", (depth + 1) * indent_size) items = list |> Enum.with_index() |> Enum.flat_map(fn {item, idx} -> item_path = path ++ [idx] key = Integer.to_string(idx) case Replacer.apply(item, replacer, key, item_path) do :skip -> [] {:keep, final_item} -> normalized_item = Normalize.normalize(final_item) [encode_list_item(normalized_item, opts, item_path, depth + 1, doc_delim, indent_size, item_prefix)] end end) actual_n = length(items) "#{prefix}#{encoded_key}[#{actual_n}#{delim_sym}]:\n#{Enum.join(items, "\n")}" end # Like encode_expanded_kv but for already-normalized, already-filtered lists # (called from encode_array_kv which has already applied the replacer). defp encode_expanded_kv_normalized( encoded_key, filtered_list, opts, path, depth, doc_delim, indent_size, prefix, n, delim_sym ) do item_prefix = String.duplicate(" ", (depth + 1) * indent_size) items = filtered_list |> Enum.with_index() |> Enum.map(fn {item, idx} -> item_path = path ++ [idx] encode_list_item(item, opts, item_path, depth + 1, doc_delim, indent_size, item_prefix) end) "#{prefix}#{encoded_key}[#{n}#{delim_sym}]:\n#{Enum.join(items, "\n")}" end # --------------------------------------------------------------------------- # Array encoding (standalone, no key context) # --------------------------------------------------------------------------- defp encode_array_standalone(list, opts, path, depth, doc_delim) do indent_size = Keyword.get(opts, :indent, 2) replacer = Keyword.get(opts, :replacer) delim_sym = delimiter_symbol(doc_delim) prefix = String.duplicate(" ", depth * indent_size) # Apply replacer to each element (using index as string key), filtering :skip items. # NOTE: we re-normalize after replacer so the final list is always normalized. filtered_list = list |> Enum.with_index() |> Enum.flat_map(fn {item, idx} -> key = Integer.to_string(idx) item_path = path ++ [idx] case Replacer.apply(item, replacer, key, item_path) do :skip -> [] {:keep, final_item} -> [Normalize.normalize(final_item)] end end) n = length(filtered_list) cond do all_primitives?(filtered_list) -> values = Enum.map(filtered_list, &Primitives.encode_primitive(&1, doc_delim)) "#{prefix}[#{n}#{delim_sym}]: #{Enum.join(values, <>)}" tabular?(filtered_list) -> [{_, _} | _] = first = hd(filtered_list) fields = Enum.map(first, fn {k, _} -> k end) encoded_fields = Enum.map(fields, &encode_key/1) fields_str = Enum.join(encoded_fields, <>) row_prefix = String.duplicate(" ", (depth + 1) * indent_size) rows = Enum.map(filtered_list, fn pairs -> values = Enum.map(fields, fn f -> v = find_value(pairs, f) Primitives.encode_primitive(v, doc_delim) end) "#{row_prefix}#{Enum.join(values, <>)}" end) "#{prefix}[#{n}#{delim_sym}]{#{fields_str}}:\n#{Enum.join(rows, "\n")}" true -> item_prefix = String.duplicate(" ", (depth + 1) * indent_size) items = filtered_list |> Enum.with_index() |> Enum.map(fn {item, idx} -> encode_list_item( item, opts, path ++ [idx], depth + 1, doc_delim, indent_size, item_prefix ) end) "#{prefix}[#{n}#{delim_sym}]:\n#{Enum.join(items, "\n")}" end end # --------------------------------------------------------------------------- # List item encoding (prefixed with "- ") # --------------------------------------------------------------------------- defp encode_list_item(item, opts, path, depth, doc_delim, indent_size, item_prefix) do case item do nil -> "#{item_prefix}- null" true -> "#{item_prefix}- true" false -> "#{item_prefix}- false" n when is_integer(n) -> "#{item_prefix}- #{n}" f when is_float(f) -> "#{item_prefix}- #{Primitives.encode_primitive(f, doc_delim)}" s when is_binary(s) -> "#{item_prefix}- #{Primitives.encode_primitive(s, doc_delim)}" :empty_object -> # Empty object as list item — bare hyphen "#{item_prefix}-" [] -> delim_sym = delimiter_symbol(doc_delim) "#{item_prefix}- [0#{delim_sym}]:" [{_, _} | _] = pairs -> # Object as list item: first field on the hyphen line (YAML-style inline). # Remaining fields are at depth+1 with normal indentation. encode_object_as_list_item(pairs, opts, path, depth, doc_delim, indent_size, item_prefix) list when is_list(list) -> sub_n = length(list) delim_sym = delimiter_symbol(doc_delim) if all_primitives?(list) do values = Enum.map(list, &Primitives.encode_primitive(&1, doc_delim)) "#{item_prefix}- [#{sub_n}#{delim_sym}]: #{Enum.join(values, <>)}" else sub_prefix = String.duplicate(" ", (depth + 1) * indent_size) sub_items = list |> Enum.with_index() |> Enum.map(fn {sub, idx} -> encode_list_item( sub, opts, path ++ [idx], depth + 1, doc_delim, indent_size, sub_prefix ) end) "#{item_prefix}- [#{sub_n}#{delim_sym}]:\n#{Enum.join(sub_items, "\n")}" end end end # --------------------------------------------------------------------------- # List-item object encoding (YAML-style: first field on the hyphen line) # --------------------------------------------------------------------------- # Encodes an object as a list item. The first rendered field is placed on the # same line as the "- " prefix; remaining fields are at depth+1 with their # normal indentation. This produces the canonical TOON list-item-object format: # # - firstKey: value # secondKey: value # # or, when the first field is an array with a block body: # # - arr[N]{f}: # row1 # sibling: val # defp encode_object_as_list_item(pairs, opts, path, depth, doc_delim, indent_size, item_prefix) do key_folding = Keyword.get(opts, :key_folding, :off) flatten_depth = Keyword.get(opts, :flatten_depth, :infinity) replacer = Keyword.get(opts, :replacer) # child_depth is the depth at which fields of this object are written child_depth = depth + 1 child_prefix = String.duplicate(" ", child_depth * indent_size) # Render all visible fields (same logic as encode_object). # Replacer receives `path` (the path to this object) and `k` (the key). field_lines = pairs |> Enum.flat_map(fn {k, v} -> {folded_key, folded_value} = Folding.fold(k, v, key_folding, flatten_depth, child_depth) child_path = path ++ [k] case Replacer.apply(folded_value, replacer, k, path) do :skip -> [] {:keep, final_v} -> normalized_v = Normalize.normalize(final_v) line = encode_kv(folded_key, normalized_v, opts, child_path, child_depth, doc_delim, indent_size) if line == "" do [] else [line] end end end) case field_lines do [] -> # All fields were skipped — emit a bare hyphen "#{item_prefix}-" [only_line] -> # Single field: replace its child_prefix with "item_prefix- " inline = String.replace_prefix(only_line, child_prefix, "#{item_prefix}- ") inline [first_line | rest_lines] -> # Multiple fields: first on hyphen line, rest unchanged inline_first = String.replace_prefix(first_line, child_prefix, "#{item_prefix}- ") ([inline_first] ++ rest_lines) |> Enum.join("\n") end end # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- defp encode_key(key) do if StringUtils.key_needs_quoting?(key) do "\"#{StringUtils.escape(key)}\"" else key end end # Returns true when every element is a TOON primitive (nil, boolean, number, string) defp all_primitives?(list) do Enum.all?(list, fn v when is_binary(v) or is_number(v) or is_boolean(v) or is_nil(v) -> true _ -> false end) end # Tabular: all elements are objects (ordered pairs) with the same keys, all values primitive. # Key order is taken from the first element; rows must have the same key set (order-insensitive). defp tabular?([]), do: false defp tabular?(list) do case hd(list) do [{_, _} | _] = first -> first_keys = Enum.map(first, fn {k, _} -> k end) |> Enum.sort() Enum.all?(list, fn [{_, _} | _] = pairs -> row_keys = Enum.map(pairs, fn {k, _} -> k end) |> Enum.sort() row_keys == first_keys and Enum.all?(pairs, fn {_, v} -> is_binary(v) or is_number(v) or is_boolean(v) or is_nil(v) end) _ -> false end) _ -> false end end # Lookup a value in an ordered pairs list by key (used for tabular row rendering) defp find_value(pairs, key) do case List.keyfind(pairs, key, 0) do {_, v} -> v nil -> nil end end defp resolve_delimiter(:comma), do: ?, defp resolve_delimiter(:tab), do: ?\t defp resolve_delimiter(:pipe), do: ?| defp resolve_delimiter(c) when is_integer(c), do: c # Returns the suffix appended to array length in headers: # comma (default) is omitted; tab and pipe are written explicitly. defp delimiter_symbol(?,), do: "" defp delimiter_symbol(?\t), do: "\t" defp delimiter_symbol(?|), do: "|" end