defmodule ExToon.Decoder.Parser do @moduledoc false # Structural parser for TOON documents. # # Consumes a list of %Scanner{} structs and produces a flat list of stream events: # %{type: :start_object} # %{type: :end_object} # %{type: :start_array, length: n} # %{type: :end_array} # %{type: :key, key: str} # %{type: :primitive, value: val} # # Design: # - Recursive descent over the line list, tracking expected depth. # - Root form detection per §5: first depth-0 line determines array vs object vs primitive. # - Array forms: inline primitive, tabular rows, expanded list items. # - Object forms: key:value pairs at a given depth. alias ExToon.Decoder.Scanner alias ExToon.{DecodeError, LiteralUtils, StringUtils} @type event :: map() # Parse a list of Scanner structs into stream events. # Returns {:ok, [event()]} | {:error, term()} @spec parse([Scanner.t()], pos_integer(), boolean()) :: {:ok, [event()]} | {:error, term()} def parse(parsed_lines, indent_size, strict) do non_blank = Enum.reject(parsed_lines, fn l -> String.trim(l.raw) == "" end) # Strict-mode: blank lines inside arrays (depth > 0 context) are invalid. # We detect them by looking for blank lines that occur between two non-blank # lines where at least one surrounding non-blank line is at depth > 0. if strict do validate_no_blank_lines_in_arrays(parsed_lines) end # Strict-mode: multiple depth-0 non-kv lines with no structure = ambiguous root. if strict do depth0_unstructured = Enum.filter(non_blank, fn l -> l.depth == 0 and not match?({:array_header, _, _, _, _, _}, Scanner.parse_header(l.content)) and not is_kv_line?(l) end) if length(depth0_unstructured) > 1 do first_extra = Enum.at(depth0_unstructured, 1) throw({:decode_error, %DecodeError{ line: first_extra.line_number, reason: :multiple_root_primitives, message: "multiple root-level primitives not allowed in strict mode at line #{first_extra.line_number}" }}) end end case detect_root_form(non_blank) do :empty -> {:ok, [%{type: :start_object}, %{type: :end_object}]} {:root_primitive, line} -> {:ok, [%{type: :primitive, value: parse_primitive_token(line.content)}]} {:root_array, _line} -> parse_root_array(non_blank, indent_size, strict) :root_object -> parse_root_object(non_blank, indent_size, strict) end end # --------------------------------------------------------------------------- # Root form detection (§5) # --------------------------------------------------------------------------- defp detect_root_form([]), do: :empty defp detect_root_form(lines) do first = hd(lines) depth0_lines = Enum.filter(lines, &(&1.depth == 0)) case Scanner.parse_header(first.content) do {:array_header, nil, _, _, _, _} -> # Keyless array header at root depth → root array {:root_array, first} {:array_header, _key, _, _, _, _} -> # Keyed array header at root → this is an object with an array-valued field :root_object _ -> if length(depth0_lines) == 1 and not is_kv_line?(first) and not String.starts_with?(first.content, "- ") do {:root_primitive, first} else :root_object end end end defp is_kv_line?(line) do Scanner.parse_kv(line.content) != :not_kv end # --------------------------------------------------------------------------- # Root array # --------------------------------------------------------------------------- defp parse_root_array([header_line | rest], indent_size, strict) do {:array_header, _nil_key, length, delimiter, fields, inline} = Scanner.parse_header(header_line.content) {body_events, _remaining} = parse_array_body(rest, 1, length, delimiter, fields, inline, indent_size, strict) {:ok, [%{type: :start_array, length: length}] ++ body_events ++ [%{type: :end_array}]} end # --------------------------------------------------------------------------- # Root object # --------------------------------------------------------------------------- defp parse_root_object(lines, indent_size, strict) do {events, _remaining} = parse_object_body(lines, 0, indent_size, strict) {:ok, [%{type: :start_object}] ++ events ++ [%{type: :end_object}]} end # --------------------------------------------------------------------------- # Object body: parse key-value pairs at the given depth # --------------------------------------------------------------------------- defp parse_object_body([], _depth, _indent_size, _strict), do: {[], []} defp parse_object_body([line | rest] = lines, depth, indent_size, strict) do if line.depth < depth do # Depth decreased — end of this object scope {[], lines} else case Scanner.parse_header(line.content) do {:array_header, key, length, delimiter, fields, inline} when key != nil -> # Array-valued field key_event = %{type: :key, key: key} start_event = %{type: :start_array, length: length} {body_events, remaining} = parse_array_body(rest, depth + 1, length, delimiter, fields, inline, indent_size, strict) end_event = %{type: :end_array} {more_events, final_remaining} = parse_object_body(remaining, depth, indent_size, strict) events = [key_event, start_event] ++ body_events ++ [end_event] ++ more_events {events, final_remaining} _ -> case Scanner.parse_kv(line.content) do {:kv, key, value_str, was_quoted} -> key_event = if was_quoted, do: %{type: :key, key: key, literal: true}, else: %{type: :key, key: key} {value_events, remaining} = parse_value(value_str, rest, depth, indent_size, strict) {more_events, final_remaining} = parse_object_body(remaining, depth, indent_size, strict) {[key_event] ++ value_events ++ more_events, final_remaining} :not_kv -> # In strict mode, an unrecognized line AT the exact object depth is an error. # Lines at deeper depth are silently skipped (they may belong to nested contexts # that the current pass doesn't own). In lenient mode, always skip. if strict and depth > 0 and line.depth == depth do throw({:decode_error, %DecodeError{ line: line.line_number, reason: :syntax_error, message: "expected key:value or array header at line #{line.line_number}, got: #{inspect(line.content)}" }}) else parse_object_body(rest, depth, indent_size, strict) end end end end end # --------------------------------------------------------------------------- # Value parsing (after "key:") # --------------------------------------------------------------------------- # Empty after colon — nested object or nested array defp parse_value("", rest, depth, indent_size, strict) do case rest do [] -> {[%{type: :start_object}, %{type: :end_object}], []} [next | _] -> if next.depth > depth do # Peek to decide: if next line is an array header without key, it's a nested array. # Otherwise it's a nested object. case Scanner.parse_header(next.content) do {:array_header, nil, length, delimiter, fields, inline} -> # Anonymous array header at child depth — nested array value [_header | after_header] = rest {body_events, remaining} = parse_array_body( after_header, depth + 2, length, delimiter, fields, inline, indent_size, strict ) {[%{type: :start_array, length: length}] ++ body_events ++ [%{type: :end_array}], remaining} _ -> # Nested object {child_events, remaining} = parse_object_body(rest, depth + 1, indent_size, strict) {[%{type: :start_object}] ++ child_events ++ [%{type: :end_object}], remaining} end else {[%{type: :start_object}, %{type: :end_object}], rest} end end end # Inline primitive value after colon defp parse_value(value_str, rest, _depth, _indent_size, _strict) do value = parse_primitive_token(value_str) {[%{type: :primitive, value: value}], rest} end # --------------------------------------------------------------------------- # Array body dispatch # --------------------------------------------------------------------------- defp parse_array_body(rest, _depth, length, delimiter, nil, inline, _indent_size, strict) when inline != nil do # Inline primitive array — all values on the header line itself. # The first clause (fields == nil, inline != nil) handles cases where the # header had no field spec and all values are on the same line. values = parse_inline_values(inline, delimiter) if strict and length(values) != length do throw({:decode_error, %DecodeError{ reason: :length_mismatch, message: "array declared length #{length} but found #{length(values)} inline values" }}) end events = Enum.map(values, &%{type: :primitive, value: &1}) {events, rest} end defp parse_array_body(rest, depth, length, delimiter, fields, inline, indent_size, strict) do cond do inline != nil -> values = parse_inline_values(inline, delimiter) if strict and length(values) != length do throw({:decode_error, %DecodeError{ reason: :length_mismatch, message: "array declared length #{length} but found #{length(values)} inline values" }}) end events = Enum.map(values, &%{type: :primitive, value: &1}) {events, rest} fields != nil -> parse_tabular_rows(rest, depth, length, delimiter, fields, indent_size, strict) true -> parse_list_items(rest, depth, length, delimiter, indent_size, strict) end end # --------------------------------------------------------------------------- # Inline values: split by delimiter, parse each as primitive # --------------------------------------------------------------------------- defp parse_inline_values(str, delimiter) do str |> Scanner.split_by_delimiter(delimiter) |> Enum.map(fn token -> token |> String.trim() |> parse_primitive_token() end) end # --------------------------------------------------------------------------- # Tabular rows # --------------------------------------------------------------------------- defp parse_tabular_rows([], _depth, _length, _delimiter, _fields, _indent_size, _strict) do {[], []} end defp parse_tabular_rows( [line | rest] = lines, depth, length, delimiter, fields, indent_size, strict ) do # Blank line inside tabular array: strict mode throws, lenient mode skips. if String.trim(line.raw) == "" do if strict do throw({:decode_error, %DecodeError{ line: line.line_number, reason: :blank_line_in_array, message: "blank line inside tabular array at line #{line.line_number}" }}) else parse_tabular_rows(rest, depth, length, delimiter, fields, indent_size, strict) end else if line.depth != depth do # If we're in strict mode and there are MORE rows at this depth than declared, # we've already consumed what was expected; this line at a different depth ends the array. {[], lines} else # Strict mode: if we have already consumed the declared number of rows (length == 0), # but there is still a valid data row at the correct depth, that is a row-count mismatch. if strict and length <= 0 do throw({:decode_error, %DecodeError{ line: line.line_number, reason: :row_count_mismatch, message: "tabular array has more rows than the declared count at line #{line.line_number}" }}) end raw_values = line.content |> Scanner.split_by_delimiter(delimiter) |> Enum.map(&String.trim/1) values = Enum.map(raw_values, &parse_primitive_token/1) # Strict mode: validate that each row has exactly the right number of fields. if strict and length(values) != length(fields) do throw({:decode_error, %DecodeError{ line: line.line_number, reason: :field_count_mismatch, message: "tabular row has #{length(values)} values but header declares #{length(fields)} fields at line #{line.line_number}" }}) end field_events = fields |> Enum.zip(values) |> Enum.flat_map(fn {field, value} -> [%{type: :key, key: field}, %{type: :primitive, value: value}] end) row_events = [%{type: :start_object}] ++ field_events ++ [%{type: :end_object}] {more_events, remaining} = parse_tabular_rows(rest, depth, length - 1, delimiter, fields, indent_size, strict) {row_events ++ more_events, remaining} end end end # --------------------------------------------------------------------------- # Expanded list items # --------------------------------------------------------------------------- defp parse_list_items(lines, depth, expected_length, delimiter, indent_size, strict) do {events, remaining} = collect_items(lines, depth, delimiter, indent_size, strict, []) # Count items by counting :start_object/:primitive events at the item level # (before any nested object/array content). We approximate by counting parse results # that would come from the item-level loop. Instead, count by tracking in collect. # We pass expected_length to collect_items for strict validation. if strict do actual = count_items(events) if actual != expected_length do throw({:decode_error, %DecodeError{ reason: :length_mismatch, message: "list array declared length #{expected_length} but found #{actual} items" }}) end end {events, remaining} end # Count top-level items in the flattened event list. # Items are: start_object at depth 0, start_array at depth 0, or bare primitives at depth 0. defp count_items(events) do {count, _depth} = Enum.reduce(events, {0, 0}, fn event, {count, depth} -> case event do %{type: :start_object} when depth == 0 -> {count + 1, depth + 1} %{type: :start_object} -> {count, depth + 1} %{type: :end_object} -> {count, depth - 1} %{type: :start_array} when depth == 0 -> {count + 1, depth + 1} %{type: :start_array} -> {count, depth + 1} %{type: :end_array} -> {count, depth - 1} %{type: :primitive} when depth == 0 -> {count + 1, depth} _ -> {count, depth} end end) count end defp collect_items([], _depth, _delimiter, _indent_size, _strict, acc) do {acc |> Enum.reverse() |> List.flatten(), []} end defp collect_items([line | rest] = lines, depth, delimiter, indent_size, strict, acc) do # Blank line (or whitespace-only line) inside an expanded list array. if String.trim(line.raw) == "" do if strict and acc != [] do # Only error on blank lines that appear INSIDE the array (after at least one item). throw({:decode_error, %DecodeError{ line: line.line_number, reason: :blank_line_in_array, message: "blank line inside list array at line #{line.line_number}" }}) else collect_items(rest, depth, delimiter, indent_size, strict, acc) end else if line.depth < depth do {acc |> Enum.reverse() |> List.flatten(), lines} else content = line.content if String.starts_with?(content, "- ") or content == "-" do item_content = if content == "-", do: "", else: String.slice(content, 2, String.length(content)) {item_events, remaining} = parse_list_item_content(item_content, rest, depth, delimiter, indent_size, strict) collect_items(remaining, depth, delimiter, indent_size, strict, [item_events | acc]) else # Not a list item — stop collecting {acc |> Enum.reverse() |> List.flatten(), lines} end end end end # --------------------------------------------------------------------------- # List item content dispatch # --------------------------------------------------------------------------- # Bare hyphen — check for nested object at depth+1 defp parse_list_item_content("", rest, depth, _delimiter, indent_size, strict) do case rest do [] -> {[%{type: :start_object}, %{type: :end_object}], []} [next | _] -> if next.depth > depth do {child_events, remaining} = parse_object_body(rest, depth + 1, indent_size, strict) {[%{type: :start_object}] ++ child_events ++ [%{type: :end_object}], remaining} else {[%{type: :start_object}, %{type: :end_object}], rest} end end end defp parse_list_item_content(item_content, rest, depth, _delimiter, indent_size, strict) do # Try array header first (for "- [N]:" or "- key[N]:" patterns) case Scanner.parse_header(item_content) do {:array_header, key, length, delim, fields, inline} when key != nil -> # Object with an array field on the hyphen line: "- key[N]{...}:" # Per spec: tabular/list body is at depth+2 (one for the item scope, one for the # array body level), while sibling object fields are at depth+1. key_event = %{type: :key, key: key} start_event = %{type: :start_array, length: length} {array_body, after_array} = parse_array_body(rest, depth + 2, length, delim, fields, inline, indent_size, strict) {more_obj, final_remaining} = parse_object_body(after_array, depth + 1, indent_size, strict) events = [%{type: :start_object}, key_event, start_event] ++ array_body ++ [%{type: :end_array}] ++ more_obj ++ [%{type: :end_object}] {events, final_remaining} {:array_header, nil, length, delim, fields, inline} -> # Root-style array as a list item: "- [N]:" start_event = %{type: :start_array, length: length} {array_body, remaining} = parse_array_body(rest, depth + 1, length, delim, fields, inline, indent_size, strict) {[start_event] ++ array_body ++ [%{type: :end_array}], remaining} :not_header -> # Try key:value (object starting on the hyphen line) case Scanner.parse_kv(item_content) do {:kv, key, value_str, was_quoted} -> key_event = if was_quoted, do: %{type: :key, key: key, literal: true}, else: %{type: :key, key: key} {value_events, after_value} = parse_value(value_str, rest, depth, indent_size, strict) {more_events, remaining} = parse_object_body(after_value, depth + 1, indent_size, strict) events = [%{type: :start_object}, key_event] ++ value_events ++ more_events ++ [%{type: :end_object}] {events, remaining} :not_kv -> # Plain primitive list item value = parse_primitive_token(String.trim(item_content)) {[%{type: :primitive, value: value}], rest} end end end # --------------------------------------------------------------------------- # Primitive token parsing # --------------------------------------------------------------------------- # Quoted string — unescape inner content defp parse_primitive_token("\"" <> _ = quoted) do len = String.length(quoted) if len >= 2 and String.last(quoted) == "\"" do inner = String.slice(quoted, 1, len - 2) case StringUtils.unescape(inner) do {:ok, s} -> s # Return raw on bad escape — strict mode catches this separately {:error, _} -> quoted end else quoted end end defp parse_primitive_token(token), do: LiteralUtils.parse_primitive(token) # --------------------------------------------------------------------------- # Blank line validation (strict mode) # --------------------------------------------------------------------------- # In strict mode, blank lines inside arrays (between list items or tabular rows) # are forbidden. Blank lines between object fields are allowed. # Detection: a blank line is inside an array if it is preceded by a line that # looks like an array body line (starts with "- " or "- " indicator OR is a # tabular row at depth > 0 — i.e., not a kv pair and not an array header). defp validate_no_blank_lines_in_arrays(parsed_lines) do do_validate_blank_lines(parsed_lines, nil) end defp do_validate_blank_lines([], _prev_line), do: :ok defp do_validate_blank_lines([line | rest], prev_line) do if String.trim(line.raw) == "" do # Check surrounding non-blank context to decide if we're inside an array. next_line = find_next_non_blank(rest) # Only throw if BOTH surrounding lines indicate we're inside an array scope. # If the next line is at depth 0 (or nil), the blank line is outside the array. prev_in_array = is_array_body_line?(prev_line) next_in_array = is_array_body_line?(next_line) next_depth = if next_line, do: next_line.depth, else: 0 prev_depth = if prev_line, do: prev_line.depth, else: 0 if prev_in_array and next_in_array and next_depth == prev_depth do throw({:decode_error, %DecodeError{ line: line.line_number, reason: :blank_line_in_array, message: "blank line inside array at line #{line.line_number}" }}) else do_validate_blank_lines(rest, prev_line) end else do_validate_blank_lines(rest, line) end end defp find_next_non_blank([]), do: nil defp find_next_non_blank([line | rest]) do if String.trim(line.raw) == "", do: find_next_non_blank(rest), else: line end # An array body line is a list item ("- " prefix or bare "-") OR # a tabular row (at depth > 0, not a kv pair, not an array header). defp is_array_body_line?(nil), do: false defp is_array_body_line?(line) do cond do String.starts_with?(line.content, "- ") or line.content == "-" -> true line.depth > 0 and Scanner.parse_kv(line.content) == :not_kv and Scanner.parse_header(line.content) == :not_header -> true true -> false end end end