defmodule ExToon.Decoder.Scanner do @moduledoc false # Line tokenizer and header parser for the TOON decoder. # # Responsibilities: # - Scan a list of raw line strings into %Scanner{} structs (depth, indent, content) # - Auto-detect indent_size from the first indented non-blank line (default 2) # - Parse array header syntax: key[N][delim]{fields}: [inline_values] # - Parse key:value syntax with quoted-string awareness # - Split values by an active delimiter while respecting quoted strings alias ExToon.{DecodeError, StringUtils} defstruct [:raw, :depth, :indent, :content, :line_number] @type t :: %__MODULE__{ raw: String.t(), depth: non_neg_integer(), # leading spaces count indent: non_neg_integer(), # trimmed content (no leading spaces) content: String.t(), line_number: non_neg_integer() } # Scan a list of line strings into Scanner structs. # Auto-detects indent_size from first indented non-blank line. # Returns {:ok, [t()], indent_size} | {:error, DecodeError.t()} @spec scan_lines([String.t()], pos_integer() | nil) :: {:ok, [t()], pos_integer()} | {:error, DecodeError.t()} def scan_lines(lines, override_indent_size \\ nil) do indent_size = override_indent_size || detect_indent_size(lines) {:ok, do_scan(lines, indent_size, 1, []), indent_size} end defp detect_indent_size(lines) do Enum.find_value(lines, 2, fn line -> spaces = count_leading_spaces(line) if spaces > 0 and String.trim(line) != "", do: spaces, else: nil end) end defp do_scan([], _indent_size, _ln, acc), do: Enum.reverse(acc) defp do_scan([line | rest], indent_size, ln, acc) do spaces = count_leading_spaces(line) content = String.slice(line, spaces, String.length(line)) depth = if indent_size > 0, do: div(spaces, indent_size), else: 0 parsed = %__MODULE__{ raw: line, depth: depth, indent: spaces, content: content, line_number: ln } do_scan(rest, indent_size, ln + 1, [parsed | acc]) end defp count_leading_spaces(line) do case Regex.run(~r/^( *)/, line) do [_, spaces] -> String.length(spaces) _ -> 0 end end # Parse an array header from a line's content. # Returns: # {:array_header, key_or_nil, length, delimiter, fields_or_nil, inline_or_nil} # | :not_header # # Examples: # "users[3]{id,name}:" → {:array_header, "users", 3, 44, ["id", "name"], nil} # "[2]:" → {:array_header, nil, 2, 44, nil, nil} # "items[2\t]: a,b" → {:array_header, "items", 2, 9, nil, "a,b"} @spec parse_header(String.t()) :: {:array_header, String.t() | nil, non_neg_integer(), integer(), [String.t()] | nil, String.t() | nil} | :not_header def parse_header(content) do # Regex: optional key (quoted or unquoted), [N][optional delim char], optional {fields}, :, rest # Using non-extended form to avoid issues with whitespace in the pattern header_regex = ~r/^("(?:[^"\\]|\\.)*"|[A-Za-z_][A-Za-z0-9_.]*|)\[(\d+)([\t|]?)\](?:\{([^}]*)\})?:\s*(.*)$/s case Regex.run(header_regex, content) do nil -> :not_header [_, key_raw, length_str, delim_char, fields_raw, rest_str] -> # Validate it is actually a header — key must be valid if present key = parse_key(key_raw) if key == :error do :not_header else length = String.to_integer(length_str) delimiter = parse_delimiter_char(delim_char) fields = parse_fields(fields_raw, delimiter) inline = if String.trim(rest_str) == "", do: nil, else: String.trim(rest_str) {:array_header, key, length, delimiter, fields, inline} end end end # Parse a key:value line. # Returns {:kv, key, value_str, was_quoted} | :not_kv # was_quoted is true when the key was enclosed in double quotes in the source. @spec parse_kv(String.t()) :: {:kv, String.t(), String.t(), boolean()} | :not_kv def parse_kv(content) do case find_unquoted_colon(content) do nil -> :not_kv idx -> key_raw = String.slice(content, 0, idx) value_str = content |> String.slice(idx + 1, String.length(content)) |> String.trim_leading() key = parse_key(key_raw) was_quoted = String.starts_with?(String.trim(key_raw), "\"") if key == :error or key == nil do :not_kv else {:kv, key, value_str, was_quoted} end end end # Find index of first colon not inside double-quoted string defp find_unquoted_colon(str), do: do_find_colon(str, 0, false) defp do_find_colon("", _idx, _in_quotes), do: nil defp do_find_colon(<<"\\", _::utf8, rest::binary>>, idx, true), do: do_find_colon(rest, idx + 2, true) defp do_find_colon(<<"\"", rest::binary>>, idx, in_quotes), do: do_find_colon(rest, idx + 1, not in_quotes) defp do_find_colon(<<":", _::binary>>, idx, false), do: idx defp do_find_colon(<<_::utf8, rest::binary>>, idx, in_quotes), do: do_find_colon(rest, idx + 1, in_quotes) # Parse a key — quoted or unquoted, returns nil for empty, :error on bad escape defp parse_key(""), do: nil defp parse_key("\"" <> _ = quoted) do len = String.length(quoted) if len >= 2 and String.last(quoted) == "\"" do inner = String.slice(quoted, 1, len - 2) case StringUtils.unescape(inner) do {:ok, s} -> s {:error, _} -> :error end else :error end end defp parse_key(unquoted) do trimmed = String.trim(unquoted) # Unquoted keys must match [A-Za-z_][A-Za-z0-9_.]* if trimmed == "" do nil else trimmed end end defp parse_delimiter_char(""), do: ?, defp parse_delimiter_char("\t"), do: ?\t defp parse_delimiter_char("|"), do: ?| defp parse_fields("", _delim), do: nil defp parse_fields(nil, _delim), do: nil defp parse_fields(fields_raw, delim) do fields_raw |> split_by_delimiter(delim) |> Enum.map(&String.trim/1) |> Enum.map(&parse_key/1) end # Split a string by the active delimiter character, respecting quoted strings. @spec split_by_delimiter(String.t(), integer()) :: [String.t()] def split_by_delimiter(str, delim) do do_split(str, delim, [], []) end defp do_split("", _delim, current, acc) do segment = current |> Enum.reverse() |> IO.iodata_to_binary() Enum.reverse([segment | acc]) end defp do_split(<<"\\", c::utf8, rest::binary>>, delim, current, acc) do do_split(rest, delim, [<>, "\\" | current], acc) end defp do_split(<<"\"", rest::binary>>, delim, current, acc) do {quoted, after_quote} = consume_quoted(rest, ["\""]) do_split(after_quote, delim, [quoted | current], acc) end defp do_split(<>, delim, current, acc) when c == delim do segment = current |> Enum.reverse() |> IO.iodata_to_binary() do_split(rest, delim, [], [segment | acc]) end defp do_split(<>, delim, current, acc) do do_split(rest, delim, [<> | current], acc) end # Consume everything up to and including the closing quote, returning the quoted segment defp consume_quoted("", acc) do {acc |> Enum.reverse() |> IO.iodata_to_binary(), ""} end defp consume_quoted(<<"\\", c::utf8, rest::binary>>, acc) do consume_quoted(rest, [<>, "\\" | acc]) end defp consume_quoted(<<"\"", rest::binary>>, acc) do segment = ["\"" | acc] |> Enum.reverse() |> IO.iodata_to_binary() {segment, rest} end defp consume_quoted(<>, acc) do consume_quoted(rest, [<> | acc]) end end