defmodule Unicode.Set.Parser do @moduledoc false import NimbleParsec import Unicode.Set.Property @doc false defguard is_hex_digit(c) when c in ?0..?9 or c in ?a..?z or c in ?A..?Z @doc false def unicode_set do choice([ property(), empty_set(), basic_set() ]) end @doc false def basic_set do ignore(ascii_char([?[])) |> optional(ascii_char([?-, ?^]) |> replace(:not)) |> times(sequence(), min: 1) |> ignore(ascii_char([?]])) |> reduce(:reduce_set_operations) |> label("set") end @doc false def empty_set do string("[-]") |> label("empty set") end @doc false def sequence do choice([ maybe_repeated_set(), range() ]) |> ignore(optional(whitespace())) |> label("sequence") end @doc false def maybe_repeated_set do parsec(:one_set) |> repeat(set_operator() |> parsec(:one_set)) end @debug_functions [] defmacrop tracer(step, a) do {caller, _} = __CALLER__.function if Mix.env() == :dev and caller in @debug_functions do quote do IO.inspect("#{unquote(caller)}", label: "Step #{unquote(step)}") IO.inspect(unquote(a), label: "argument") end else quote do _ = {unquote(step), unquote(a)} end end end @doc false def reduce_set_operations([set_a]) do tracer(0, [set_a]) set_a end def reduce_set_operations([set_a, operator, set_b]) when operator in [:difference, :intersection] do tracer(1, [set_a, operator, set_b]) {operator, [set_a, set_b]} end def reduce_set_operations([set_a, operator, set_b | repeated_sets]) when operator in [:difference, :intersection] do tracer(2, [set_a, operator, set_b | repeated_sets]) reduce_set_operations([{operator, [set_a, set_b]} | repeated_sets]) end def reduce_set_operations([{:in, ranges1}, {:in, ranges2} | rest]) do tracer(2, [{:in, ranges1}, {:in, ranges2} | rest]) reduce_set_operations([{:in, Enum.sort(ranges1 ++ ranges2)} | rest]) end def reduce_set_operations([:not, {:in, ranges1}, {:in, ranges2} | rest]) do tracer(3, [:not, {:in, ranges1}, {:in, ranges2} | rest]) reduce_set_operations([:not, {:in, Enum.sort(ranges1 ++ ranges2)} | rest]) end def reduce_set_operations([:not, {:in, ranges} | rest]) do tracer(4, [:not, {:in, ranges} | rest]) reduce_set_operations([{:not_in, ranges} | rest]) end def reduce_set_operations([:not, {:not_in, ranges}]) do tracer(5, [:not, {:not_in, ranges}]) reduce_set_operations([{:in, ranges}]) end def reduce_set_operations([:not | ranges]) do tracer(6, [:not | ranges]) reduce_set_operations([{:not_in, ranges}]) end def reduce_set_operations([set_a | rest]) do tracer(7, [set_a | rest]) {:union, [set_a, reduce_set_operations(rest)]} end @doc false def set_operator do ignore(optional(whitespace())) |> choice([ ascii_char([?&]) |> replace(:intersection), ascii_char([?-]) |> replace(:difference) ]) |> ignore(optional(whitespace())) end @doc false def range do choice([ character_range(), string_range() ]) |> reduce(:reduce_range) |> post_traverse(:check_valid_range) |> label("range") end @doc false def character_range do char() |> ignore(optional(whitespace())) |> optional( ignore(ascii_char([?-])) |> ignore(optional(whitespace())) |> concat(char()) ) end @doc false # Of the forrm {abc} or {abc-def} def string_range do string() |> wrap |> ignore(optional(whitespace())) |> optional( ignore(ascii_char([?-])) |> ignore(optional(whitespace())) |> concat(string() |> wrap) ) end @doc false def reduce_range([[bracketed]]) when is_list(bracketed), do: {:in, Enum.map(bracketed, &{&1, &1})} def reduce_range([[from]]) when is_integer(from), do: {:in, [{from, from}]} def reduce_range([[from], [to]]) when is_integer(from) and is_integer(to), do: {:in, [{from, to}]} def reduce_range([from]), do: {:in, [{from, from}]} def reduce_range([from, to]), do: {:in, [{from, to}]} @doc false def check_valid_range(_rest, [in: [{from, to}]] = args, context, _, _) when is_integer(from) and is_integer(to) do {args, context} end def check_valid_range(_rest, [in: [{from, from}]] = args, context, _, _) do {args, context} end def check_valid_range(_rest, [in: [{from, to}]] = args, context, _, _) do if length(from) == 1 or length(to) == 1 do {:error, "String ranges must be longer than one character. Found " <> format_string_range(from, to)} else {args, context} end end @doc false def property do choice([ perl_property(), posix_property() ]) |> post_traverse(:reduce_property) |> label("property") end @doc false def posix_property do ignore(string("[:")) |> optional(ascii_char([?^]) |> replace(:not)) |> concat(property_name()) |> optional(operator() |> ignore(optional(whitespace())) |> concat(value_2())) |> ignore(string(":]")) |> label("posix property") end @doc false def perl_property do ignore(ascii_char([?\\])) |> choice([ascii_char([?P]) |> replace(:not), ignore(ascii_char([?p]))]) |> ignore(ascii_char([?{])) |> concat(property_name()) |> optional(operator() |> ignore(optional(whitespace())) |> concat(value_1())) |> ignore(ascii_char([?}])) |> label("perl property") end @doc false def operator do choice([ utf8_char([0x2260]) |> replace(:not_in), ascii_char([?=]) |> replace(:in) ]) end @doc false def reduce_property(_rest, [value, :in, property, :not], context, _line, _offset) do tracer(1, [value, :in, property, :not]) case fetch_property!(property, value) do %{parsed: parsed} -> {{:not_in, parsed}, context} ranges -> {[{:not_in, ranges}], context} end end def reduce_property(_rest, [value, :not_in, property, :not], context, _line, _offset) do tracer(2, [value, :not_in, property, :not]) case fetch_property!(property, value) do %{parsed: parsed} -> {parsed, context} ranges -> {[{:in, ranges}], context} end end def reduce_property(_rest, [value, operator, property], context, _line, _offset) when operator in [:in, :not_in] do tracer(3, [value, operator, property]) case fetch_property!(property, value) do %{parsed: parsed} -> {[{operator, parsed}], context} ranges -> {[{operator, ranges}], context} end end def reduce_property(_rest, [value, :not], context, _line, _offset) do tracer(4, [value, :not]) case fetch_property!(:script_or_category, value) do %{parsed: [{:not_in, parsed}]} -> {[{:in, parsed}], context} %{parsed: [{:in, parsed}]} -> {[{:not_in, parsed}], context} %{parsed: parsed} -> {[{:not_in, parsed}], context} ranges -> {[{:not_in, ranges}], context} end end def reduce_property(_rest, [value], context, _line, _offset) do tracer(5, [value]) case fetch_property!(:script_or_category, value) do %{parsed: [{:not_in, parsed}]} -> {[{:not_in, parsed}], context} %{parsed: [{:in, parsed}]} -> {[{:in, parsed}], context} %{parsed: parsed} -> {parsed, context} ranges -> {[{:in, ranges}], context} end end @doc false @alphanumeric [?a..?z, ?A..?Z, ?0..?9] def property_name do ignore(optional(whitespace())) |> ascii_char(@alphanumeric) |> repeat(ascii_char(@alphanumeric ++ [?_, ?\s])) |> ignore(optional(whitespace())) |> reduce(:to_lower_string) |> label("property name") end @doc false def value_1 do times( choice([ ignore(ascii_char([?\\])) |> concat(quoted()), ascii_char([{:not, ?}}]) ]), min: 1 ) |> reduce(:to_lower_string) end @doc false def value_2 do times( choice([ ignore(ascii_char([?\\])) |> concat(quoted()), ascii_char([{:not, ?:}]) ]), min: 1 ) |> reduce(:to_lower_string) end @doc false def to_lower_string(args) do args |> List.to_string() |> String.replace(" ", "_") |> String.downcase() end @doc false @whitespace_chars [0x20, 0x9..0xD, 0x85, 0x200E, 0x200F, 0x2028, 0x2029] def whitespace_char do ascii_char(@whitespace_chars) end @doc false def whitespace do times(whitespace_char(), min: 1) end @doc false def string do ignore(ascii_char([?{])) |> times(ignore(optional(whitespace())) |> concat(char()), min: 1) |> ignore(optional(whitespace())) |> ignore(ascii_char([?}])) end @doc false # ++ @whitespace_chars @syntax_chars [?&, ?-, ?[, ?], ?\\, ?{, ?}] @not_syntax_chars Enum.map(@syntax_chars, fn c -> {:not, c} end) def char do choice([ ignore(ascii_char([?\\])) |> concat(quoted()), utf8_char(@not_syntax_chars) ]) end @doc false def quoted do choice([ ignore(ascii_char([?x])) |> choice([ ascii_char([?0]) |> times(hex(), 5), ascii_char([?1]) |> ascii_char([?0]) |> times(hex(), 4) ]), string("N{") |> concat(property_name()) |> ascii_char([?}]), ignore(ascii_char([?u])) |> choice([times(hex(), 4), bracketed_hex()]), ignore(ascii_char([?x])) |> choice([times(hex(), 2), bracketed_hex()]), utf8_char([0x0..0x10FFFF]) ]) |> reduce(:hex_to_codepoint) |> label("quoted character") end @doc false def bracketed_hex do ignore(ascii_char([?{])) |> ignore(optional(whitespace())) |> concat(hex_codepoint()) |> repeat(ignore(optional(whitespace())) |> concat(hex_codepoint())) |> ignore(optional(whitespace())) |> ignore(ascii_char([?}])) |> wrap |> label("bracketed hex") end @doc false def hex_codepoint do choice([ times(hex(), min: 1, max: 5), ascii_char([?1]) |> ascii_char([?0]) |> times(hex(), 4) ]) |> wrap |> label("hex codepoint") end @doc false def hex do ascii_char([?a..?f, ?A..?F, ?0..?9]) |> label("hex character") end @doc false # Its just an escaped char def hex_to_codepoint([?t]), do: ?\t def hex_to_codepoint([?n]), do: ?\n def hex_to_codepoint([?r]), do: ?\r def hex_to_codepoint([c]) when not is_hex_digit(c), do: c # Actual hex-encoded codepoints def hex_to_codepoint([arg | _rest] = args) when is_list(arg) do Enum.map(args, &hex_to_codepoint/1) end def hex_to_codepoint(args) do args |> List.to_string() |> String.to_integer(16) end @doc false # Applied to a regex def repetition do ignore(optional(whitespace())) |> choice([ ascii_char([?*]) |> replace({:repeat, min: 0, max: :infinity}), ascii_char([?+]) |> replace({:repeat, min: 1, max: :infinity}), ascii_char([??]) |> replace({:repeat, min: 0, max: 1}), iterations() ]) end @doc false def iterations do ignore(ascii_char([?{])) |> ignore(optional(whitespace())) |> integer(min: 1) |> ignore(optional(whitespace())) |> ignore(ascii_char([?,])) |> ignore(optional(whitespace())) |> integer(min: 1) |> ignore(optional(whitespace())) |> ignore(ascii_char([?}])) |> reduce(:iteration) end @doc false def iteration([from, to]) do {:repeat, min: from, max: to} end @doc false def anchor do ignore(optional(whitespace())) |> ascii_char([?$]) |> replace(:end) end # Helpers # ------- defp format_string_range(from, to) do "{#{List.to_string(from)}}-{#{List.to_string(to)}}" end end