-module(starfuzz@normalize). -compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch, inline]). -define(FILEPATH, "src/starfuzz/normalize.gleam"). -export([new/0, with_lowercase/1, with_trim/1, with_collapsed_whitespace/1, without_punctuation/1, with_punctuation_as_space/1, apply/2, basic/1, search_text/1]). -export_type([normalizer/0]). -if(?OTP_RELEASE >= 27). -define(MODULEDOC(Str), -moduledoc(Str)). -define(DOC(Str), -doc(Str)). -else. -define(MODULEDOC(Str), -compile([])). -define(DOC(Str), -compile([])). -endif. ?MODULEDOC( " A module providing text normalization capabilities.\n" " Normalization is typically the first step before calculating text similarities,\n" " ensuring formatting differences (casing, spaces, punctuation) do not distort scores.\n" ). -type normalizer() :: {normalizer, boolean(), boolean(), boolean(), boolean(), boolean()}. -file("src/starfuzz/normalize.gleam", 20). ?DOC(" Creates a new default normalizer with all options disabled.\n"). -spec new() -> normalizer(). new() -> {normalizer, false, false, false, false, false}. -file("src/starfuzz/normalize.gleam", 31). ?DOC(" Configures the normalizer to convert all text to lowercase.\n"). -spec with_lowercase(normalizer()) -> normalizer(). with_lowercase(Normalizer) -> {normalizer, true, erlang:element(3, Normalizer), erlang:element(4, Normalizer), erlang:element(5, Normalizer), erlang:element(6, Normalizer)}. -file("src/starfuzz/normalize.gleam", 36). ?DOC(" Configures the normalizer to trim leading and trailing whitespaces.\n"). -spec with_trim(normalizer()) -> normalizer(). with_trim(Normalizer) -> {normalizer, erlang:element(2, Normalizer), true, erlang:element(4, Normalizer), erlang:element(5, Normalizer), erlang:element(6, Normalizer)}. -file("src/starfuzz/normalize.gleam", 41). ?DOC(" Configures the normalizer to collapse multiple consecutive whitespaces into a single space.\n"). -spec with_collapsed_whitespace(normalizer()) -> normalizer(). with_collapsed_whitespace(Normalizer) -> {normalizer, erlang:element(2, Normalizer), erlang:element(3, Normalizer), true, erlang:element(5, Normalizer), erlang:element(6, Normalizer)}. -file("src/starfuzz/normalize.gleam", 46). ?DOC(" Configures the normalizer to strip all punctuation marks from the text.\n"). -spec without_punctuation(normalizer()) -> normalizer(). without_punctuation(Normalizer) -> {normalizer, erlang:element(2, Normalizer), erlang:element(3, Normalizer), erlang:element(4, Normalizer), true, false}. -file("src/starfuzz/normalize.gleam", 51). ?DOC(" Configures the normalizer to replace punctuation marks with spaces.\n"). -spec with_punctuation_as_space(normalizer()) -> normalizer(). with_punctuation_as_space(Normalizer) -> {normalizer, erlang:element(2, Normalizer), erlang:element(3, Normalizer), erlang:element(4, Normalizer), false, true}. -file("src/starfuzz/normalize.gleam", 55). -spec is_punctuation(binary()) -> boolean(). is_punctuation(Char) -> case Char of <<"!"/utf8>> -> true; <<"\""/utf8>> -> true; <<"#"/utf8>> -> true; <<"$"/utf8>> -> true; <<"%"/utf8>> -> true; <<"&"/utf8>> -> true; <<"'"/utf8>> -> true; <<"("/utf8>> -> true; <<")"/utf8>> -> true; <<"*"/utf8>> -> true; <<"+"/utf8>> -> true; <<","/utf8>> -> true; <<"-"/utf8>> -> true; <<"."/utf8>> -> true; <<"/"/utf8>> -> true; <<":"/utf8>> -> true; <<";"/utf8>> -> true; <<"<"/utf8>> -> true; <<"="/utf8>> -> true; <<">"/utf8>> -> true; <<"?"/utf8>> -> true; <<"@"/utf8>> -> true; <<"["/utf8>> -> true; <<"\\"/utf8>> -> true; <<"]"/utf8>> -> true; <<"^"/utf8>> -> true; <<"_"/utf8>> -> true; <<"`"/utf8>> -> true; <<"{"/utf8>> -> true; <<"|"/utf8>> -> true; <<"}"/utf8>> -> true; <<"~"/utf8>> -> true; <<"“"/utf8>> -> true; <<"”"/utf8>> -> true; <<"‘"/utf8>> -> true; <<"’"/utf8>> -> true; <<"—"/utf8>> -> true; <<"–"/utf8>> -> true; _ -> false end. -file("src/starfuzz/normalize.gleam", 64). -spec is_whitespace(binary()) -> boolean(). is_whitespace(Char) -> case Char of <<" "/utf8>> -> true; <<"\t"/utf8>> -> true; <<"\n"/utf8>> -> true; <<"\r"/utf8>> -> true; _ -> false end. -file("src/starfuzz/normalize.gleam", 72). ?DOC(" Applies the normalizer pipeline to the input string.\n"). -spec apply(normalizer(), binary()) -> binary(). apply(Normalizer, Input) -> Result = Input, Result@1 = case erlang:element(2, Normalizer) of true -> string:lowercase(Result); false -> Result end, Result@2 = case erlang:element(5, Normalizer) orelse erlang:element( 6, Normalizer ) of true -> Graphemes = gleam@string:to_graphemes(Result@1), Processed = gleam@list:map( Graphemes, fun(G) -> case is_punctuation(G) of true -> case erlang:element(6, Normalizer) of true -> <<" "/utf8>>; false -> <<""/utf8>> end; false -> G end end ), gleam@string:join(Processed, <<""/utf8>>); false -> Result@1 end, Result@3 = case erlang:element(3, Normalizer) of true -> gleam@string:trim(Result@2); false -> Result@2 end, Result@4 = case erlang:element(4, Normalizer) of true -> Graphemes@1 = gleam@string:to_graphemes(Result@3), {Acc, _} = gleam@list:fold( Graphemes@1, {[], false}, fun(State, G@1) -> {List_acc, Is_space_state} = State, Current_is_space = is_whitespace(G@1), case Current_is_space of true -> case Is_space_state of true -> {List_acc, true}; false -> {[<<" "/utf8>> | List_acc], true} end; false -> {[G@1 | List_acc], false} end end ), gleam@string:join(lists:reverse(Acc), <<""/utf8>>); false -> Result@3 end, Result@4. -file("src/starfuzz/normalize.gleam", 131). ?DOC(" A preset performing lowercase mapping, trimming, punctuation stripping, and collapsing spaces.\n"). -spec basic(binary()) -> binary(). basic(Input) -> _pipe = new(), _pipe@1 = with_lowercase(_pipe), _pipe@2 = with_trim(_pipe@1), _pipe@3 = without_punctuation(_pipe@2), _pipe@4 = with_collapsed_whitespace(_pipe@3), apply(_pipe@4, Input). -file("src/starfuzz/normalize.gleam", 141). ?DOC(" A preset performing lowercase mapping, trimming, replacing punctuation with spaces, and collapsing spaces.\n"). -spec search_text(binary()) -> binary(). search_text(Input) -> _pipe = new(), _pipe@1 = with_lowercase(_pipe), _pipe@2 = with_trim(_pipe@1), _pipe@3 = with_punctuation_as_space(_pipe@2), _pipe@4 = with_collapsed_whitespace(_pipe@3), _pipe@5 = apply(_pipe@4, Input), gleam@string:trim(_pipe@5).