-module(glex). -compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch]). -export([new_rule/2, new/0, add_rule/3, add_ignore/3, build/2, ok/1, valid_only/1, ignored_only/1, invalid_only/1, next/1, iterator/1, lex/1]). -export_type([token/0, position/0, rule/0, lexer/0]). -type token() :: {valid, binary(), binary()} | {ignored, binary(), binary()} | {invalid, binary()} | end_of_file. -type position() :: {position, integer()}. -opaque rule() :: {rule, binary(), binary(), gleam@regex:regex()}. -opaque lexer() :: {lexer, list(rule()), list(rule()), binary(), integer()}. -spec new_rule(binary(), binary()) -> rule(). new_rule(Name, Pattern) -> Options = {options, false, false}, Pattern@1 = case Pattern of <<"^"/utf8, _/binary>> -> Pattern; _ -> <<"^"/utf8, Pattern/binary>> end, _assert_subject = gleam@regex:compile(Pattern@1, Options), {ok, Regex} = case _assert_subject of {ok, _} -> _assert_subject; _assert_fail -> erlang:error(#{gleam_error => let_assert, message => <<"Assertion pattern match failed"/utf8>>, value => _assert_fail, module => <<"glex"/utf8>>, function => <<"new_rule"/utf8>>, line => 28}) end, {rule, Name, Pattern@1, Regex}. -spec new() -> lexer(). new() -> {lexer, [], [], <<""/utf8>>, 0}. -spec add_rule(lexer(), binary(), binary()) -> lexer(). add_rule(Lexer, Name, Pattern) -> Rule = new_rule(Name, Pattern), {lexer, gleam@list:concat([erlang:element(2, Lexer), [Rule]]), erlang:element(3, Lexer), erlang:element(4, Lexer), erlang:element(5, Lexer)}. -spec add_ignore(lexer(), binary(), binary()) -> lexer(). add_ignore(Lexer, Name, Pattern) -> Rule = new_rule(Name, Pattern), {lexer, erlang:element(2, Lexer), gleam@list:concat([erlang:element(3, Lexer), [Rule]]), erlang:element(4, Lexer), erlang:element(5, Lexer)}. -spec build(lexer(), binary()) -> lexer(). build(Lexer, Source) -> {lexer, erlang:element(2, Lexer), erlang:element(3, Lexer), Source, 0}. -spec ok(list({token(), position()})) -> boolean(). ok(Tokens) -> _pipe = Tokens, gleam@list:all(_pipe, fun(Token) -> case Token of {{invalid, _}, _} -> false; _ -> true end end). -spec valid_only(list({token(), position()})) -> list({token(), position()}). valid_only(Tokens) -> _pipe = Tokens, gleam@list:filter(_pipe, fun(Token) -> case Token of {{valid, _, _}, _} -> true; _ -> false end end). -spec ignored_only(list({token(), position()})) -> list({token(), position()}). ignored_only(Tokens) -> _pipe = Tokens, gleam@list:filter(_pipe, fun(Token) -> case Token of {{ignored, _, _}, _} -> true; _ -> false end end). -spec invalid_only(list({token(), position()})) -> list({token(), position()}). invalid_only(Tokens) -> _pipe = Tokens, gleam@list:filter(_pipe, fun(Token) -> case Token of {{invalid, _}, _} -> true; _ -> false end end). -spec advance(lexer(), binary(), integer()) -> lexer(). advance(Lexer, Source, Offset) -> {lexer, erlang:element(2, Lexer), erlang:element(3, Lexer), Source, erlang:element(5, Lexer) + Offset}. -spec wrap_token(lexer(), token()) -> {token(), position()}. wrap_token(Lexer, Token) -> {Token, {position, erlang:element(5, Lexer)}}. -spec byte_size(binary()) -> integer(). byte_size(String) -> erlang:byte_size(<>). -spec lex_rules(list(rule()), binary()) -> {ok, {binary(), binary()}} | {error, nil}. lex_rules(Rules, Source) -> _pipe = Rules, _pipe@3 = gleam@list:flat_map( _pipe, fun(Rule) -> _pipe@1 = erlang:element(4, Rule), _pipe@2 = gleam@regex:scan(_pipe@1, Source), gleam@list:map( _pipe@2, fun(Match) -> {erlang:element(2, Rule), erlang:element(2, Match)} end ) end ), gleam@list:first(_pipe@3). -spec next(lexer()) -> {lexer(), {token(), position()}}. next(Lexer) -> Ignore = lex_rules(erlang:element(3, Lexer), erlang:element(4, Lexer)), case Ignore of {ok, {Name, Value}} -> Rest = begin _pipe = erlang:element(4, Lexer), gleam@string:drop_left(_pipe, gleam@string:length(Value)) end, {advance(Lexer, Rest, gleam@string:length(Value)), wrap_token(Lexer, {ignored, Name, Value})}; {error, nil} -> Rules = lex_rules( erlang:element(2, Lexer), erlang:element(4, Lexer) ), case Rules of {ok, {Name@1, Value@1}} -> Rest@1 = begin _pipe@1 = erlang:element(4, Lexer), gleam@string:drop_left( _pipe@1, gleam@string:length(Value@1) ) end, {advance(Lexer, Rest@1, gleam@string:length(Value@1)), wrap_token(Lexer, {valid, Name@1, Value@1})}; {error, nil} -> case gleam@string:pop_grapheme(erlang:element(4, Lexer)) of {error, nil} -> {Lexer, {end_of_file, {position, erlang:element(5, Lexer)}}}; {ok, {Grapheme, Rest@2}} -> {advance(Lexer, Rest@2, byte_size(Grapheme)), wrap_token(Lexer, {invalid, Grapheme})} end end end. -spec iterator(lexer()) -> gleam@iterator:iterator({token(), position()}). iterator(Lexer) -> gleam@iterator:unfold(Lexer, fun(Lexer@1) -> case next(Lexer@1) of {_, {end_of_file, _}} -> done; {Lexer@2, Wraped_token} -> {next, Wraped_token, Lexer@2} end end). -spec lex(lexer()) -> list({token(), position()}). lex(Lexer) -> _pipe = iterator(Lexer), gleam@iterator:to_list(_pipe).