-module(eyg@parser@lexer). -compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch, inline]). -define(FILEPATH, "src/eyg/parser/lexer.gleam"). -export([lex/1]). -file("src/eyg/parser/lexer.gleam", 19). -spec done(integer()) -> fun((GDT, integer(), GDV) -> {ok, {{GDT, integer()}, integer(), GDV}} | {error, any()}). done(Start) -> fun(T, Size, Rest) -> {ok, {{T, Start}, Start + Size, Rest}} end. -file("src/eyg/parser/lexer.gleam", 135). -spec to_string(bitstring()) -> binary(). to_string(Bits) -> case gleam@bit_array:to_string(Bits) of {ok, S} -> S; {error, _} -> erlang:error(#{gleam_error => panic, message => <<"invalid utf8 in token"/utf8>>, file => <>, module => <<"eyg/parser/lexer"/utf8>>, function => <<"to_string"/utf8>>, line => 139}) end. -file("src/eyg/parser/lexer.gleam", 159). -spec whitespace( bitstring(), bitstring(), fun((eyg@parser@token:token(), integer(), bitstring()) -> GEI) ) -> GEI. whitespace(Buffer, Rest, Done) -> case Rest of <<"\r\n"/utf8, Rest@1/bitstring>> -> whitespace( gleam@bit_array:append(Buffer, <<"\r\n"/utf8>>), Rest@1, Done ); <<"\n"/utf8, Rest@2/bitstring>> -> whitespace( gleam@bit_array:append(Buffer, <<"\n"/utf8>>), Rest@2, Done ); <<" "/utf8, Rest@3/bitstring>> -> whitespace( gleam@bit_array:append(Buffer, <<" "/utf8>>), Rest@3, Done ); <<"\t"/utf8, Rest@4/bitstring>> -> whitespace( gleam@bit_array:append(Buffer, <<"\t"/utf8>>), Rest@4, Done ); _ -> Done( {whitespace, to_string(Buffer)}, erlang:byte_size(Buffer), Rest ) end. -file("src/eyg/parser/lexer.gleam", 230). -spec keyword_or_name(binary()) -> eyg@parser@token:token(). keyword_or_name(Buffer) -> case Buffer of <<"let"/utf8>> -> 'let'; <<"match"/utf8>> -> match; <<"perform"/utf8>> -> perform; <<"deep"/utf8>> -> deep; <<"handle"/utf8>> -> handle; <<"import"/utf8>> -> import; _ -> {name, Buffer} end. -file("src/eyg/parser/lexer.gleam", 258). -spec integer( bitstring(), bitstring(), fun((eyg@parser@token:token(), integer(), bitstring()) -> GEO) ) -> GEO. integer(Buffer, Rest, Done) -> case Rest of <<"1"/utf8, Rest@1/bitstring>> -> integer(gleam@bit_array:append(Buffer, <<"1"/utf8>>), Rest@1, Done); <<"2"/utf8, Rest@2/bitstring>> -> integer(gleam@bit_array:append(Buffer, <<"2"/utf8>>), Rest@2, Done); <<"3"/utf8, Rest@3/bitstring>> -> integer(gleam@bit_array:append(Buffer, <<"3"/utf8>>), Rest@3, Done); <<"4"/utf8, Rest@4/bitstring>> -> integer(gleam@bit_array:append(Buffer, <<"4"/utf8>>), Rest@4, Done); <<"5"/utf8, Rest@5/bitstring>> -> integer(gleam@bit_array:append(Buffer, <<"5"/utf8>>), Rest@5, Done); <<"6"/utf8, Rest@6/bitstring>> -> integer(gleam@bit_array:append(Buffer, <<"6"/utf8>>), Rest@6, Done); <<"7"/utf8, Rest@7/bitstring>> -> integer(gleam@bit_array:append(Buffer, <<"7"/utf8>>), Rest@7, Done); <<"8"/utf8, Rest@8/bitstring>> -> integer(gleam@bit_array:append(Buffer, <<"8"/utf8>>), Rest@8, Done); <<"9"/utf8, Rest@9/bitstring>> -> integer(gleam@bit_array:append(Buffer, <<"9"/utf8>>), Rest@9, Done); <<"0"/utf8, Rest@10/bitstring>> -> integer(gleam@bit_array:append(Buffer, <<"0"/utf8>>), Rest@10, Done); _ -> Done({integer, to_string(Buffer)}, erlang:byte_size(Buffer), Rest) end. -file("src/eyg/parser/lexer.gleam", 274). -spec is_lower_grapheme(bitstring()) -> boolean(). is_lower_grapheme(Grapheme) -> case Grapheme of <<"a"/utf8>> -> true; <<"b"/utf8>> -> true; <<"c"/utf8>> -> true; <<"d"/utf8>> -> true; <<"e"/utf8>> -> true; <<"f"/utf8>> -> true; <<"g"/utf8>> -> true; <<"h"/utf8>> -> true; <<"i"/utf8>> -> true; <<"j"/utf8>> -> true; <<"k"/utf8>> -> true; <<"l"/utf8>> -> true; <<"m"/utf8>> -> true; <<"n"/utf8>> -> true; <<"o"/utf8>> -> true; <<"p"/utf8>> -> true; <<"q"/utf8>> -> true; <<"r"/utf8>> -> true; <<"s"/utf8>> -> true; <<"t"/utf8>> -> true; <<"u"/utf8>> -> true; <<"v"/utf8>> -> true; <<"w"/utf8>> -> true; <<"x"/utf8>> -> true; <<"y"/utf8>> -> true; <<"z"/utf8>> -> true; _ -> false end. -file("src/eyg/parser/lexer.gleam", 306). -spec is_upper_grapheme(bitstring()) -> boolean(). is_upper_grapheme(Grapheme) -> case Grapheme of <<"A"/utf8>> -> true; <<"B"/utf8>> -> true; <<"C"/utf8>> -> true; <<"D"/utf8>> -> true; <<"E"/utf8>> -> true; <<"F"/utf8>> -> true; <<"G"/utf8>> -> true; <<"H"/utf8>> -> true; <<"I"/utf8>> -> true; <<"J"/utf8>> -> true; <<"K"/utf8>> -> true; <<"L"/utf8>> -> true; <<"M"/utf8>> -> true; <<"N"/utf8>> -> true; <<"O"/utf8>> -> true; <<"P"/utf8>> -> true; <<"Q"/utf8>> -> true; <<"R"/utf8>> -> true; <<"S"/utf8>> -> true; <<"T"/utf8>> -> true; <<"U"/utf8>> -> true; <<"V"/utf8>> -> true; <<"W"/utf8>> -> true; <<"X"/utf8>> -> true; <<"Y"/utf8>> -> true; <<"Z"/utf8>> -> true; _ -> false end. -file("src/eyg/parser/lexer.gleam", 338). -spec is_digit_grapheme(bitstring()) -> boolean(). is_digit_grapheme(Grapheme) -> case Grapheme of <<"1"/utf8>> -> true; <<"2"/utf8>> -> true; <<"3"/utf8>> -> true; <<"4"/utf8>> -> true; <<"5"/utf8>> -> true; <<"6"/utf8>> -> true; <<"7"/utf8>> -> true; <<"8"/utf8>> -> true; <<"9"/utf8>> -> true; <<"0"/utf8>> -> true; _ -> false end. -file("src/eyg/parser/lexer.gleam", 354). -spec bytes_take_first(bitstring()) -> bitstring(). bytes_take_first(Raw) -> case Raw of <> -> <>; <<>> -> <<>>; _ -> erlang:error(#{gleam_error => panic, message => <<"unexpected bit string"/utf8>>, file => <>, module => <<"eyg/parser/lexer"/utf8>>, function => <<"bytes_take_first"/utf8>>, line => 359}) end. -file("src/eyg/parser/lexer.gleam", 363). -spec bytes_drop_first(bitstring()) -> bitstring(). bytes_drop_first(Raw) -> case Raw of <<_, Slice/bitstring>> -> Slice; <<>> -> <<>>; _ -> erlang:error(#{gleam_error => panic, message => <<"unexpected bit string"/utf8>>, file => <>, module => <<"eyg/parser/lexer"/utf8>>, function => <<"bytes_drop_first"/utf8>>, line => 368}) end. -file("src/eyg/parser/lexer.gleam", 143). -spec comment( bitstring(), bitstring(), fun((eyg@parser@token:token(), integer(), bitstring()) -> GCL) ) -> GCL. comment(Buffer, Rest, Done) -> case Rest of <<"\r\n"/utf8, _/bitstring>> -> Done( {comment, to_string(Buffer)}, erlang:byte_size(Buffer) + 2, Rest ); <<"\n"/utf8, _/bitstring>> -> Done( {comment, to_string(Buffer)}, erlang:byte_size(Buffer) + 2, Rest ); <<>> -> Done( {comment, to_string(Buffer)}, erlang:byte_size(Buffer) + 2, <<>> ); _ -> Next_byte = bytes_take_first(Rest), Rest@1 = bytes_drop_first(Rest), comment(gleam@bit_array:append(Buffer, Next_byte), Rest@1, Done) end. -file("src/eyg/parser/lexer.gleam", 174). -spec string( bitstring(), integer(), bitstring(), fun((eyg@parser@token:token(), integer(), bitstring()) -> GFF) ) -> GFF. string(Buffer, Length, Rest, Done) -> case Rest of <<"\""/utf8, Rest@1/bitstring>> -> Done({string, to_string(Buffer)}, Length + 1, Rest@1); <<"\\"/utf8, Rest@2/bitstring>> -> case Rest@2 of <<"\""/utf8, Rest@3/bitstring>> -> string( gleam@bit_array:append(Buffer, <<"\""/utf8>>), Length + 2, Rest@3, Done ); <<"\\"/utf8, Rest@4/bitstring>> -> string( gleam@bit_array:append(Buffer, <<"\\"/utf8>>), Length + 2, Rest@4, Done ); <<"t"/utf8, Rest@5/bitstring>> -> string( gleam@bit_array:append(Buffer, <<"\t"/utf8>>), Length + 2, Rest@5, Done ); <<"r"/utf8, Rest@6/bitstring>> -> string( gleam@bit_array:append(Buffer, <<"\r"/utf8>>), Length + 2, Rest@6, Done ); <<"n"/utf8, Rest@7/bitstring>> -> string( gleam@bit_array:append(Buffer, <<"\n"/utf8>>), Length + 2, Rest@7, Done ); <<>> -> Done( {unterminated_string, to_string( gleam@bit_array:append(Buffer, <<"\\"/utf8>>) )}, Length + 1, <<>> ); _ -> Done( {invalid_escape, to_string( gleam@bit_array:append( gleam@bit_array:append( Buffer, <<"\\"/utf8>> ), Rest@2 ) )}, Length, <<>> ) end; <<>> -> Done({unterminated_string, to_string(Buffer)}, Length, <<>>); _ -> Next_byte = bytes_take_first(Rest), Rest@8 = bytes_drop_first(Rest), string( gleam@bit_array:append(Buffer, Next_byte), Length + 1, Rest@8, Done ) end. -file("src/eyg/parser/lexer.gleam", 216). -spec name( bitstring(), bitstring(), fun((eyg@parser@token:token(), integer(), bitstring()) -> GFL) ) -> GFL. name(Buffer, Raw, Done) -> Next_byte = bytes_take_first(Raw), Rest = bytes_drop_first(Raw), case (is_lower_grapheme(Next_byte) orelse is_digit_grapheme(Next_byte)) orelse (Next_byte =:= <<"_"/utf8>>) of true -> name(gleam@bit_array:append(Buffer, Next_byte), Rest, Done); false -> Done( keyword_or_name(to_string(Buffer)), erlang:byte_size(Buffer), Raw ) end. -file("src/eyg/parser/lexer.gleam", 242). -spec uppername( bitstring(), bitstring(), fun((eyg@parser@token:token(), integer(), bitstring()) -> GFQ) ) -> GFQ. uppername(Buffer, Raw, Done) -> Next_byte = bytes_take_first(Raw), Rest = bytes_drop_first(Raw), case ((is_upper_grapheme(Next_byte) orelse is_lower_grapheme(Next_byte)) orelse is_digit_grapheme(Next_byte)) orelse (Next_byte =:= <<"_"/utf8>>) of true -> uppername(gleam@bit_array:append(Buffer, Next_byte), Rest, Done); false -> Done({uppername, to_string(Buffer)}, erlang:byte_size(Buffer), Raw) end. -file("src/eyg/parser/lexer.gleam", 23). -spec pop(bitstring(), integer()) -> {ok, {{eyg@parser@token:token(), integer()}, integer(), bitstring()}} | {error, nil}. pop(Raw, Start) -> Done = done(Start), case Raw of <<"//"/utf8, Rest/bitstring>> -> comment(<<>>, Rest, Done); <<"\r\n"/utf8, Rest@1/bitstring>> -> whitespace(<<"\r\n"/utf8>>, Rest@1, Done); <<"\n"/utf8, Rest@2/bitstring>> -> whitespace(<<"\n"/utf8>>, Rest@2, Done); <<" "/utf8, Rest@3/bitstring>> -> whitespace(<<" "/utf8>>, Rest@3, Done); <<"\t"/utf8, Rest@4/bitstring>> -> whitespace(<<"\t"/utf8>>, Rest@4, Done); <<"("/utf8, Rest@5/bitstring>> -> Done(left_paren, 1, Rest@5); <<")"/utf8, Rest@6/bitstring>> -> Done(right_paren, 1, Rest@6); <<"{"/utf8, Rest@7/bitstring>> -> Done(left_brace, 1, Rest@7); <<"}"/utf8, Rest@8/bitstring>> -> Done(right_brace, 1, Rest@8); <<"["/utf8, Rest@9/bitstring>> -> Done(left_square, 1, Rest@9); <<"]"/utf8, Rest@10/bitstring>> -> Done(right_square, 1, Rest@10); <<"="/utf8, Rest@11/bitstring>> -> Done(equal, 1, Rest@11); <<"->"/utf8, Rest@12/bitstring>> -> Done(right_arrow, 2, Rest@12); <<","/utf8, Rest@13/bitstring>> -> Done(comma, 1, Rest@13); <<".."/utf8, Rest@14/bitstring>> -> Done(dot_dot, 2, Rest@14); <<"."/utf8, Rest@15/bitstring>> -> Done(dot, 1, Rest@15); <<":"/utf8, Rest@16/bitstring>> -> Done(colon, 1, Rest@16); <<"-"/utf8, Rest@17/bitstring>> -> Done(minus, 1, Rest@17); <<"!"/utf8, Rest@18/bitstring>> -> Done(bang, 1, Rest@18); <<"|"/utf8, Rest@19/bitstring>> -> Done(bar, 1, Rest@19); <<"#"/utf8, Rest@20/bitstring>> -> Done(hash, 1, Rest@20); <<"@"/utf8, Rest@21/bitstring>> -> Done(at, 1, Rest@21); <<"\""/utf8, Rest@22/bitstring>> -> string(<<>>, 1, Rest@22, Done); <<"1"/utf8, Rest@23/bitstring>> -> integer(<<"1"/utf8>>, Rest@23, Done); <<"2"/utf8, Rest@24/bitstring>> -> integer(<<"2"/utf8>>, Rest@24, Done); <<"3"/utf8, Rest@25/bitstring>> -> integer(<<"3"/utf8>>, Rest@25, Done); <<"4"/utf8, Rest@26/bitstring>> -> integer(<<"4"/utf8>>, Rest@26, Done); <<"5"/utf8, Rest@27/bitstring>> -> integer(<<"5"/utf8>>, Rest@27, Done); <<"6"/utf8, Rest@28/bitstring>> -> integer(<<"6"/utf8>>, Rest@28, Done); <<"7"/utf8, Rest@29/bitstring>> -> integer(<<"7"/utf8>>, Rest@29, Done); <<"8"/utf8, Rest@30/bitstring>> -> integer(<<"8"/utf8>>, Rest@30, Done); <<"9"/utf8, Rest@31/bitstring>> -> integer(<<"9"/utf8>>, Rest@31, Done); <<"0"/utf8, Rest@32/bitstring>> -> integer(<<"0"/utf8>>, Rest@32, Done); _ -> Next_byte = bytes_take_first(Raw), Rest@33 = bytes_drop_first(Raw), case Next_byte of <<"_"/utf8>> -> name(Next_byte, Rest@33, Done); <<"a"/utf8>> -> name(Next_byte, Rest@33, Done); <<"b"/utf8>> -> name(Next_byte, Rest@33, Done); <<"c"/utf8>> -> name(Next_byte, Rest@33, Done); <<"d"/utf8>> -> name(Next_byte, Rest@33, Done); <<"e"/utf8>> -> name(Next_byte, Rest@33, Done); <<"f"/utf8>> -> name(Next_byte, Rest@33, Done); <<"g"/utf8>> -> name(Next_byte, Rest@33, Done); <<"h"/utf8>> -> name(Next_byte, Rest@33, Done); <<"i"/utf8>> -> name(Next_byte, Rest@33, Done); <<"j"/utf8>> -> name(Next_byte, Rest@33, Done); <<"k"/utf8>> -> name(Next_byte, Rest@33, Done); <<"l"/utf8>> -> name(Next_byte, Rest@33, Done); <<"m"/utf8>> -> name(Next_byte, Rest@33, Done); <<"n"/utf8>> -> name(Next_byte, Rest@33, Done); <<"o"/utf8>> -> name(Next_byte, Rest@33, Done); <<"p"/utf8>> -> name(Next_byte, Rest@33, Done); <<"q"/utf8>> -> name(Next_byte, Rest@33, Done); <<"r"/utf8>> -> name(Next_byte, Rest@33, Done); <<"s"/utf8>> -> name(Next_byte, Rest@33, Done); <<"t"/utf8>> -> name(Next_byte, Rest@33, Done); <<"u"/utf8>> -> name(Next_byte, Rest@33, Done); <<"v"/utf8>> -> name(Next_byte, Rest@33, Done); <<"w"/utf8>> -> name(Next_byte, Rest@33, Done); <<"x"/utf8>> -> name(Next_byte, Rest@33, Done); <<"y"/utf8>> -> name(Next_byte, Rest@33, Done); <<"z"/utf8>> -> name(Next_byte, Rest@33, Done); <<"A"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"B"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"C"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"D"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"E"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"F"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"G"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"H"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"I"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"J"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"K"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"L"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"M"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"N"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"O"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"P"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"Q"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"R"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"S"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"T"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"U"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"V"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"W"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"X"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"Y"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<"Z"/utf8>> -> uppername(Next_byte, Rest@33, Done); <<>> -> {error, nil}; _ -> String = to_string(Raw), {Char, Rest@34} = begin _pipe = gleam_stdlib:string_pop_grapheme(String), gleam@result:unwrap(_pipe, {String, <<""/utf8>>}) end, Size = erlang:byte_size(<>), Done({unexpected_grapheme, Char}, Size, <>) end end. -file("src/eyg/parser/lexer.gleam", 12). -spec loop(bitstring(), integer(), list({eyg@parser@token:token(), integer()})) -> list({eyg@parser@token:token(), integer()}). loop(Raw, Offset, Acc) -> case pop(Raw, Offset) of {ok, {Token, Offset@1, Rest}} -> loop(Rest, Offset@1, [Token | Acc]); {error, nil} -> lists:reverse(Acc) end. -file("src/eyg/parser/lexer.gleam", 7). -spec lex(binary()) -> list({eyg@parser@token:token(), integer()}). lex(Raw) -> Bits = gleam_stdlib:identity(Raw), loop(Bits, 0, []).