-module(prequel@internals@token). -compile([no_auto_import, nowarn_unused_vars]). -export([to_string/1, scan/1]). -export_type([token/0]). -type token() :: open_bracket | close_bracket | open_parens | close_parens | colon | circle_lollipop | star_lollipop | arrow_lollipop | minus | ampersand | {simple_comment, binary()} | {number, binary()} | {word, binary()}. -spec to_string(token()) -> binary(). to_string(Token) -> case Token of open_bracket -> <<"{"/utf8>>; close_bracket -> <<"}"/utf8>>; open_parens -> <<"("/utf8>>; close_parens -> <<")"/utf8>>; colon -> <<":"/utf8>>; circle_lollipop -> <<"-o"/utf8>>; star_lollipop -> <<"-*"/utf8>>; arrow_lollipop -> <<"->"/utf8>>; minus -> <<"-"/utf8>>; ampersand -> <<"&"/utf8>>; {simple_comment, Comment} -> <<"//"/utf8, Comment/binary>>; {number, Number} -> Number; {word, Word} -> Word end. -spec scan_comment_body( gleam@string_builder:string_builder(), integer(), binary() ) -> {binary(), integer(), binary()}. scan_comment_body(Acc, Size, Source) -> case Source of <<"\r\n"/utf8, Rest/binary>> -> {gleam@string_builder:to_string(Acc), Size, Rest}; <<"\n"/utf8, Rest/binary>> -> {gleam@string_builder:to_string(Acc), Size, Rest}; Rest@1 -> case gleam@string:pop_grapheme(Rest@1) of {error, nil} -> {gleam@string_builder:to_string(Acc), Size, Rest@1}; {ok, {Grapheme, Rest@2}} -> scan_comment_body( gleam@string_builder:append(Acc, Grapheme), Size + 1, Rest@2 ) end end. -spec scan_word(gleam@string_builder:string_builder(), integer(), binary()) -> {binary(), integer(), binary()}. scan_word(Acc, Size, Source) -> case Source of <<"//"/utf8, _/binary>> -> {gleam@string_builder:to_string(Acc), Size, Source}; <<"///"/utf8, _/binary>> -> {gleam@string_builder:to_string(Acc), Size, Source}; <<"////"/utf8, _/binary>> -> {gleam@string_builder:to_string(Acc), Size, Source}; <<":"/utf8, _/binary>> -> {gleam@string_builder:to_string(Acc), Size, Source}; <<"("/utf8, _/binary>> -> {gleam@string_builder:to_string(Acc), Size, Source}; <<")"/utf8, _/binary>> -> {gleam@string_builder:to_string(Acc), Size, Source}; <<"{"/utf8, _/binary>> -> {gleam@string_builder:to_string(Acc), Size, Source}; <<"}"/utf8, _/binary>> -> {gleam@string_builder:to_string(Acc), Size, Source}; <<" "/utf8, _/binary>> -> {gleam@string_builder:to_string(Acc), Size, Source}; <<"\n"/utf8, _/binary>> -> {gleam@string_builder:to_string(Acc), Size, Source}; <<"\r"/utf8, _/binary>> -> {gleam@string_builder:to_string(Acc), Size, Source}; <<"\t"/utf8, _/binary>> -> {gleam@string_builder:to_string(Acc), Size, Source}; Rest -> case gleam@string:pop_grapheme(Rest) of {error, nil} -> {gleam@string_builder:to_string(Acc), Size, Rest}; {ok, {Grapheme, Rest@1}} -> scan_word( gleam@string_builder:append(Acc, Grapheme), Size + 1, Rest@1 ) end end. -spec do_scan( binary(), integer(), integer(), list({token(), prequel@span:span()}) ) -> list({token(), prequel@span:span()}). do_scan(Source, Line, Column, Acc) -> case Source of <<""/utf8>> -> gleam@list:reverse(Acc); <<"\n"/utf8, Rest/binary>> -> do_scan(Rest, Line + 1, 1, Acc); <<" "/utf8, Rest@1/binary>> -> do_scan(Rest@1, Line, Column + 1, Acc); <<"\t"/utf8, Rest@1/binary>> -> do_scan(Rest@1, Line, Column + 1, Acc); <<"\r"/utf8, Rest@1/binary>> -> do_scan(Rest@1, Line, Column + 1, Acc); <<"{"/utf8, Rest@2/binary>> -> Lexeme = {open_bracket, prequel@span:point(Line, Column)}, do_scan(Rest@2, Line, Column + 1, [Lexeme | Acc]); <<"}"/utf8, Rest@3/binary>> -> Lexeme@1 = {close_bracket, prequel@span:point(Line, Column)}, do_scan(Rest@3, Line, Column + 1, [Lexeme@1 | Acc]); <<"("/utf8, Rest@4/binary>> -> Lexeme@2 = {open_parens, prequel@span:point(Line, Column)}, do_scan(Rest@4, Line, Column + 1, [Lexeme@2 | Acc]); <<")"/utf8, Rest@5/binary>> -> Lexeme@3 = {close_parens, prequel@span:point(Line, Column)}, do_scan(Rest@5, Line, Column + 1, [Lexeme@3 | Acc]); <<":"/utf8, Rest@6/binary>> -> Lexeme@4 = {colon, prequel@span:point(Line, Column)}, do_scan(Rest@6, Line, Column + 1, [Lexeme@4 | Acc]); <<"-o"/utf8, Rest@7/binary>> -> Lexeme@5 = {circle_lollipop, prequel@span:segment(Line, Column, Column + 1)}, do_scan(Rest@7, Line, Column + 2, [Lexeme@5 | Acc]); <<"-*"/utf8, Rest@8/binary>> -> Lexeme@6 = {star_lollipop, prequel@span:segment(Line, Column, Column + 1)}, do_scan(Rest@8, Line, Column + 2, [Lexeme@6 | Acc]); <<"->"/utf8, Rest@9/binary>> -> Lexeme@7 = {arrow_lollipop, prequel@span:segment(Line, Column, Column + 1)}, do_scan(Rest@9, Line, Column + 2, [Lexeme@7 | Acc]); <<"-"/utf8, Rest@10/binary>> -> Lexeme@8 = {minus, prequel@span:point(Line, Column)}, do_scan(Rest@10, Line, Column + 1, [Lexeme@8 | Acc]); <<"&"/utf8, Rest@11/binary>> -> Lexeme@9 = {ampersand, prequel@span:point(Line, Column)}, do_scan(Rest@11, Line, Column + 1, [Lexeme@9 | Acc]); <<"//"/utf8, Rest@12/binary>> -> {Comment_body, Size, Rest@13} = scan_comment_body( gleam@string_builder:new(), 2, Rest@12 ), Span = prequel@span:segment(Line, Column, (Column + Size) - 1), Lexeme@10 = {{simple_comment, Comment_body}, Span}, do_scan(Rest@13, Line + 1, 1, [Lexeme@10 | Acc]); <<"0"/utf8, _/binary>> -> {Number, Size@1, Rest@14} = scan_number( gleam@string_builder:new(), 0, Source ), Span@1 = prequel@span:segment(Line, Column, (Column + Size@1) - 1), Lexeme@11 = {{number, Number}, Span@1}, do_scan(Rest@14, Line, Column + Size@1, [Lexeme@11 | Acc]); <<"1"/utf8, _/binary>> -> {Number, Size@1, Rest@14} = scan_number( gleam@string_builder:new(), 0, Source ), Span@1 = prequel@span:segment(Line, Column, (Column + Size@1) - 1), Lexeme@11 = {{number, Number}, Span@1}, do_scan(Rest@14, Line, Column + Size@1, [Lexeme@11 | Acc]); <<"2"/utf8, _/binary>> -> {Number, Size@1, Rest@14} = scan_number( gleam@string_builder:new(), 0, Source ), Span@1 = prequel@span:segment(Line, Column, (Column + Size@1) - 1), Lexeme@11 = {{number, Number}, Span@1}, do_scan(Rest@14, Line, Column + Size@1, [Lexeme@11 | Acc]); <<"3"/utf8, _/binary>> -> {Number, Size@1, Rest@14} = scan_number( gleam@string_builder:new(), 0, Source ), Span@1 = prequel@span:segment(Line, Column, (Column + Size@1) - 1), Lexeme@11 = {{number, Number}, Span@1}, do_scan(Rest@14, Line, Column + Size@1, [Lexeme@11 | Acc]); <<"4"/utf8, _/binary>> -> {Number, Size@1, Rest@14} = scan_number( gleam@string_builder:new(), 0, Source ), Span@1 = prequel@span:segment(Line, Column, (Column + Size@1) - 1), Lexeme@11 = {{number, Number}, Span@1}, do_scan(Rest@14, Line, Column + Size@1, [Lexeme@11 | Acc]); <<"5"/utf8, _/binary>> -> {Number, Size@1, Rest@14} = scan_number( gleam@string_builder:new(), 0, Source ), Span@1 = prequel@span:segment(Line, Column, (Column + Size@1) - 1), Lexeme@11 = {{number, Number}, Span@1}, do_scan(Rest@14, Line, Column + Size@1, [Lexeme@11 | Acc]); <<"6"/utf8, _/binary>> -> {Number, Size@1, Rest@14} = scan_number( gleam@string_builder:new(), 0, Source ), Span@1 = prequel@span:segment(Line, Column, (Column + Size@1) - 1), Lexeme@11 = {{number, Number}, Span@1}, do_scan(Rest@14, Line, Column + Size@1, [Lexeme@11 | Acc]); <<"7"/utf8, _/binary>> -> {Number, Size@1, Rest@14} = scan_number( gleam@string_builder:new(), 0, Source ), Span@1 = prequel@span:segment(Line, Column, (Column + Size@1) - 1), Lexeme@11 = {{number, Number}, Span@1}, do_scan(Rest@14, Line, Column + Size@1, [Lexeme@11 | Acc]); <<"8"/utf8, _/binary>> -> {Number, Size@1, Rest@14} = scan_number( gleam@string_builder:new(), 0, Source ), Span@1 = prequel@span:segment(Line, Column, (Column + Size@1) - 1), Lexeme@11 = {{number, Number}, Span@1}, do_scan(Rest@14, Line, Column + Size@1, [Lexeme@11 | Acc]); <<"9"/utf8, _/binary>> -> {Number, Size@1, Rest@14} = scan_number( gleam@string_builder:new(), 0, Source ), Span@1 = prequel@span:segment(Line, Column, (Column + Size@1) - 1), Lexeme@11 = {{number, Number}, Span@1}, do_scan(Rest@14, Line, Column + Size@1, [Lexeme@11 | Acc]); Rest@15 -> {Word, Size@2, Rest@16} = scan_word( gleam@string_builder:new(), 0, Rest@15 ), Lexeme@12 = {{word, Word}, prequel@span:segment(Line, Column, (Column + Size@2) - 1)}, do_scan(Rest@16, Line, Column + Size@2, [Lexeme@12 | Acc]) end. -spec scan(binary()) -> list({token(), prequel@span:span()}). scan(Source) -> do_scan(Source, 1, 1, []). -spec scan_number(gleam@string_builder:string_builder(), integer(), binary()) -> {binary(), integer(), binary()}. scan_number(Acc, Size, Content) -> case Content of <<"0"/utf8, Rest/binary>> -> scan_number( gleam@string_builder:append(Acc, <<"0"/utf8>>), Size + 1, Rest ); <<"1"/utf8, Rest@1/binary>> -> scan_number( gleam@string_builder:append(Acc, <<"1"/utf8>>), Size + 1, Rest@1 ); <<"2"/utf8, Rest@2/binary>> -> scan_number( gleam@string_builder:append(Acc, <<"2"/utf8>>), Size + 1, Rest@2 ); <<"3"/utf8, Rest@3/binary>> -> scan_number( gleam@string_builder:append(Acc, <<"3"/utf8>>), Size + 1, Rest@3 ); <<"4"/utf8, Rest@4/binary>> -> scan_number( gleam@string_builder:append(Acc, <<"4"/utf8>>), Size + 1, Rest@4 ); <<"5"/utf8, Rest@5/binary>> -> scan_number( gleam@string_builder:append(Acc, <<"5"/utf8>>), Size + 1, Rest@5 ); <<"6"/utf8, Rest@6/binary>> -> scan_number( gleam@string_builder:append(Acc, <<"6"/utf8>>), Size + 1, Rest@6 ); <<"7"/utf8, Rest@7/binary>> -> scan_number( gleam@string_builder:append(Acc, <<"7"/utf8>>), Size + 1, Rest@7 ); <<"8"/utf8, Rest@8/binary>> -> scan_number( gleam@string_builder:append(Acc, <<"8"/utf8>>), Size + 1, Rest@8 ); <<"9"/utf8, Rest@9/binary>> -> scan_number( gleam@string_builder:append(Acc, <<"9"/utf8>>), Size + 1, Rest@9 ); Rest@10 -> {gleam@string_builder:to_string(Acc), Size, Rest@10} end.