import eyg/parser/token as t import gleam/bit_array import gleam/list import gleam/string pub fn lex(raw) { loop(raw, 0, []) } fn loop(raw, offset, acc) { case pop(raw, offset) { Ok(#(token, offset, rest)) -> loop(rest, offset, [token, ..acc]) Error(Nil) -> list.reverse(acc) } } fn done(at start) { fn(t, size, rest) { Ok(#(#(t, start), start + size, rest)) } } fn pop(raw, start) { let done = done(at: start) // If we track whitespace token then can always return original start case raw { "//" <> rest -> comment("", rest, done) "\r\n" <> rest -> whitespace("\r\n", rest, done) "\n" <> rest -> whitespace("\n", rest, done) " " <> rest -> whitespace(" ", rest, done) "\t" <> rest -> whitespace("\t", rest, done) "(" <> rest -> done(t.LeftParen, 1, rest) ")" <> rest -> done(t.RightParen, 1, rest) "{" <> rest -> done(t.LeftBrace, 1, rest) "}" <> rest -> done(t.RightBrace, 1, rest) "[" <> rest -> done(t.LeftSquare, 1, rest) "]" <> rest -> done(t.RightSquare, 1, rest) "=" <> rest -> done(t.Equal, 1, rest) "->" <> rest -> done(t.RightArrow, 2, rest) "," <> rest -> done(t.Comma, 1, rest) ".." <> rest -> done(t.DotDot, 2, rest) "." <> rest -> done(t.Dot, 1, rest) ":" <> rest -> done(t.Colon, 1, rest) "-" <> rest -> done(t.Minus, 1, rest) "!" <> rest -> done(t.Bang, 1, rest) "|" <> rest -> done(t.Bar, 1, rest) "#" <> rest -> done(t.Hash, 1, rest) "@" <> rest -> done(t.At, 1, rest) "\"" <> rest -> string("", 1, rest, done) "1" <> rest -> integer("1", rest, done) "2" <> rest -> integer("2", rest, done) "3" <> rest -> integer("3", rest, done) "4" <> rest -> integer("4", rest, done) "5" <> rest -> integer("5", rest, done) "6" <> rest -> integer("6", rest, done) "7" <> rest -> integer("7", rest, done) "8" <> rest -> integer("8", rest, done) "9" <> rest -> integer("9", rest, done) "0" <> rest -> integer("0", rest, done) _ -> { let next_byte = byte_slice_range(raw, 0, 1) let rest = byte_slice_from(raw, 1) case next_byte { "_" | "a" | "b" | "c" | "d" | "e" | "f" | "g" | "h" | "i" | "j" | "k" | "l" | "m" | "n" | "o" | "p" | "q" | "r" | "s" | "t" | "u" | "v" | "w" | "x" | "y" | "z" -> name(next_byte, rest, done) "A" | "B" | "C" | "D" | "E" | "F" | "G" | "H" | "I" | "J" | "K" | "L" | "M" | "N" | "O" | "P" | "Q" | "R" | "S" | "T" | "U" | "V" | "W" | "X" | "Y" | "Z" -> uppername(next_byte, rest, done) "" -> Error(Nil) _ -> done(t.UnexpectedGrapheme(raw), string.byte_size(raw), "") } } } } fn comment(buffer, rest, done) { case rest { // leave newline on stack to end up as witespace "\r\n" <> _ | "\n" <> _ -> done(t.Comment(buffer), string.byte_size(buffer) + 2, rest) _ -> { let next_byte = byte_slice_range(rest, 0, 1) let rest = byte_slice_from(rest, 1) comment(buffer <> next_byte, rest, done) } } } fn whitespace(buffer, rest, done) { case rest { "\r\n" <> rest -> whitespace(buffer <> "\r\n", rest, done) "\n" <> rest -> whitespace(buffer <> "\n", rest, done) " " <> rest -> whitespace(buffer <> " ", rest, done) "\t" <> rest -> whitespace(buffer <> "\t", rest, done) _ -> done(t.Whitespace(buffer), string.byte_size(buffer), rest) } } fn string(buffer, length, rest, done) { case rest { "\"" <> rest -> done(t.String(buffer), length + 1, rest) "\\" <> rest -> case rest { "\"" <> rest -> string(buffer <> "\"", length + 2, rest, done) "\\" <> rest -> string(buffer <> "\\", length + 2, rest, done) "t" <> rest -> string(buffer <> "\t", length + 2, rest, done) "r" <> rest -> string(buffer <> "\r", length + 2, rest, done) "n" <> rest -> string(buffer <> "\n", length + 2, rest, done) "" -> done(t.UnterminatedString(buffer <> "\\"), length + 1, "") _ -> done(t.InvalidEscape(buffer <> "\\" <> rest), length, "") } "" -> done(t.UnterminatedString(buffer), length, "") _ -> { let next_byte = byte_slice_range(rest, 0, 1) let rest = byte_slice_from(rest, 1) string(buffer <> next_byte, length + 1, rest, done) } } } fn name(buffer, raw, done) { let next_byte = byte_slice_range(raw, 0, 1) let rest = byte_slice_from(raw, 1) case is_lower_grapheme(next_byte) || is_digit_grapheme(next_byte) || next_byte == "_" { True -> name(buffer <> next_byte, rest, done) False -> done(keyword_or_name(buffer), string.byte_size(buffer), raw) } } fn keyword_or_name(buffer) { case buffer { "let" -> t.Let "match" -> t.Match "perform" -> t.Perform "deep" -> t.Deep "handle" -> t.Handle "import" -> t.Import _ -> t.Name(buffer) } } fn uppername(buffer, raw, done) { let next_byte = byte_slice_range(raw, 0, 1) let rest = byte_slice_from(raw, 1) case is_upper_grapheme(next_byte) || is_lower_grapheme(next_byte) || is_digit_grapheme(next_byte) || next_byte == "_" { True -> uppername(buffer <> next_byte, rest, done) False -> done(t.Uppername(buffer), string.byte_size(buffer), raw) } } fn integer(buffer, rest, done) { case rest { "1" <> rest -> integer(buffer <> "1", rest, done) "2" <> rest -> integer(buffer <> "2", rest, done) "3" <> rest -> integer(buffer <> "3", rest, done) "4" <> rest -> integer(buffer <> "4", rest, done) "5" <> rest -> integer(buffer <> "5", rest, done) "6" <> rest -> integer(buffer <> "6", rest, done) "7" <> rest -> integer(buffer <> "7", rest, done) "8" <> rest -> integer(buffer <> "8", rest, done) "9" <> rest -> integer(buffer <> "9", rest, done) "0" <> rest -> integer(buffer <> "0", rest, done) _ -> done(t.Integer(buffer), string.byte_size(buffer), rest) } } fn is_lower_grapheme(grapheme) { case grapheme { "a" | "b" | "c" | "d" | "e" | "f" | "g" | "h" | "i" | "j" | "k" | "l" | "m" | "n" | "o" | "p" | "q" | "r" | "s" | "t" | "u" | "v" | "w" | "x" | "y" | "z" -> True _ -> False } } fn is_upper_grapheme(grapheme) { case grapheme { "A" | "B" | "C" | "D" | "E" | "F" | "G" | "H" | "I" | "J" | "K" | "L" | "M" | "N" | "O" | "P" | "Q" | "R" | "S" | "T" | "U" | "V" | "W" | "X" | "Y" | "Z" -> True _ -> False } } fn is_digit_grapheme(grapheme) { case grapheme { "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "0" -> True _ -> False } } fn byte_slice_range(raw, from, to) { case bit_array.from_string(raw) { <<>> -> "" <<_:bytes-size(from), slice:bytes-size(to - from), _:bytes>> -> { let assert Ok(slice) = bit_array.to_string(slice) slice } _ -> panic } } fn byte_slice_from(raw, from) { case bit_array.from_string(raw) { <<>> -> "" <<_:bytes-size(from), slice:bytes>> -> { let assert Ok(slice) = bit_array.to_string(slice) slice } _ -> panic } }