Definitions.

YEAR = [0-9][0-9][0-9][0-9]

MONTH = [0-9][0-9]

DAY = [0-9][0-9]

HOUR = (([01][0-9])|(2[0-3]))

MINUTE = ([0-5][0-9])

SECOND = ([0-5][0-9](\.[0-9]+)?)

FLOAT_EXP = ([Ee][+\-]?[0-9]+)

DECIMAL_EXP = ([Dd][+\-]?[0-9]+)

DEC_FRAC = ((\.[0-9](_?[0-9])*)|\.)

NON_ZERO_DECIMAL = ([1-9](_?[0-9])*)

HEX_DIGIT = [0-9a-fA-F]

WS = (\t|\v|\f|\s|\r\n|\r|\n)

NL = (\r\n|\r|\n)

SYMBOL_TEXT_ALLOWED = (([\x{0020}-\x{0026}])|([\x{0028}-\x{005B}])|([\x{005D}-\x{10FFFF}])|(\t|\v|\f|\s))

STRING_SHORT_TEXT_ALLOWED = (([\x{0020}-\x{0021}])|([\x{0023}-\x{005B}])|([\x{005D}-\x{10FFFF}])|(\t|\v|\f|\s))

STRING_LONG_TEXT_ALLOWED = (([\x{0020}-\x{0026}])|([\x{0028}-\x{005B}])|([\x{005D}-\x{10FFFF}])|(\t|\v|\f|\s|\n\r|\n|\r))

COMMON_ESCAPE = (\\(a|b|t|n|f|r|v|\?|0|\'|\"|/|\\|{NL}))

HEX_ESCAPE = (\\x{HEX_DIGIT}{HEX_DIGIT})

UNICODE_ESCAPE = (\\u{HEX_DIGIT}{HEX_DIGIT}{HEX_DIGIT}{HEX_DIGIT})|(\\U000{HEX_DIGIT}{HEX_DIGIT}{HEX_DIGIT}{HEX_DIGIT}{HEX_DIGIT})|(\\U0010{HEX_DIGIT}{HEX_DIGIT}{HEX_DIGIT}{HEX_DIGIT})

TEXT_ESCAPE = ({COMMON_ESCAPE}|{HEX_ESCAPE}|{UNICODE_ESCAPE})

CLOB_ESCAPE = ({COMMON_ESCAPE}|{HEX_ESCAPE})

BASE_64_CHAR = ([0-9a-zA-Z+/]{WS}*)

BASE_64_QUARTET = ({BASE_64_CHAR}{BASE_64_CHAR}{BASE_64_CHAR}{BASE_64_CHAR})

BASE_64_PAD_1 = ({BASE_64_CHAR}{BASE_64_CHAR}{BASE_64_CHAR}=)

BASE_64_PAD_2 = ({BASE_64_CHAR}{BASE_64_CHAR}={WS}*=)

LOB_START = (\{\{)
LOB_END = (\}\})
SYMBOL_QUOTE = \'
SHORT_QUOTE = \"
LONG_QUOTE = (\'\'\')

Rules.

{WS}+ : {token, {whitespace, TokenLine, TokenChars}}.

%% Inline comment: // to end of line (or EOF)
//[^\r\n]* : {token, {comment, TokenLine, TokenChars}}.

%% Block comment: /* ... */ (may span multiple lines)
/\*([^*]|\*+[^*/])*\*+/ : {token, {comment, TokenLine, TokenChars}}.

null\.(null|bool|int|float|decimal|timestamp|symbol|string|clob|blob|list|sexp|struct) :
    {token,{typed_null,TokenLine,TokenChars}}.

null\.[a-zA-Z]+ :
    {error, "invalid typed null: " ++ TokenChars}.

null\. :
    {error, "incomplete typed null: " ++ TokenChars}.

null :
    {token,{null,TokenLine}}.

bool|int|float|decimal|timestamp|symbol|string|clob|blob|list|sexp|struct :
    {token,{type,TokenLine,TokenChars}}.

true|false :
    {token,{bool,TokenLine,TokenChars}}.

{YEAR}-{MONTH}-{DAY}(T({HOUR}:{MINUTE}(:{SECOND})?(Z|([+\-]{HOUR}:{MINUTE}))?)?)?|{YEAR}-{MONTH}T|{YEAR}T :
    {token,{timestamp,TokenLine,TokenChars}}. % convert

-?0[bB][01](_?[01])* :
    {token,{bin_int,TokenLine,TokenChars}}. % convert

-?(0|{NON_ZERO_DECIMAL}) :
    {token,{dec_int,TokenLine,TokenChars}}. % convert, strip _ and then list_to_integer/1

-?0[xX]{HEX_DIGIT}(_?{HEX_DIGIT})* :
    {token,{hex_int,TokenLine,TokenChars}}. % convert

[+\-]inf|nan :
    {token,{s_float,TokenLine,TokenChars}}. % convert

-?(0|{NON_ZERO_DECIMAL}){DEC_FRAC}?{FLOAT_EXP} :
    {token,{float,TokenLine,TokenChars}}. % convert

-?(0|{NON_ZERO_DECIMAL}){DEC_FRAC}?{DECIMAL_EXP}? :
    {token,{decimal,TokenLine,TokenChars}}. % convert

{SYMBOL_QUOTE}({TEXT_ESCAPE}|{SYMBOL_TEXT_ALLOWED})*{SYMBOL_QUOTE} :
    {token,{quoted_symbol,TokenLine,TokenChars}}. % convert

\$[0-9]+ :
    {token,{symbol_id,TokenLine,TokenChars}}.

[$_a-zA-Z](([$_a-zA-Z])|([0-9]))* :
    {token,{id_symbol,TokenLine,TokenChars}}. % convert

{SHORT_QUOTE}({TEXT_ESCAPE}|{STRING_SHORT_TEXT_ALLOWED})*{SHORT_QUOTE} :
    {token,{short_string,TokenLine,strip_quotes(TokenChars)}}.

{LONG_QUOTE}(\'?\'?({TEXT_ESCAPE}|{STRING_LONG_TEXT_ALLOWED}))*{LONG_QUOTE} :
    {token,{long_string,TokenLine,strip_quotes(TokenChars)}}.

{LOB_START}{WS}*{SHORT_QUOTE}({CLOB_ESCAPE}|[\x{0020}-\x{0021}]|[\x{0023}-\x{005b}]|[\x{005d}-\x{007f}]|\t|\v|\f|\s)*{SHORT_QUOTE}{WS}*{LOB_END} :
    {token,{short_clob,TokenLine,TokenChars}}. % convert

{LOB_START}({WS}*{LONG_QUOTE}(\'?\'?({CLOB_ESCAPE}|([\x{0020}-\x{0026}]|[\x{0028}-\x{005b}]|[\x{005d}-\x{007f}]|(\t|\v|\f|\s|\n\r|\n|\r))))*{LONG_QUOTE})+{WS}*{LOB_END} :
    {token,{long_clob,TokenLine,TokenChars}}. % convert

{LOB_START}[0-9a-zA-Z+/=\t\v\f\s\r\n]*{LOB_END} :
    {token,{blob,TokenLine,TokenChars}}. % convert

\[ : {token,{'[', TokenLine}}.
\] : {token,{']', TokenLine}}.
\(\: : {token,{macro_start, TokenLine}}.
\( : {token,{'(', TokenLine}}.
\) : {token,{')', TokenLine}}.
\{ : {token,{'{', TokenLine}}.
\} : {token,{'}', TokenLine}}.
\, : {token,{',', TokenLine}}.
\:\: : {token,{'::', TokenLine}}.
\: : {token,{':', TokenLine}}.
\. : {token,{'.', TokenLine}}.

[\!\#\%\&\*\+\-\/\;\<\=\>\?\@\^\`\|\~] : {token, {non_dot_operator, TokenLine, TokenChars}}.

Erlang code.

%% Strip surrounding quotes from string token chars.
strip_quotes([$" | Rest]) ->
    lists:droplast(Rest);
strip_quotes([$', $', $' | Rest]) ->
    lists:droplast(lists:droplast(lists:droplast(Rest))).