Tokens emitted by the Tokenizers.
Summary
Types
@type close_annotation_token() :: {:close_annotation, unused :: any(), token_meta()}
@type close_block_token() :: {:close_block, unused :: any(), token_meta()}
@type comment_token() :: {:comment, {comment_type(), String.t()}, token_meta()}
@type comment_type() :: :line | :span | :multiline
@type dquote_string_token() :: {:dquote_string, String.t(), token_meta()}
@type equal_token() :: {:=, unused :: any(), token_meta()}
@type fold_token() :: {:fold, unused :: any(), token_meta()}
@type newline_token() :: {:nl, unused :: any(), token_meta()}
@type open_annotation_token() :: {:open_annotation, unused :: any(), token_meta()}
@type open_block_token() :: {:open_block, unused :: any(), token_meta()}
@type raw_string_token() :: {:raw_string, String.t(), token_meta()}
@type semicolon_token() :: {:sc, unused :: any(), token_meta()}
@type slashdash_token() :: {:slashdash, unused :: any(), token_meta()}
@type space_token() :: {:space, {String.t(), len :: non_neg_integer()}, token_meta()}
@type term_token() :: {:term, String.t(), token_meta()}