//// For usage examples and strategies for working with this library, check out
//// the programs located in the `test/examples` directory.
////
//// ## Notes
////
//// - Don't forget to work with the `Input` that is returned by any
//// "inputting" function rather than the original.
//// - If something is marked as being "unspecified", do not depend on it.
//// It may change at any time without a major version bump. This mainly
//// applies to the various `*_to_string` functions.
////
import gleam/bit_array
import gleam/bool
import gleam/dict.{type Dict}
import gleam/int
import gleam/list
import gleam/option.{type Option, None, Some}
import gleam/result
import gleam/string
/// NodeJS Number.MAX_SAFE_INTEGER (ocaml has it as max_int)
const u_eoi = 9_007_199_254_740_991
const u_start_doc = 9_007_199_254_740_990
const u_end_doc = 9_007_199_254_740_989
/// newline
const u_nl: Int = 0x000A
/// carriage return
const u_cr: Int = 0x000D
/// space
const u_space: Int = 0x0020
/// quote
const u_quot: Int = 0x0022
/// #
const u_sharp: Int = 0x0023
/// &
const u_amp: Int = 0x0026
/// '
const u_apos: Int = 0x0027
/// -
const u_minus: Int = 0x002D
/// /
const u_slash: Int = 0x002F
/// :
const u_colon: Int = 0x003A
/// ;
const u_scolon: Int = 0x003B
/// <
const u_lt: Int = 0x003C
///: Int =
const u_eq: Int = 0x003D
/// >
const u_gt: Int = 0x003E
/// ?
const u_qmark: Int = 0x003F
/// !
const u_emark: Int = 0x0021
/// [
const u_lbrack: Int = 0x005B
/// ]
const u_rbrack: Int = 0x005D
/// x
const u_x: Int = 0x0078
/// BOM
const u_bom: Int = 0xFEFF
/// 9
const u_9: Int = 0x0039
/// F
const u_cap_f: Int = 0x0046
/// D
const u_cap_d: Int = 0x0044
const s_cdata: String = "CDATA["
const ns_xml: String = "http://www.w3.org/XML/1998/namespace"
const ns_xmlns: String = "http://www.w3.org/2000/xmlns/"
const n_xml: String = "xml"
const n_xmlns: String = "xmlns"
const n_space: String = "space"
const n_version: String = "version"
const n_encoding: String = "encoding"
const n_standalone: String = "standalone"
const v_yes: String = "yes"
const v_no: String = "no"
const v_preserve: String = "preserve"
const v_default: String = "default"
const v_version_1_0: String = "1.0"
const v_version_1_1: String = "1.1"
const v_utf_8: String = "utf-8"
const v_utf_16: String = "utf-16"
const v_utf_16be: String = "utf-16be"
const v_utf_16le: String = "utf-16le"
const v_iso_8859_1: String = "iso-8859-1"
const v_iso_8859_15: String = "iso-8859-15"
const v_us_ascii: String = "us-ascii"
const v_ascii: String = "ascii"
// =============================================================================
// Unicode character lexers
// =============================================================================
@internal
pub type UnicodeLexerError {
UnicodeLexerEoi
UnicodeLexerMalformed
}
fn make_input_uchar(
uchar_lexer: fn(InputStream) -> Result(#(Int, InputStream), UnicodeLexerError),
) -> fn(Input) -> Result(#(Int, Input), InputError) {
fn(input: Input) {
case uchar_lexer(input.stream) {
Error(e) ->
Error(input_error_new(
input,
internal_input_error_from_unicode_lexer_error(e),
))
Ok(#(uchar, stream)) -> {
let input = Input(..input, stream: stream)
Ok(#(uchar, input))
}
}
}
}
/// Gets the next uchar byte advancing any needed internal state.
fn input_uchar_byte() -> fn(Input) -> Result(#(Int, Input), InputError) {
make_input_uchar(uchar_byte)
}
fn uchar_byte(
stream: InputStream,
) -> Result(#(Int, InputStream), UnicodeLexerError) {
case stream {
[byte, ..rest] -> Ok(#(byte, rest))
[] -> Error(UnicodeLexerEoi)
}
}
fn input_uchar_iso_8859_1() -> fn(Input) -> Result(#(Int, Input), InputError) {
make_input_uchar(uchar_iso_8859_1)
}
fn uchar_iso_8859_1(
stream: InputStream,
) -> Result(#(Int, InputStream), UnicodeLexerError) {
case stream {
[byte, ..rest] -> Ok(#(byte, rest))
[] -> Error(UnicodeLexerEoi)
}
}
fn input_uchar_iso_8859_15() -> fn(Input) -> Result(#(Int, Input), InputError) {
make_input_uchar(uchar_iso_8859_15)
}
// https://www.iana.org/assignments/charset-reg/ISO-8859-15
fn uchar_iso_8859_15(
stream: InputStream,
) -> Result(#(Int, InputStream), UnicodeLexerError) {
case stream {
// €
[0x00A4, ..rest] -> Ok(#(0x20AC, rest))
// Š
[0x00A6, ..rest] -> Ok(#(0x0160, rest))
// š
[0x00A8, ..rest] -> Ok(#(0x0161, rest))
// Ž
[0x00B4, ..rest] -> Ok(#(0x017D, rest))
// ž
[0x00B8, ..rest] -> Ok(#(0x017E, rest))
// Œ
[0x00BC, ..rest] -> Ok(#(0x0152, rest))
// œ
[0x00BD, ..rest] -> Ok(#(0x0153, rest))
// Ÿ
[0x00BE, ..rest] -> Ok(#(0x0178, rest))
// Other
[char, ..rest] -> Ok(#(char, rest))
// Empty stream
[] -> Error(UnicodeLexerEoi)
}
}
fn input_uchar_utf8() -> fn(Input) -> Result(#(Int, Input), InputError) {
make_input_uchar(uchar_utf8)
}
@internal
pub fn uchar_utf8(stream: InputStream) -> Result(
#(Int, InputStream),
UnicodeLexerError,
) {
case stream {
[byte0, ..bytes] -> {
case byte0 {
n if 0 <= n && n <= 127 -> Ok(#(byte0, bytes))
n if 128 <= n && n <= 193 -> Error(UnicodeLexerMalformed)
n if 194 <= n && n <= 223 -> do_uchar_utf8_len2(bytes, byte0)
n if 224 <= n && n <= 239 -> do_uchar_utf8_len3(bytes, byte0)
n if 240 <= n && n <= 244 -> do_uchar_utf8_len4(bytes, byte0)
_ -> Error(UnicodeLexerMalformed)
}
}
[] -> Error(UnicodeLexerEoi)
}
}
fn do_uchar_utf8_len2(
stream: InputStream,
byte0: Int,
) -> Result(#(Int, InputStream), UnicodeLexerError) {
case stream {
[byte1, ..stream] -> {
case byte1 |> most_significant_bytes_are_not_10 {
True -> Error(UnicodeLexerMalformed)
False -> {
let result =
int.bitwise_and(byte0, 0x1F)
|> int.bitwise_shift_left(6)
|> int.bitwise_or(int.bitwise_and(byte1, 0x3F))
Ok(#(result, stream))
}
}
}
[] -> Error(UnicodeLexerEoi)
}
}
fn do_uchar_utf8_len3(
stream: InputStream,
byte0: Int,
) -> Result(#(Int, InputStream), UnicodeLexerError) {
case stream {
[byte1, byte2, ..stream] -> {
case byte2 |> most_significant_bytes_are_not_10 {
True -> Error(UnicodeLexerMalformed)
False -> {
case byte0 == 0xE0 && { byte1 < 0xA0 || 0xBF < byte1 } {
True -> Error(UnicodeLexerMalformed)
False -> {
case byte0 == 0xED && { byte1 < 0x80 || 0x9F < byte1 } {
True -> Error(UnicodeLexerMalformed)
False -> {
case byte1 |> most_significant_bytes_are_not_10 {
True -> Error(UnicodeLexerMalformed)
False -> {
let b0 =
byte0
|> int.bitwise_and(0x0F)
|> int.bitwise_shift_left(12)
let b1 =
byte1
|> int.bitwise_and(0x3F)
|> int.bitwise_shift_left(6)
let b2 = byte2 |> int.bitwise_and(0x3F)
let result =
b0 |> int.bitwise_or(b1) |> int.bitwise_or(b2)
Ok(#(result, stream))
}
}
}
}
}
}
}
}
}
[] -> Error(UnicodeLexerEoi)
_ -> Error(UnicodeLexerMalformed)
}
}
fn do_uchar_utf8_len4(
stream: InputStream,
byte0: Int,
) -> Result(#(Int, InputStream), UnicodeLexerError) {
case stream {
[byte1, byte2, byte3, ..stream] -> {
case
most_significant_bytes_are_not_10(byte3)
|| most_significant_bytes_are_not_10(byte2)
{
True -> Error(UnicodeLexerMalformed)
False -> {
case byte0 == 0xF0 && { byte1 < 0x90 || 0xBF < byte1 } {
True -> Error(UnicodeLexerMalformed)
False -> {
case byte0 == 0xF4 && { byte1 < 0x80 || 0x8F < byte1 } {
True -> Error(UnicodeLexerMalformed)
False -> {
case byte1 |> most_significant_bytes_are_not_10 {
True -> Error(UnicodeLexerMalformed)
False -> {
let b0 =
byte0
|> int.bitwise_and(0x07)
|> int.bitwise_shift_left(18)
let b1 =
byte1
|> int.bitwise_and(0x3F)
|> int.bitwise_shift_left(12)
let b2 =
byte2
|> int.bitwise_and(0x3F)
|> int.bitwise_shift_left(6)
let b3 = byte3 |> int.bitwise_and(0x3F)
let result =
b0
|> int.bitwise_or(b1)
|> int.bitwise_or(b2)
|> int.bitwise_or(b3)
Ok(#(result, stream))
}
}
}
}
}
}
}
}
}
[] -> Error(UnicodeLexerEoi)
_ -> Error(UnicodeLexerMalformed)
}
}
/// Are the most significant bits of an 8 bit int NOT `10`?
fn most_significant_bytes_are_not_10(n: Int) -> Bool {
// this is being used instead of this OCaml code `if b1 lsr 6 != 0b10 then
// raise Malformed else ()`, since gleam doens't offer the logical shift right
//
// NOTE: we could probably use the arithmetic shift as we would only be using
// positive numbers--might be worth switching.
int.bitwise_and(n, 0b11000000) != 0b10000000
}
fn int16_be(
stream: InputStream,
) -> Result(#(Int, InputStream), UnicodeLexerError) {
case stream {
[byte0, byte1, ..stream] -> {
let char = byte0 |> int.bitwise_shift_left(8) |> int.bitwise_or(byte1)
Ok(#(char, stream))
}
[] -> Error(UnicodeLexerEoi)
_ -> Error(UnicodeLexerMalformed)
}
}
fn int16_le(
stream: InputStream,
) -> Result(#(Int, InputStream), UnicodeLexerError) {
case stream {
[byte0, byte1, ..stream] -> {
let char = byte1 |> int.bitwise_shift_left(8) |> int.bitwise_or(byte0)
Ok(#(char, stream))
}
[] -> Error(UnicodeLexerEoi)
_ -> Error(UnicodeLexerMalformed)
}
}
fn uchar_utf16(
int16,
) -> fn(InputStream) -> Result(#(Int, InputStream), UnicodeLexerError) {
fn(stream) {
case int16(stream) {
Error(e) -> Error(e)
Ok(#(char0, stream)) -> {
case char0 {
char0 if char0 < 0xD800 || char0 > 0xDFFF -> Ok(#(char0, stream))
char0 if char0 > 0xDBFF -> Error(UnicodeLexerMalformed)
char0 -> {
case int16(stream) {
Error(e) -> Error(e)
Ok(#(char1, stream)) -> {
let char =
int.bitwise_or(
int.bitwise_shift_left(int.bitwise_and(char0, 0x3FF), 10),
int.bitwise_and(char1, 0x3FF),
)
+ 0x10000
Ok(#(char, stream))
}
}
}
}
}
}
}
}
fn input_uchar_utf16be() -> fn(Input) -> Result(#(Int, Input), InputError) {
make_input_uchar(uchar_utf16(int16_be))
}
fn input_uchar_utf16le() -> fn(Input) -> Result(#(Int, Input), InputError) {
make_input_uchar(uchar_utf16(int16_le))
}
fn input_uchar_ascii() -> fn(Input) -> Result(#(Int, Input), InputError) {
make_input_uchar(uchar_ascii)
}
fn uchar_ascii(
stream: InputStream,
) -> Result(#(Int, InputStream), UnicodeLexerError) {
case stream {
[byte, ..rest] if byte <= 127 -> Ok(#(byte, rest))
[] -> Error(UnicodeLexerEoi)
_ -> Error(UnicodeLexerMalformed)
}
}
// =============================================================================
// Basic types and values
// =============================================================================
/// The type for character encodings
///
pub type Encoding {
Utf8
/// UTF-16 endianness is determined from the
/// [BOM](https://www.unicode.org/faq/utf_bom.html#BOM).
///
Utf16
/// UTF-16 big-endian
///
Utf16Be
/// UTF-16 big-endian
///
Utf16Le
Iso8859x1
Iso8859x15
UsAscii
}
/// Type for names of attribute and elements. An empty `uri` represents a
/// name without a namespace, i.e., an unprefixed name that is not under the
/// scope of a default namespace.
///
pub type Name {
Name(
// NOTE! Internally the URI is actually still a prefix for much of the code
// until it is mapped.
/// The URI of the `Name`.
///
/// Note that this likely* will not be the literal value of the prefix
/// string before the `:`. E.g.,
///
/// ```xml
///
///
///
/// ```
///
/// The `b` tag would look something like this:
///
/// ```gleam
/// Tag(
/// name: Name(uri: "https://www.example.com/snazzy", local: "b"),
/// attributes: []
/// )
/// ```
///
/// Note how the `uri` is not `"snazzy"`, but
/// `"https://www.example.com/snazzy"`.
///
/// *I say "likely", because you could define a `namespace_callback` that
/// maps URIs to themselves rather than a URI.
///
uri: String,
/// The non-prefixed (i.e., `local`) part of the `Name`.
///
local: String,
)
}
/// Convert `name` into an unspecified string representation.
///
pub fn name_to_string(name: Name) -> String {
let Name(uri, local) = name
case uri {
"" -> string.inspect(local)
uri -> string.inspect(uri <> ":" <> local)
}
}
/// Type for attributes.
///
/// ## Example
///
/// In following XML fragment ``, the attribute
/// `color="green"` would look like this:
///
/// ```gleam
/// Attribute(name: Name(uri: "", local: "color"), value: "green")
/// ```
///
pub type Attribute {
Attribute(
/// The `name` of the `Attribute`
name: Name,
/// The `value` of the `Attribute`
value: String,
)
}
/// Convert `attribute` into an unspecified string representation.
///
pub fn attribute_to_string(attribute: Attribute) -> String {
"Attribute(name: "
<> name_to_string(attribute.name)
<> ", value: "
<> string.inspect(attribute.value)
<> ")"
}
/// Convert `attributes` into an unspecified string representation.
///
pub fn attributes_to_string(attributes: List(Attribute)) -> String {
case attributes {
[] -> "[]"
attributes ->
"[" <> string.join(list.map(attributes, attribute_to_string), ", ") <> "]"
}
}
/// The type for an element tag.
///
pub type Tag {
Tag(
/// Name of the tag
///
name: Name,
/// Attribute list of the tag
///
attributes: List(Attribute),
)
}
/// Convert `tag` into an unspecified string representation.
///
pub fn tag_to_string(tag: Tag) -> String {
"Tag(name: "
<> name_to_string(tag.name)
<> ", attributes: "
<> attributes_to_string(tag.attributes)
<> ")"
}
/// The type for signals
///
/// A well-formed sequence of signals belongs to the language of the document
/// grammar:
///
/// ```
/// document := Dtd tree ;
/// tree := ElementStart child ElementEnd ;
/// child := ( Data trees ) | trees ;
/// trees := ( tree child ) | epsilon ;
/// ```
///
/// Note the `trees` production which expresses the fact there there will never
/// be two consecutive `Data` signals in the children of an element.
///
/// The `Input` type and functions that work with it deal only with well-formed
/// signal sequences, else `Errors` are returned.
///
pub type Signal {
Dtd(Option(String))
ElementStart(Tag)
ElementEnd
Data(String)
}
/// Convert `signal` into an unspecified string representation.
///
pub fn signal_to_string(signal: Signal) -> String {
case signal {
Dtd(Some(data)) -> "Dtd(" <> data <> ")"
Dtd(None) -> "Dtd(None)"
ElementStart(tag) -> "ElementStart(" <> tag_to_string(tag) <> ")"
ElementEnd -> "ElementEnd"
Data(data) -> "Data(" <> string.inspect(data) <> ")"
}
}
/// Convert `signals` into an unspecified string representation.
///
pub fn signals_to_string(signals: List(Signal)) -> String {
string.join(list.map(signals, signal_to_string), "\n")
}
fn signal_start_stream() {
Data("")
}
// =============================================================================
// Input
// =============================================================================
/// The type for error positions
///
type Position {
Position(line: Int, column: Int)
}
/// Convert `position` into an unspecified string representation.
///
fn position_to_string(position: Position) -> String {
"Position(line: "
<> int.to_string(position.line)
<> ", column: "
<> int.to_string(position.column)
<> ")"
}
type InternalInputError {
ExpectedCharSeqs(expected: List(String), actual: String)
ExpectedRootElement
IllegalCharRef(String)
IllegalCharSeq(String)
MalformedCharStream
MaxBufferSize
UnexpectedEoi
UnknownEncoding(String)
UnknownEntityRef(String)
UnknownNsPrefix(String)
// New
UnicodeLexerErrorEoi
UnicodeLexerErrorMalformed
InvalidArgument(String)
}
fn internal_input_error_from_unicode_lexer_error(
unicode_lexer_error: UnicodeLexerError,
) -> InternalInputError {
case unicode_lexer_error {
UnicodeLexerEoi -> UnicodeLexerErrorEoi
UnicodeLexerMalformed -> UnicodeLexerErrorMalformed
}
}
fn internal_error_message(input_error: InternalInputError) -> String {
let bracket = fn(l, v, r) { l <> v <> r }
case input_error {
ExpectedCharSeqs(expected, actual) -> {
let expected =
list.fold(expected, "", fn(acc, v) { acc <> bracket("\"", v, "\", ") })
"expected one of these character sequence: "
<> expected
<> "found \""
<> actual
<> "\""
}
ExpectedRootElement -> "expected root element"
IllegalCharRef(msg) -> bracket("illegal character reference (#", msg, ")")
IllegalCharSeq(msg) ->
bracket("character sequence illegal here (\"", msg, "\")")
MalformedCharStream -> "malformed character stream"
MaxBufferSize -> "maximal buffer size exceeded"
UnexpectedEoi -> "unexpected end of input"
UnknownEncoding(msg) -> bracket("unknown encoding (", msg, ")")
UnknownEntityRef(msg) -> bracket("unknown entity reference (", msg, ")")
UnknownNsPrefix(msg) -> bracket("unknown namespace prefix (", msg, ")")
// New
UnicodeLexerErrorEoi -> "unicode lexer error eoi"
UnicodeLexerErrorMalformed -> "unicode lexer error malformed"
InvalidArgument(msg) -> bracket("invalid argument (", msg, ")")
}
}
// Note: this in an exception in the ocaml code
/// The type of error returned by any "inputing" functions.
///
pub opaque type InputError {
InputError(Position, InternalInputError)
}
fn input_error_new(input: Input, input_error: InternalInputError) -> InputError {
InputError(Position(line: input.line, column: input.column), input_error)
}
/// Converts the `input_error` into a non-specified human readable format.
///
pub fn input_error_to_string(input_error: InputError) -> String {
let InputError(position, input_error) = input_error
"ERROR "
<> position_to_string(position)
<> " "
<> internal_error_message(input_error)
}
/// Limits of "things" in the XML
type Limit {
/// '<' qname
LimitStartTag(Name)
/// '' qname whitespace*
LimitEndTag(Name)
/// '' qname (processing instruction)
LimitPi(Name)
/// '