defmodule MDEx do
@external_resource "README.md"
@inner_moduledoc """
## Parsing
Converts Markdown to an AST data structure that can be inspected and manipulated to change the content of the document programmatically.
The data structure format is inspired on [Floki](https://github.com/philss/floki) (with `:attributes_as_maps = true`) so we can keep similar APIs and keep the same mental model when
working with these documents, either Markdown or HTML, where each node is represented as a struct holding the node name as the struct name and its attributes and children, for eg:
%MDEx.Heading{
level: 1
nodes: [...],
}
The parent node that represents the root of the document is the `MDEx.Document` struct,
where you can find more more information about the AST and what operations are available.
The complete list of nodes is listed in the the section `Document Nodes`.
## Formatting
Formatting is the process of converting from one format to another, for example from AST or Markdown to HTML.
Formatting to XML and to Markdown is also supported.
You can use `MDEx.parse_document/2` to generate an AST or any of the `to_*` functions to convert to Markdown (CommonMark), HTML, JSON, or XML.
"""
@moduledoc "README.md"
|> File.read!()
|> String.split("")
|> Enum.fetch!(1)
|> Kernel.<>(@inner_moduledoc)
alias MDEx.Native
alias MDEx.Document
alias MDEx.DecodeError
alias MDEx.InvalidInputError
import MDEx.Document, only: [is_fragment: 1]
require Logger
@typedoc """
Input source document.
## Examples
* From Markdown to HTML
iex> MDEx.to_html!("# Hello")
"
Hello
"
* From Markdown to `MDEx.Document`
iex> MDEx.parse_document!("Hello")
%MDEx.Document{
nodes: [
%MDEx.Paragraph{nodes: [%MDEx.Text{literal: "Hello"}]}
]
}
* From `MDEx.Document` to HTML
iex> MDEx.to_html!(%MDEx.Document{
...> nodes: [
...> %MDEx.Paragraph{nodes: [%MDEx.Text{literal: "Hello"}]}
...> ]
...> })
"
Hello
"
You can also leverage `MDEx.Document` as an intermediate data type to convert between formats:
* From JSON to HTML:
iex> json = ~s|{"nodes":[{"nodes":[{"literal":"Hello","node_type":"MDEx.Text"}],"level":1,"setext":false,"node_type":"MDEx.Heading"}],"node_type":"MDEx.Document"}|
iex> {:json, json} |> MDEx.parse_document!() |> MDEx.to_html!()
"
Hello
"
"""
@type source :: markdown :: String.t() | Document.t()
@deprecated "Use `MDEx.Document.default_extension_options/0` instead"
def default_extension_options, do: Document.default_extension_options()
@deprecated "Use `MDEx.Document.default_parse_options/0` instead"
def default_parse_options, do: Document.default_parse_options()
@deprecated "Use `MDEx.Document.default_render_options/0` instead"
def default_render_options, do: Document.default_render_options()
@deprecated "Use `MDEx.Document.default_syntax_highlight_options/0` instead"
def default_syntax_highlight_options, do: Document.default_syntax_highlight_options()
@deprecated "Use `MDEx.Document.default_sanitize_options/0` instead"
def default_sanitize_options, do: Document.default_sanitize_options()
@doc """
Parse `source` and returns `MDEx.Document`.
Source can be either a Markdown string or a tagged JSON string.
## Examples
* Parse Markdown with default options:
iex> MDEx.parse_document!(\"""
...> # Languages
...>
...> - Elixir
...> - Rust
...> \""")
%MDEx.Document{
nodes: [
%MDEx.Heading{nodes: [%MDEx.Text{literal: "Languages"}], level: 1, setext: false},
%MDEx.List{
nodes: [
%MDEx.ListItem{
nodes: [%MDEx.Paragraph{nodes: [%MDEx.Text{literal: "Elixir"}]}],
list_type: :bullet,
marker_offset: 0,
padding: 2,
start: 1,
delimiter: :period,
bullet_char: "-",
tight: false
},
%MDEx.ListItem{
nodes: [%MDEx.Paragraph{nodes: [%MDEx.Text{literal: "Rust"}]}],
list_type: :bullet,
marker_offset: 0,
padding: 2,
start: 1,
delimiter: :period,
bullet_char: "-",
tight: false
}
],
list_type: :bullet,
marker_offset: 0,
padding: 2,
start: 1,
delimiter: :period,
bullet_char: "-",
tight: true
}
]
}
* Parse Markdown with custom options:
iex> MDEx.parse_document!("Darth Vader is ||Luke's father||", extension: [spoiler: true])
%MDEx.Document{
nodes: [
%MDEx.Paragraph{
nodes: [
%MDEx.Text{literal: "Darth Vader is "},
%MDEx.SpoileredText{nodes: [%MDEx.Text{literal: "Luke's father"}]}
]
}
]
}
* Parse JSON:
iex> json = ~s|{"nodes":[{"nodes":[{"literal":"Title","node_type":"MDEx.Text"}],"level":1,"setext":false,"node_type":"MDEx.Heading"}],"node_type":"MDEx.Document"}|
iex> MDEx.parse_document!({:json, json})
%MDEx.Document{
nodes: [
%MDEx.Heading{
nodes: [%MDEx.Text{literal: "Title"} ],
level: 1,
setext: false
}
]
}
"""
@spec parse_document(markdown :: String.t() | {:json, String.t()}, MDEx.Document.options()) :: {:ok, Document.t()} | {:error, any()}
# TODO: support :xml
def parse_document(source, options \\ [])
def parse_document(markdown, options) when is_binary(markdown) do
document =
options
|> Keyword.put_new(:markdown, markdown)
|> MDEx.new()
{:ok, document}
end
def parse_document({:json, json}, options) when is_binary(json) do
case Jason.decode(json, keys: :atoms!) do
{:ok, decoded} ->
document = %{MDEx.new(options) | nodes: json_to_node(decoded).nodes}
{:ok, document}
{:error, error} ->
{:error, %DecodeError{error: error}}
end
end
defp json_to_node(json) do
{node_type, node} = Map.pop!(json, :node_type)
node_type = Module.concat([node_type])
node = map_nodes(node)
struct(node_type, node)
end
defp map_nodes(%{nodes: nodes} = node) do
%{node | nodes: Enum.map(nodes, &json_to_node/1)}
end
defp map_nodes(node), do: node
@doc """
Same as `parse_document/2` but raises if the parsing fails.
"""
@spec parse_document!(markdown :: String.t() | {:json, String.t()}, MDEx.Document.options()) :: Document.t()
def parse_document!(source, options \\ []) do
case parse_document(source, options) do
{:ok, document} -> document
{:error, error} -> raise error
end
end
@doc """
Parse a `markdown` string and returns only the node that represents the fragment.
Usually that means filtering out the parent document and paragraphs.
That's useful to generate fragment nodes and inject them into the document
when you're manipulating it.
Use `parse_document/2` to generate a complete document.
> #### Experimental {: .warning}
>
> Consider this function experimental and subject to change.
## Examples
iex> MDEx.parse_fragment("# Elixir")
{:ok, %MDEx.Heading{nodes: [%MDEx.Text{literal: "Elixir"}], level: 1, setext: false}}
iex> MDEx.parse_fragment("
\\n"}}
"""
@spec parse_fragment(String.t(), MDEx.Document.options()) :: {:ok, Document.md_node()} | nil
def parse_fragment(markdown, options \\ []) when is_binary(markdown) do
case parse_document(markdown, options) do
{:ok, %Document{nodes: [%MDEx.Paragraph{nodes: [node]}]}} -> {:ok, node}
{:ok, %Document{nodes: [node]}} -> {:ok, node}
_ -> nil
end
end
@doc """
Same as `parse_fragment/2` but raises if the parsing fails or returns `nil`.
> #### Experimental {: .warning}
>
> Consider this function experimental and subject to change.
"""
@spec parse_fragment!(String.t(), MDEx.Document.options()) :: Document.md_node()
def parse_fragment!(markdown, options \\ []) when is_binary(markdown) do
case parse_fragment(markdown, options) do
{:ok, fragment} -> fragment
_ -> raise %InvalidInputError{found: markdown}
end
end
@doc """
Convert Markdown or `MDEx.Document` to HTML.
## Examples
iex> MDEx.to_html("# MDEx")
{:ok, "
{:ok, 'MDEx'}"
## Options
- `:sanitize` - cleans HTML after rendering. Defaults to `MDEx.Document.default_sanitize_options()/0`.
- `keyword` - `t:sanitize_options/0`
- `nil` - do not sanitize output.
- `:escape` - which entities should be escaped. Defaults to `[:content, :curly_braces_in_code]`.
- `:content` - escape common chars like `<`, `>`, `&`, and others in the HTML content;
- `:curly_braces_in_code` - escape `{` and `}` only inside `` tags, particularly useful for compiling HTML in LiveView;
"""
@spec safe_html(
String.t(),
options :: [
sanitize: MDEx.Document.sanitize_options() | nil,
escape: [atom()]
]
) :: String.t()
def safe_html(unsafe_html, options \\ []) when is_binary(unsafe_html) and is_list(options) do
sanitize =
options
|> opt([:sanitize], Document.default_sanitize_options())
|> case do
nil ->
nil
options ->
options
|> NimbleOptions.validate!(MDEx.Document.sanitize_options_schema())
|> MDEx.Document.adapt_sanitize_options()
end
escape_content = opt(options, [:escape, :content], true)
escape_curly_braces_in_code = opt(options, [:escape, :curly_braces_in_code], true)
Native.safe_html(unsafe_html, sanitize, escape_content, escape_curly_braces_in_code)
end
# if Code.ensure_loaded?(Phoenix.LiveView.Rendered) do
# @doc """
# Utility function to convert a `Phoenix.LiveView.Rendered` struct to HTML (string).
#
# ## Example
#
# iex> assigns = %{url: "https://elixir-lang.org", title: "Elixir Lang"}
# iex> ~MD|<.link href={URI.parse(@url)}>{@title}|HEEX |> MDEx.rendered_to_html()
# "Elixir"
# """
# @spec rendered_to_html(Phoenix.LiveView.Rendered.t()) :: String.t()
# def rendered_to_html(%Phoenix.LiveView.Rendered{} = rendered) do
# rendered
# |> Phoenix.HTML.html_escape()
# |> Phoenix.HTML.safe_to_string()
# end
# else
# def rendered_to_html(_rendered) do
# raise "MDEx.rendered_to_html/1 requires Phoenix.LiveView to be available"
# end
# end
defp opt(options, keys, default) do
case get_in(options, keys) do
nil -> default
val -> val
end
end
@doc """
Builds a new `MDEx.Document` instance.
`MDEx.Document` is the core data structure used across MDEx. It holds the full CommonMark AST and
exposes rich `Access`, `Enumerable`, and pipeline APIs so you can traverse, manipulate, and enrich
Markdown before turning it into HTML/JSON/XML/CommonMark/Delta. `MDEx.new/1` is the entry point for
building those trees:
* Supply `:markdown` to parse raw Markdown right away (the default).
* Pass any built-in options (`:extension`, `:parse`, `:render`, `:syntax_highlight`, `:sanitize`) to
shape how the document will be parsed and rendered.
* Chain pipeline helpers such as `MDEx.Document.append_steps/2`, `MDEx.Document.update_nodes/3`, or your own plugin
modules to programmatically modify the AST.
Once you finish manipulating the document, call one of the `MDEx.to_*` functions to output the final result to a format.
## Options
- `:markdown` (`t:String.t/0`) Raw Markdown to parse into the document. Defaults to `""`
- `:extension` (`t:MDEx.Document.extension_options/0`) Enable extensions. Defaults to `MDEx.Document.default_extension_options/0`
- `:parse` - (`t:parse_options/0`) Modify parsing behavior. Defaults to `MDEx.Document.default_parse_options/0`
- `:render` - (`t:render_options/0`) Modify rendering behavior. Defaults to `MDEx.Document.default_render_options/0`
- `:syntax_highlight` - (`t:syntax_highlight_options/0` | `nil`) Modify syntax highlighting behavior or `nil` to disable. Defaults to `MDEx.Document.default_syntax_highlight_options/0`
- `:sanitize` - (`t:sanitize_options/0` | `nil`) Modify sanitization behavior or `nil` to disable sanitization. Use `MDEx.Document.default_sanitize_options/0` to enable a default set of sanitization options. Defaults to `nil`.
Note that `:sanitize` and `:unsafe` are disabled by default. See [Safety](https://hexdocs.pm/mdex/safety.html) for more info.
## Examples
iex> MDEx.new(markdown: "# Hello") |> MDEx.to_html!()
"
\\nInjected"
"""
@spec new(keyword()) :: Document.t()
def new(options \\ []) do
# TODO: remove :document in v1.0
{deprecated_document_opt, options} = Keyword.pop(options, :document, nil)
{markdown, options} = Keyword.pop(options, :markdown, nil)
markdown = deprecated_document_opt || markdown || ""
unless is_binary(markdown) do
raise ArgumentError, ":markdown option must be a binary, got: #{inspect(markdown)}"
end
%Document{options: Document.default_options()}
|> Document.put_options(options)
|> Document.parse_markdown!(markdown)
end
@doc """
Convert a given `text` string to a format that can be used as an "anchor", such as in a Table of Contents.
This uses the same algorithm GFM uses for anchor ids, so it can be used reliably.
> #### Repeated anchors
> GFM will dedupe multiple repeated anchors with the same value by appending
> an incrementing number to the end of the anchor. That is beyond the scope of
> this function, so you will have to handle it yourself.
## Examples
iex> MDEx.anchorize("Hello World")
"hello-world"
iex> MDEx.anchorize("Hello, World!")
"hello-world"
iex> MDEx.anchorize("Hello -- World")
"hello----world"
iex> MDEx.anchorize("Hello World 123")
"hello-world-123"
iex> MDEx.anchorize("你好世界")
"你好世界"
"""
@spec anchorize(String.t()) :: String.t()
def anchorize(text), do: Native.text_to_anchor(text)
# TODO: remove in v1.0
defp pop_deprecated_document_option(options) do
{deprecated_document, options_without} = Keyword.pop(options, :document)
if not is_nil(deprecated_document) do
IO.warn(
"option :document is deprecated. Use MDEx.new(markdown: ...) or call MDEx.Document.parse_markdown!/2 before invoking MDEx.to_* functions."
)
end
{deprecated_document, options_without}
end
defp prepare_document_for_conversion(document, options, deprecated_document) do
document
|> ensure_document_options()
|> Document.put_options(options)
|> maybe_apply_document_option(deprecated_document)
|> Document.run()
end
defp maybe_apply_document_option(document, nil), do: document
defp maybe_apply_document_option(document, markdown) when is_binary(markdown) do
Document.parse_markdown!(document, markdown)
end
defp maybe_apply_document_option(document, %Document{} = replacement) do
%{document | nodes: replacement.nodes}
end
defp maybe_apply_document_option(_document, other) do
raise ArgumentError, "option :document must be a binary or %MDEx.Document{}, got: #{inspect(other)}"
end
defp ensure_document_options(%Document{options: []} = document) do
Document.put_options(document, Document.default_options())
end
defp ensure_document_options(%Document{} = document), do: document
end