defmodule Meeseeks do
alias Meeseeks.{Context, Document, Error, Parser, Result, Select, Selector, TupleTree}
@moduledoc """
Meeseeks is an Elixir library for parsing and extracting data from HTML and
XML with CSS or XPath selectors.
```elixir
import Meeseeks.CSS
html = HTTPoison.get!("https://news.ycombinator.com/").body
for story <- Meeseeks.all(html, css("tr.athing")) do
title = Meeseeks.one(story, css(".title a"))
%{title: Meeseeks.text(title),
url: Meeseeks.attr(title, "href")}
end
#=> [%{title: "...", url: "..."}, %{title: "...", url: "..."}, ...]
```
## Dependencies
Meeseeks depends on [html5ever](https://github.com/servo/html5ever) via
[meeseeks_html5ever](https://github.com/mischov/meeseeks_html5ever).
Because html5ever is a Rust library, you will need to have the Rust
compiler [installed](https://www.rust-lang.org/en-US/install.html).
This dependency is necessary because there are no HTML5 spec compliant
parsers written in Elixir/Erlang.
## Getting Started
### Parse
Start by parsing a source (HTML/XML string or `Meeseeks.TupleTree`) into
a `Meeseeks.Document` so that it can be queried.
`Meeseeks.parse/1` parses the source as HTML, but `Meeseeks.parse/2`
accepts a second argument of either `:html` or `:xml` that specifies how
the source is parsed.
```elixir
document = Meeseeks.parse("")
#=> Meeseeks.Document<{...}>
```
The selection functions accept an unparsed source, parsing it as HTML, but
parsing is expensive so parse ahead of time when running multiple
selections on the same document.
### Select
Next, use one of Meeseeks's selection functions - `fetch_all`, `all`,
`fetch_one`, or `one` - to search for nodes.
All these functions accept a queryable (a source, a document, or a
`Meeseeks.Result`), one or more `Meeseeks.Selector`s, and optionally an
initial context.
`all` returns a (possibly empty) list of results representing every node
matching one of the provided selectors, while `one` returns a result
representing the first node to match a selector (depth-first) or nil if
there is no match.
`fetch_all` and `fetch_one` work like `all` and `one` respectively, but
wrap the result in `{:ok, ...}` if there is a match or return
`{:error, %Meeseeks.Error{type: :select, reason: :no_match}}` if there is
not.
To generate selectors, use the `css` macro provided by `Meeseeks.CSS` or
the `xpath` macro provided by `Meeseeks.XPath`.
```elixir
import Meeseeks.CSS
result = Meeseeks.one(document, css("#main p"))
#=> #Meeseeks.Result<{ 1
}>
import Meeseeks.XPath
result = Meeseeks.one(document, xpath("//*[@id='main']//p"))
#=> #Meeseeks.Result<{ 1
}>
```
### Extract
Retrieve information from the `Meeseeks.Result` with an extraction
function.
The extraction functions are `attr`, `attrs`, `data`, `dataset`, `html`,
`own_text`, `tag`, `text`, `tree`.
```elixir
Meeseeks.tag(result)
#=> "p"
Meeseeks.text(result)
#=> "1"
Meeseeks.tree(result)
#=> {"p", [], ["1"]}
```
The extraction functions `html` and `tree` work on `Meeseeks.Document`s in
addition to `Meeseeks.Result`s.
```elixir
Meeseeks.html(document)
#=> ""
```
## Custom Selectors
Meeseeks is designed to have extremely extensible selectors, and creating
a custom selector is as easy as defining a struct that implements the
`Meeseeks.Selector` behaviour.
```elixir
defmodule CommentContainsSelector do
use Meeseeks.Selector
alias Meeseeks.Document
defstruct value: ""
def match(selector, %Document.Comment{} = node, _document, _context) do
String.contains?(node.content, selector.value)
end
def match(_selector, _node, _document, _context) do
false
end
end
selector = %CommentContainsSelector{value: "TODO"}
Meeseeks.one("", selector)
#=> #Meeseeks.Result<{ }>
```
To learn more, check the documentation for `Meeseeks.Selector` and
`Meeseeks.Selector.Combinator`
"""
@type queryable :: Parser.source() | Document.t() | Result.t()
@type extractable :: Document.t() | Result.t() | nil
@type selectors :: Selector.t() | [Selector.t()]
# Parse
@doc """
Parses a string or `Meeseeks.TupleTree` into a `Meeseeks.Document`.
`parse/1` parses as HTML, while `parse/2` accepts a second argument of
either `:html` or `:xml` that specifies how the source is parsed.
## Examples
iex> Meeseeks.parse("")
#Meeseeks.Document<{...}>
iex> Meeseeks.parse({"div", [{"id", "main"}], [{"p", [], ["Hello, Meeseeks!"]}]})
#Meeseeks.Document<{...}>
iex> Meeseeks.parse("GGK", :xml)
#Meeseeks.Document<{...}>
"""
@spec parse(Parser.source()) :: Document.t() | {:error, Error.t()}
def parse(source) do
Parser.parse(source)
end
@spec parse(Parser.source(), Parser.type()) :: Document.t() | {:error, Error.t()}
def parse(source, parser) do
Parser.parse(source, parser)
end
# Select
@doc """
Returns `{:ok, [Result, ...]}` if one of more nodes in the queryable match
a selector, or `{:error, %Meeseeks.Error{type: :select, reason: :no_match}}`
if none do.
Optionally accepts a `Meeseeks.Context` map.
Parses the source if it is not a `Meeseeks.Document` or `Meeseeks.Result`,
and may return `{:error, %Meeseeks.Error{type: parser}` if there is a parse
error.
If multiple selections are being ran on the same unparsed source, parse
first to avoid unnecessary computation.
## Examples
iex> import Meeseeks.CSS
iex> Meeseeks.fetch_all("", css("#main p")) |> elem(1) |> List.first()
#Meeseeks.Result<{ 1
}>
"""
@spec fetch_all(queryable, selectors) :: {:ok, [Result.t()]} | {:error, Error.t()}
def fetch_all(queryable, selectors) do
fetch_all(queryable, selectors, %{})
end
@spec fetch_all(queryable, selectors, Context.t()) :: {:ok, [Result.t()]} | {:error, Error.t()}
def fetch_all(queryable, selectors, context)
def fetch_all({:error, _} = error, _selectors, _context), do: error
def fetch_all(%Document{} = queryable, selectors, context) do
Select.fetch_all(queryable, selectors, context)
end
def fetch_all(%Result{} = queryable, selectors, context) do
Select.fetch_all(queryable, selectors, context)
end
def fetch_all(source, selectors, context) do
case parse(source) do
{:error, reason} -> {:error, reason}
document -> Select.fetch_all(document, selectors, context)
end
end
@doc """
Returns `[Result, ...]` if one or more nodes in the queryable match a
selector, or `[]` if none do.
Optionally accepts a `Meeseeks.Context` map.
Parses the source if it is not a `Meeseeks.Document` or `Meeseeks.Result`,
and may return `{:error, %Meeseeks.Error{type: parser}` if there is a parse
error.
If multiple selections are being ran on the same unparsed source, parse
first to avoid unnecessary computation.
## Examples
iex> import Meeseeks.CSS
iex> Meeseeks.all("", css("#main p")) |> List.first()
#Meeseeks.Result<{ 1
}>
"""
@spec all(queryable, selectors) :: [Result.t()] | {:error, Error.t()}
def all(queryable, selectors) do
all(queryable, selectors, %{})
end
@spec all(queryable, selectors, Context.t()) :: [Result.t()] | {:error, Error.t()}
def all(queryable, selectors, context)
def all({:error, _} = error, _selectors, _context), do: error
def all(%Document{} = queryable, selectors, context) do
Select.all(queryable, selectors, context)
end
def all(%Result{} = queryable, selectors, context) do
Select.all(queryable, selectors, context)
end
def all(source, selectors, context) do
case parse(source) do
{:error, reason} -> {:error, reason}
document -> Select.all(document, selectors, context)
end
end
@doc """
Returns `{:ok, Result}` for the first node in the queryable (depth-first)
matching a selector, or
`{:error, %Meeseeks.Error{type: :select, reason: :no_match}}` if none do.
Optionally accepts a `Meeseeks.Context` map.
Parses the source if it is not a `Meeseeks.Document` or `Meeseeks.Result`,
and may return `{:error, %Meeseeks.Error{type: parser}` if there is a parse
error.
If multiple selections are being ran on the same unparsed source, parse
first to avoid unnecessary computation.
## Examples
iex> import Meeseeks.CSS
iex> Meeseeks.fetch_one("", css("#main p")) |> elem(1)
#Meeseeks.Result<{ 1
}>
"""
@spec fetch_one(queryable, selectors) :: {:ok, Result.t()} | {:error, Error.t()}
def fetch_one(queryable, selectors) do
fetch_one(queryable, selectors, %{})
end
@spec fetch_one(queryable, selectors, Context.t()) :: {:ok, Result.t()} | {:error, Error.t()}
def fetch_one(queryable, selectors, context)
def fetch_one({:error, _} = error, _selectors, _context), do: error
def fetch_one(%Document{} = queryable, selectors, context) do
Select.fetch_one(queryable, selectors, context)
end
def fetch_one(%Result{} = queryable, selectors, context) do
Select.fetch_one(queryable, selectors, context)
end
def fetch_one(source, selectors, context) do
case parse(source) do
{:error, reason} -> {:error, reason}
document -> Select.fetch_one(document, selectors, context)
end
end
@doc """
Returns a `Result` for the first node in the queryable (depth-first)
matching a selector, or `nil` if none do.
Optionally accepts a `Meeseeks.Context` map.
Parses the source if it is not a `Meeseeks.Document` or `Meeseeks.Result`,
and may return `{:error, %Meeseeks.Error{type: parser}` if there is a parse
error.
If multiple selections are being ran on the same unparsed source, parse
first to avoid unnecessary computation.
## Examples
iex> import Meeseeks.CSS
iex> Meeseeks.one("", css("#main p"))
#Meeseeks.Result<{ 1
}>
"""
@spec one(queryable, selectors) :: Result.t() | nil | {:error, Error.t()}
def one(queryable, selectors) do
one(queryable, selectors, %{})
end
@spec one(queryable, selectors, Context.t()) :: Result.t() | nil | {:error, Error.t()}
def one(queryable, selectors, context)
def one({:error, _} = error, _selectors, _context), do: error
def one(%Document{} = queryable, selectors, context) do
Select.one(queryable, selectors, context)
end
def one(%Result{} = queryable, selectors, context) do
Select.one(queryable, selectors, context)
end
def one(source, selectors, context) do
case parse(source) do
{:error, reason} -> {:error, reason}
document -> Select.one(document, selectors, context)
end
end
@doc """
Returns the accumulated result of walking the queryable, accumulating nodes
that match a selector. Prefer `all` or `one`- `select` should only be used
when a custom accumulator is required.
Requires that a `Meeseeks.Accumulator` has been added to the context via
`Meeseeks.Context.add_accumulator/2`, and will raise an error if it hasn't.
Parses the source if it is not a `Meeseeks.Document` or `Meeseeks.Result`,
and may return `{:error, %Meeseeks.Error{type: parser}` if there is a parse
error.
If multiple selections are being ran on the same unparsed source, parse
first to avoid unnecessary computation.
## Examples
iex> import Meeseeks.CSS
iex> accumulator = %Meeseeks.Accumulator.One{}
iex> context = Meeseeks.Context.add_accumulator(%{}, accumulator)
iex> Meeseeks.select("", css("#main p"), context)
#Meeseeks.Result<{ 1
}>
"""
@spec select(queryable, selectors, Context.t()) :: any | {:error, Error.t()}
def select(queryable, selectors, context)
def select({:error, _} = error, _selectors, _context), do: error
def select(%Document{} = queryable, selectors, context) do
Select.select(queryable, selectors, context)
end
def select(%Result{} = queryable, selectors, context) do
Select.select(queryable, selectors, context)
end
def select(source, selectors, context) do
case parse(source) do
{:error, reason} -> {:error, reason}
document -> Select.select(document, selectors, context)
end
end
# Extract
@doc """
Returns the value of an attribute in a result, or nil if there isn't one.
Nil input returns `nil`.
## Examples
iex> import Meeseeks.CSS
iex> result = Meeseeks.one("Hi
", css("#example"))
#Meeseeks.Result<{ Hi
}>
iex> Meeseeks.attr(result, "id")
"example"
"""
@spec attr(extractable, String.t()) :: String.t() | nil
def attr(extractable, attribute)
def attr(nil, _), do: nil
def attr(%Result{} = result, attribute), do: Result.attr(result, attribute)
def attr(x, _attribute), do: raise_cannot_extract(x, "attr/2")
@doc """
Returns a result's attributes list, which may be empty, or nil if the
result represents a node without attributes.
Nil input returns `nil`.
## Examples
iex> import Meeseeks.CSS
iex> result = Meeseeks.one("Hi
", css("#example"))
#Meeseeks.Result<{ Hi
}>
iex> Meeseeks.attrs(result)
[{"id", "example"}]
"""
@spec attrs(extractable) :: [{String.t(), String.t()}] | nil
def attrs(extractable)
def attrs(nil), do: nil
def attrs(%Result{} = result), do: Result.attrs(result)
def attrs(x), do: raise_cannot_extract(x, "attrs/1")
@doc """
Returns the combined data of a result or the result's children, which may
be an empty string.
Data is the content of `", css("#example"))
#Meeseeks.Result<{ }>
iex> Meeseeks.data(result2)
"Hi"
"""
@spec data(extractable) :: String.t() | nil
def data(extractable)
def data(nil), do: nil
def data(%Result{} = result), do: Result.data(result)
def data(x), do: raise_cannot_extract(x, "data/1")
@doc """
Returns a map of a result's data attributes, or nil if the result
represents a node without attributes.
Behaves like HTMLElement.dataset; only valid data attributes are included,
and attribute names have "data-" removed and are converted to camelCase.
See: https://developer.mozilla.org/en-US/docs/Web/API/HTMLElement/dataset
Nil input returns `nil`.
## Examples
iex> import Meeseeks.CSS
iex> result = Meeseeks.one("", css("#example"))
#Meeseeks.Result<{ }>
iex> Meeseeks.dataset(result)
%{"xVal" => "1", "yVal" => "2"}
"""
@spec dataset(extractable) :: %{optional(String.t()) => String.t()} | nil
def dataset(extractable)
def dataset(nil), do: nil
def dataset(%Result{} = result), do: Result.dataset(result)
def dataset(x), do: raise_cannot_extract(x, "dataset/1")
@doc """
Returns a string representing the combined HTML of a document or result
and its descendants.
Nil input returns `nil`.
## Examples
iex> import Meeseeks.CSS
iex> document = Meeseeks.parse("Hi
")
iex> Meeseeks.html(document)
"Hi
"
iex> result = Meeseeks.one(document, css("#example"))
#Meeseeks.Result<{ Hi
}>
iex> Meeseeks.html(result)
"Hi
"
"""
@spec html(extractable) :: String.t() | nil
def html(extractable)
def html(nil), do: nil
def html(%Document{} = document), do: Document.html(document)
def html(%Result{} = result), do: Result.html(result)
def html(x), do: raise_cannot_extract(x, "html/1")
@doc """
Returns the combined text of a result or the result's children, which may
be an empty string.
Nil input returns `nil`.
## Examples
iex> import Meeseeks.CSS
iex> result = Meeseeks.one("Hello, World!
", css("div"))
#Meeseeks.Result<{ Hello, World!
}>
iex> Meeseeks.own_text(result)
"Hello,"
"""
@spec own_text(extractable) :: String.t() | nil
def own_text(extractable)
def own_text(nil), do: nil
def own_text(%Result{} = result), do: Result.own_text(result)
def own_text(x), do: raise_cannot_extract(x, "own_text/1")
@doc """
Returns a result's tag, or `nil` if the result represents a node without a
tag.
Nil input returns `nil`.
## Examples
iex> import Meeseeks.CSS
iex> result = Meeseeks.one("Hi
", css("#example"))
#Meeseeks.Result<{ Hi
}>
iex> Meeseeks.tag(result)
"div"
"""
@spec tag(extractable) :: String.t() | nil
def tag(extractable)
def tag(nil), do: nil
def tag(%Result{} = result), do: Result.tag(result)
def tag(x), do: raise_cannot_extract(x, "tag/1")
@doc """
Returns the combined text of a result or the result's descendants, which
may be an empty string.
Nil input returns `nil`.
## Examples
iex> import Meeseeks.CSS
iex> result = Meeseeks.one("Hello, World!
", css("div"))
#Meeseeks.Result<{ Hello, World!
}>
iex> Meeseeks.text(result)
"Hello, World!"
"""
@spec text(extractable) :: String.t() | nil
def text(extractable)
def text(nil), do: nil
def text(%Result{} = result), do: Result.text(result)
def text(x), do: raise_cannot_extract(x, "text/1")
@doc """
Returns the `Meeseeks.TupleTree` of a document or result and its
descendants.
Nil input returns `nil`.
## Examples
iex> import Meeseeks.CSS
iex> document = Meeseeks.parse("Hi
")
iex> Meeseeks.tree(document)
[{"html", [],
[{"head", [], []},
{"body", [], [{"div", [{"id", "example"}], ["Hi"]}]}]}]
iex> result = Meeseeks.one(document, css("#example"))
#Meeseeks.Result<{ Hi
}>
iex> Meeseeks.tree(result)
{"div", [{"id", "example"}], ["Hi"]}
"""
@spec tree(extractable) :: TupleTree.t() | nil
def tree(extractable)
def tree(nil), do: nil
def tree(%Document{} = document), do: Document.tree(document)
def tree(%Result{} = result), do: Result.tree(result)
def tree(x), do: raise_cannot_extract(x, "tree/1")
defp raise_cannot_extract(target, extractor) do
raise "Cannot run Meeseeks.#{extractor} on #{inspect(target)}"
end
end