defmodule Gulib.HTMLParse do @moduledoc """ HTML的解析器 """ @self_closing_tags [ "area", "base", "br", "col", "command", "embed", "hr", "img", "input", "keygen", "link", "meta", "param", "source", "track", "wbr" ] @doc """ 将数组形式的属性列表,转成Keyword形式 """ def parse_attrs(attrs) do attrs |> Enum.map(fn({key, value}) -> {:"#{key}", value} end) |> Keyword.new end @doc """ 将Keyword形式的属性列表,转成List形式 """ def flatten_attrs(attrs) do attrs |> Enum.map(fn({key, value}) -> {"#{key}", value} end) end def parse(html) when is_binary(html) do html_tree = html |> Floki.parse() cond do is_tuple(html_tree) or is_list(html_tree) -> html_tree true -> {"div", [{"html-tree", "auto"}], [html_tree]} end end def parse(html_tree) when is_tuple(html_tree) or is_list(html_tree) do html_tree end def ergodic_map(html_tree_list, collections, collect_fn) when is_list(html_tree_list) and is_function(collect_fn) do current_collections = html_tree_list |> Enum.map(fn(html_tree) -> ergodic_map(html_tree, collections, collect_fn) end) |> List.flatten collections ++ current_collections end def ergodic_map(html_tree, collections, collect_fn) when is_function(collect_fn) do impl_ergodic_map(html_tree, collections, collect_fn) end defp impl_ergodic_map({name, attrs, rest}, collections, collect_fn) when is_function(collect_fn) do new_collections = (collect_fn.({name, attrs, rest}, collections) || []) current_collections = rest |> Enum.map(fn(html_tree) -> impl_ergodic_map(html_tree, collections, collect_fn) end) |> List.flatten collections ++ new_collections ++ current_collections end defp impl_ergodic_map(other, collections, collect_fn) when is_function(collect_fn) do collect_fn.(other, collections) || [] end # 遍历重新修改HTML结构 def transform(html_tree_list, transformation) when is_list(html_tree_list) do Enum.map(html_tree_list, fn(html_tree) -> transform(html_tree, transformation) end) end def transform(html_tree, transformation) do impl_transformation(html_tree, transformation, %{ancestors: []}) end defp impl_transformation({name, attrs, rest}, transformation, opts) do ancestors = opts |> Map.get(:ancestors, []) ancestors = List.insert_at(ancestors, -1, name) opts = opts |> Map.put(:ancestors, ancestors) {new_name, new_attrs, new_rest} = transformation.({name, attrs, rest}, opts) new_rest = Enum.map(new_rest, fn(html_tree) -> impl_transformation(html_tree, transformation, opts) end) {new_name, new_attrs, new_rest} end defp impl_transformation(other, transformation, opts) do transformation.(other, opts) end defdelegate attribute(html_tree, attribute_name), to: Floki defdelegate attribute(html, selector, attribute_name), to: Floki defdelegate filter_out(html_tree, selector), to: Floki defdelegate find(html, selector), to: Floki defdelegate text(html, opts \\ [deep: true, js: false, sep: ""]), to: Floki def raw_html(html_tree), do: raw_html(html_tree, "") defp raw_html({"div", [{"html-tree", "auto"}], [html_tree]}, _html), do: html_tree defp raw_html([], html), do: html defp raw_html(tuple, html) when is_tuple(tuple), do: raw_html([tuple], html) defp raw_html([string|tail], html) when is_binary(string), do: raw_html(tail, html <> string) defp raw_html([{:comment, comment}|tail], html), do: raw_html(tail, html <> "") defp raw_html([{type, attrs, children}|tail], html) do raw_html(tail, html <> tag_for(type, tag_attrs(attrs), children)) end defp raw_html(_, _), do: "" defp tag_attrs(attr) when is_binary(attr), do: attr defp tag_attrs(attr_list) do attr_list |> Enum.reduce("", &build_attrs/2) |> String.trim end defp build_attrs({attr, value}, attrs), do: ~s(#{attrs} #{attr}="#{value}") defp build_attrs(attr, attrs), do: "#{attrs} #{attr}" defp tag_for(type, attrs, _children) when type in @self_closing_tags do case attrs do "" -> "<#{type}/>" _ -> "<#{type} #{attrs}/>" end end defp tag_for(type, attrs, children) do case attrs do "" -> "<#{type}>#{raw_html(children)}" _ -> "<#{type} #{attrs}>#{raw_html(children)}" end end end