defmodule Funkspector do @moduledoc """ Funkspector is a web scraper that lets you extract data from web pages. """ alias Funkspector.{Resolver, Document, PageScraper, SitemapScraper, TextSitemapScraper} @doc """ Given a URL, it will follow the redirections and return the final URL and the final response. ## Examples iex> { :ok, final_url, _response } = Funkspector.resolve("http://github.com") iex> final_url "https://github.com/" """ def resolve(url, options \\ %{}) do options = Map.merge(default_options(), options) Resolver.resolve(url, options) end @doc """ Parses an HTML document. This can be used to request a document by passing its URL, like: Funkspector.page_scrape("https://example.com") Or to scrape an already loaded document, by passing its HTML contents: Funkspector.page_scrape("https://example.com", contents: "...") ## Example: request a document iex> { :ok, document } = Funkspector.page_scrape("https://jaimeiniesta.com") iex> Enum.take(document.data.links.http.external, 3) ["http://www.archive.elixirconf.eu/elixirconf2016", "https://steadyhq.com/", "https://stuart.com/"] ## Example: site not found iex> Funkspector.page_scrape("https://notfoundwebsite.com") {:error, "https://notfoundwebsite.com", %HTTPoison.Error{reason: :nxdomain, id: nil}} """ def page_scrape(url, options \\ %{}) do scrape(url, options, &PageScraper.scrape/1) end @doc """ Parses an XML sitemap. This can be used to request a document by passing its URL, like: Funkspector.sitemap_scrape("https://example.com") Or to scrape an already loaded document, by passing its XML contents: Funkspector.sitemap_scrape("https://example.com/sitemap.xml", contents: "...") ## Example iex> { :ok, document } = Funkspector.sitemap_scrape("https://rocketvalidator.com/sitemap.xml") iex> length document.data.locs 2243 iex> Enum.take(document.data.locs, 3) ["https://rocketvalidator.com/", "https://rocketvalidator.com/html-validation", "https://rocketvalidator.com/accessibility-validation"] """ def sitemap_scrape(url, options \\ %{}) do scrape(url, options, &SitemapScraper.scrape/1) end @doc """ Parses a text sitemap. This can be used to request a document by passing its URL, like: Funkspector.text_sitemap_scrape("https://example.com") Or to scrape an already loaded document, by passing its text contents: Funkspector.text_sitemap_scrape("https://example.com/sitemap.txt", contents: "...") ## Example iex> { :ok, document } = Funkspector.text_sitemap_scrape("https://rocketvalidator.com/sitemap.txt") iex> length document.data.lines 2243 iex> Enum.take(document.data.lines, 3) ["https://rocketvalidator.com/", "https://rocketvalidator.com/html-validation", "https://rocketvalidator.com/accessibility-validation"] """ def text_sitemap_scrape(url, options \\ %{}) do scrape(url, options, &TextSitemapScraper.scrape/1) end ##################### # Private functions # ##################### defp default_options do %{ hackney: [:insecure], timeout: 28_000, recv_timeout: 25_000, user_agent: "Funkspector/0.6.0 (+https://hex.pm/packages/funkspector)" } end def scrape(url, options, scraping_function) do options = Map.merge(default_options(), options) case request_or_load_contents(url, options) do {:ok, document} -> scraping_function.(document) error -> error end end defp request_or_load_contents(url, options) do case options[:contents] do nil -> Document.request(url, options) contents when is_binary(contents) -> Document.load(url, contents) end end end