defmodule Wiki.Site do @moduledoc """ Retrieves sites from a wiki farm with the SiteMatrix extension installed. """ @metawiki_api "https://meta.wikimedia.org/w/api.php" alias Wiki.Error defmodule Spec do @moduledoc """ Container for a single site. """ @type t :: %__MODULE__{ base_url: String.t(), closed: boolean(), dbname: String.t(), dir: String.t(), lang: String.t(), name: String.t(), private: boolean() } @enforce_keys [:base_url] defstruct [ :base_url, :closed, :dbname, :dir, :lang, :name, :private ] end @doc """ Get all sites for a wiki farm. ## Arguments - `api` - Action API URL for a site participating in the farm. ## Return value List of site specifications. TODO: * Memoize result for a configurable amount of time. * Continuation when more than 5000 sites are available. """ @spec get_all(String.t()) :: {:ok, [Spec.t()]} | {:error, Error} def get_all(api \\ @metawiki_api) do with {:ok, response} <- fetch_sitematrix(api), %{result: %{"sitematrix" => sitematrix}} <- response do {:ok, sitematrix |> flatten_sitematrix() |> Enum.map(&site_spec/1)} end end defp site_spec(site) do %Spec{ base_url: site["url"], closed: site["closed"] == true, dbname: site["dbname"], dir: site["dir"] || "ltr", lang: site["lang"], name: site["name"] || site["sitename"], private: site["private"] == true } end defp fetch_sitematrix(api) do api |> Wiki.Action.new() |> Wiki.Action.get( action: :sitematrix, smsiteprop: [:dbname, :lang, :sitename, :url] ) end defp flatten_sitematrix(all) do (all |> Map.drop(["count", "specials"]) |> flatten_language_wikis()) ++ all["specials"] end defp flatten_language_wikis(language_sites) do language_sites |> Map.delete("specials") |> Map.values() |> Enum.flat_map(fn group -> group["site"] |> Enum.map(fn site -> site |> Map.merge(%{ "dir" => group["dir"], # FIXME: Lacking translation and grammar "name" => group["localname"] <> " " <> site["sitename"] }) end) end) end @doc """ Get a single site, matching on dbname ## Arguments * `dbname` - Wiki ID, for example "enwiki" * `api` - Action API URL ## Return value Site spec """ @spec get(String.t(), String.t()) :: {:ok, Spec.t()} | {:error, Error} def get(api \\ @metawiki_api, dbname) do with {:ok, sites} <- get_all(api), # TODO: clean up these mixed clauses site when not is_nil(site) <- Enum.find(sites, nil, fn x -> x.dbname == dbname end) do {:ok, site} else {:error, error} -> {:error, error} nil -> {:error, %Error{message: "Site #{dbname} not found."}} end end @doc """ Assertive variant of `get`. """ @spec get!(String.t(), String.t()) :: Spec.t() def get!(api \\ @metawiki_api, dbname) do case get(api, dbname) do {:ok, site} -> site {:error, error} -> raise error end end @doc """ Get the Action API for a site ```elixir "enwiki" |> Wiki.Site.get() |> Wiki.Site.action_api() # "https://en.wikipedia.org/w/api.php" ``` ## Arguments * `site` - Populated site structure. ## Return value Calculated Action API. TODO: Only works for the default configuration, needs to be either customizable or autodetected. """ @spec action_api(Spec.t()) :: String.t() def action_api(site) do site.base_url <> "/w/api.php" end end