defmodule Wiki.Site do @moduledoc """ Retrieves sites from a wiki farm with the SiteMatrix extension installed. """ @metawiki_api "https://meta.wikimedia.org/w/api.php" defmodule Spec do @moduledoc """ Container for a single site. """ @type t :: %__MODULE__{ base_url: String.t(), closed: boolean(), dbname: String.t(), dir: String.t(), lang: String.t(), name: String.t(), private: boolean() } @enforce_keys [:base_url] defstruct [ :base_url, :closed, :dbname, :dir, :lang, :name, :private ] end @doc """ Get all sites for a wiki farm. ## Arguments - `api` - Action API URL for a site participating in the farm. ## Return value List of site specifications. TODO: * Memoize result for a configurable amount of time. * Continuation when more than 5000 sites are available. """ @spec get_all(String.t()) :: Enumerable.t() def get_all(api \\ @metawiki_api) do api |> Wiki.Action.new() |> Wiki.Action.get( action: :sitematrix, smsiteprop: [:dbname, :lang, :sitename, :url] ) |> Map.get(:result) |> Map.get("sitematrix") |> Map.delete("count") |> flatten_sitematrix() |> Enum.map(fn site -> %Spec{ base_url: site["url"], closed: site["closed"] == true, dbname: site["dbname"], dir: site["dir"] || "ltr", lang: site["lang"], name: site["name"] || site["sitename"], private: site["private"] == true } end) end defp flatten_sitematrix(all) do (all |> Map.drop(["count", "specials"]) |> flatten_language_wikis()) ++ all["specials"] end defp flatten_language_wikis(language_sites) do language_sites |> Map.delete("specials") |> Map.values() |> Enum.flat_map(fn group -> group["site"] |> Enum.map(fn site -> site |> Map.merge(%{ "dir" => group["dir"], # FIXME: Lacking translation and grammar "name" => group["localname"] <> " " <> site["sitename"] }) end) end) end @doc """ Get a single site, matching on dbname ## Arguments * `dbname` - Wiki ID, for example "enwiki" * `api` - Action API URL ## Return value Site spec or `nil` if none matched. """ @spec get(String.t(), String.t()) :: Spec.t() | nil def get(api \\ @metawiki_api, dbname) do get_all(api) |> Enum.find(nil, fn x -> x.dbname == dbname end) end @doc """ Get the Action API for a site ```elixir Wiki.Site.get("enwiki") |> Wiki.Site.action_api() # "https://en.wikipedia.org/w/api.php" ``` ## Arguments * `site` - Populated site structure. ## Return value Calculated Action API. TODO: Only works for the default configuration, will need to autodetect otherwise. """ @spec action_api(Spec.t()) :: String.t() def action_api(site) do # TODO: read about computed values site.base_url <> "/w/api.php" end end