defmodule Wiki.SiteMatrix do @moduledoc """ Retrieves sites from a wiki farm with the SiteMatrix extension installed. """ @metawiki_api "https://meta.wikimedia.org/w/api.php" alias Wiki.Error defmodule Spec do @moduledoc """ Container for a single site. """ @type t :: %__MODULE__{ base_url: String.t(), closed: boolean(), dbname: String.t(), dir: String.t(), lang: String.t(), name: String.t(), private: boolean() } @enforce_keys [:base_url] defstruct [ :base_url, :closed, :dbname, :dir, :lang, :name, :private ] end @type client_options :: [Wiki.Action.client_option() | {:api, binary()}] @doc """ ## Options - `api` - Action API URL for a site participating in the farm. Defaults to #{@metawiki_api}. """ @opaque sitematrix_state :: %{ api: binary(), sites: %{binary() => Spec.t()} } @spec new(client_options()) :: sitematrix_state() def new(opts \\ []) do api = opts[:api] || @metawiki_api %{ api: api, sites: do_get_all(api, opts) } end @doc """ Get all sites for a wiki farm. ## Arguments * `sitematrix` - Result of Sitematrix.new() ## Return value List of site specifications. """ @spec get_all(sitematrix_state()) :: {:ok, [Spec.t()]} | {:error, any} def get_all(sitematrix) do {:ok, Map.values(sitematrix.sites)} end @spec action_client(binary(), client_options()) :: Wiki.Action.Session.t() defp action_client(api, opts) do api |> Wiki.Action.new(opts) end @spec do_get_all(binary(), client_options()) :: map() defp do_get_all(api, opts) do action_client(api, opts) |> fetch_sitematrix() |> Enum.map(&site_spec/1) |> Map.new(fn site -> {site.dbname, site} end) end defp site_spec(site) do %Spec{ base_url: site["url"], closed: site["closed"] == true, dbname: site["dbname"], dir: site["dir"] || "ltr", lang: site["lang"], name: site["name"] || site["sitename"], private: site["private"] == true } end defp fetch_sitematrix(client) do client |> Wiki.Action.stream( action: :sitematrix, smsiteprop: [:dbname, :lang, :sitename, :url] ) |> Enum.flat_map(fn response -> flatten_sitematrix(response["sitematrix"]) end) end defp flatten_sitematrix(all) do (all |> Map.drop(["count", "specials"]) |> flatten_language_wikis()) ++ (all["specials"] || []) end defp flatten_language_wikis(language_sites) do language_sites |> Map.delete("specials") |> Map.values() |> Enum.flat_map(fn group -> group["site"] |> Enum.map(fn site -> site |> Map.merge(%{ "dir" => group["dir"], # FIXME: Lacking translation and grammar "name" => group["localname"] <> " " <> site["sitename"] }) end) end) end @doc """ Get a single site, matching on dbname ## Arguments * `sitematrix` - Result of Sitematrix.new() * `dbname` - Wiki ID, for example "enwiki" ## Return value Site spec or error """ @spec get(sitematrix_state(), String.t()) :: {:ok, Spec.t()} | {:error, any} def get(sitematrix, dbname) do Map.fetch(sitematrix.sites, dbname) |> case do :error -> {:error, %Error{message: "Site #{dbname} not found."}} x -> x end end @doc """ Assertive variant of `get`. """ @spec get!(sitematrix_state(), String.t()) :: Spec.t() def get!(sitematrix, dbname) do case get(sitematrix, dbname) do {:ok, site} -> site {:error, error} -> raise error end end @doc """ Get the Action API for a known site ```elixir Wiki.SiteMatrix.new() |> Wiki.SiteMatrix.get("enwiki") |> Wiki.SiteMatrix.action_api() # "https://en.wikipedia.org/w/api.php" ``` As a convenience, the site can be referenced as a bare string, in which case it will be looked up in the default Wikimedia farm. Note that this will be uncached and so inappropriate for most production use. ```elixir Wiki.SiteMatrix.action_api("dewiki") ``` ## Arguments * `site` - Populated site structure. ## Return value Calculated Action API. """ @spec action_api(String.t() | Spec.t()) :: String.t() def action_api(site) def action_api(site) when is_binary(site) do # FIXME: Globally cached site matrix for this case new() |> get!(site) |> action_api() end def action_api(site) do site.base_url <> "/w/api.php" end end