defmodule Wiki.SiteMatrix do @moduledoc """ Retrieves sites from a wiki farm with the SiteMatrix extension installed. Most applications will connect to a single wiki, or iterate over sites from a single site matrix, so a singleton default can be set up as a child of your Application. ``` children = [ Wiki.SiteMatrix.Default ] ``` To override defaults, supply additional options: ``` children = [ {Wiki.SiteMatrix.Default, api: "https://meta.wikimedia.beta.wmflabs.org/w/api.php"} ] ``` The `get` and `get_all` methods include a variation where the initial SiteMatrix parameter is omitted, and the application default will be used instead. ``` Wiki.SiteMatrix.get(:dewiki) ``` """ @metawiki_api "https://meta.wikimedia.org/w/api.php" alias Wiki.Error alias Wiki.SiteMatrix alias Wiki.SiteMatrix.Default @typedoc """ dbname as an atom or a string """ @type site_name :: atom() | String.t() @typedoc """ Site spec or dbname """ @type site_handle :: SiteMatrix.Spec.t() | site_name @type client_options :: [ Wiki.Action.client_option() | {:api, binary()} | {:key_fun, (SiteMatrix.Spec.t() -> any())} ] defmodule Spec do @moduledoc """ Container for a single site. """ @type t :: %__MODULE__{ base_url: String.t(), closed: boolean(), dbname: String.t(), dir: String.t(), lang: String.t(), name: String.t(), opts: Wiki.SiteMatrix.client_options(), private: boolean(), project: String.t() } @enforce_keys [:base_url] defstruct [ :base_url, :closed, :dbname, :dir, :lang, :name, :private, :project, opts: [] ] end @opaque sitematrix_state :: %{ api: binary(), sites: %{binary() => Spec.t()} } @doc """ ## Options - `api` - Action API URL for a site participating in the farm. Defaults to #{@metawiki_api}. - `key_fun` - Function on a site spec, returning the key to build a lookup with, later passed as the `key` argument to `get`. Defaults to returning the site's `dbname` (eg. "enwiki"). """ @spec new(client_options()) :: sitematrix_state() def new(opts \\ []) do api = opts[:api] || @metawiki_api key_fun = opts[:key_fun] || fn site -> site.dbname end %{ api: api, sites: do_get_all(api, opts) |> Map.new(fn site -> {key_fun.(site), site} end) } end @doc """ Get all sites for a wiki farm. ## Arguments * `sitematrix` - Result of Sitematrix.new() ## Return value List of site specifications. """ @spec get_all(sitematrix_state()) :: {:ok, [Spec.t()]} | {:error, any} def get_all(sitematrix) do {:ok, Map.values(sitematrix.sites)} end @doc """ Get all sites from the default site matrix @see get_all/1 """ @spec get_all() :: {:ok, [Spec.t()]} | {:error, any} def get_all, do: get_all(default_sitematrix()) @spec action_client(binary(), client_options()) :: Wiki.Action.Session.t() defp action_client(api, opts), do: Wiki.Action.new(api, opts) @spec do_get_all(binary(), client_options()) :: [Spec.t()] defp do_get_all(api, opts) do action_client(api, opts) |> fetch_sitematrix() |> Enum.map(&parse_site_spec/1) end defp parse_site_spec(site) do %Spec{ base_url: site["url"], closed: site["closed"] == true, dbname: site["dbname"], dir: site["dir"] || "ltr", lang: site["lang"], name: site["name"] || site["sitename"], private: site["private"] == true, project: site["code"] } end defp fetch_sitematrix(client) do client |> Wiki.Action.stream( action: :sitematrix, smsiteprop: [:code, :dbname, :lang, :sitename, :url] ) |> Enum.flat_map(fn response -> flatten_sitematrix(response["sitematrix"]) end) end defp flatten_sitematrix(all) do (all |> Map.drop(["count", "specials"]) |> flatten_language_wikis()) ++ (all["specials"] || []) end defp flatten_language_wikis(language_sites) do language_sites |> Map.delete("specials") |> Map.values() |> Enum.flat_map(fn group -> group_name = group["localname"] || group["name"] || group["code"] group["site"] |> Enum.map(fn site -> site |> Map.merge(%{ "dir" => group["dir"], # FIXME: Lacking translation and grammar "name" => group_name <> " " <> site["sitename"] }) end) end) end @doc """ Look up a single site by key ## Arguments * `sitematrix` - Result of Sitematrix.new() * `key` - The dbname (eg. :enwiki) or value of another key if the sitematrix has installed a custom `key_fun`. ## Return value Site spec or error """ @spec get(sitematrix_state(), site_name()) :: {:ok, Spec.t()} | {:error, any} def get(sitematrix, key) do resolve_site(sitematrix, key) |> case do :error -> {:error, %Error{message: "Site #{to_string(key)} not found."}} x -> x end end @spec resolve_site(sitematrix_state(), site_name()) :: {:ok, Spec.t()} | :error defp resolve_site(sitematrix, key) defp resolve_site(sitematrix, key) when is_atom(key), do: resolve_site(sitematrix, to_string(key)) defp resolve_site(sitematrix, key) when is_binary(key), do: Map.fetch(sitematrix.sites, key) @doc """ Assertive variant of `get`. """ @spec get!(sitematrix_state(), site_name()) :: Spec.t() def get!(sitematrix, key) do case get(sitematrix, key) do {:ok, site} -> site {:error, error} -> raise error end end @doc """ Look up in the default site matrix @see get/2 """ @spec get(site_name()) :: {:ok, Spec.t()} | {:error, any} def get(key), do: get(default_sitematrix(), key) @doc """ Look up in the default site matrix @see get!/2 """ @spec get!(site_name()) :: Spec.t() def get!(key), do: get!(default_sitematrix(), key) @doc """ Get the Action API for a known site ```elixir Wiki.SiteMatrix.action_api(:enwiki) # "https://en.wikipedia.org/w/api.php" ``` As a convenience, the site can be referenced as a bare string, in which case it will be looked up in the default Wikimedia farm. Note that this will be uncached and so inappropriate for most production use. ```elixir Wiki.SiteMatrix.action_api(:dewiki) ``` ## Arguments * `site` - Populated site structure. ## Return value Calculated Action API. """ @spec action_api(site_handle()) :: String.t() def action_api(site) def action_api(site) when is_atom(site) or is_binary(site) do default_sitematrix() |> get!(site) |> action_api() end def action_api(spec) when is_struct(spec, Spec), do: spec.base_url <> "/w/api.php" @doc """ Utility method to get the shared SiteMatrix. """ @spec default_sitematrix() :: sitematrix_state() def default_sitematrix, do: Default.get_or_start() defmodule Default do @moduledoc """ Optional singleton to hold a shared SiteMatrix. """ require Logger alias Wiki.SiteMatrix use Agent @doc """ Start a single SiteMatrix which will be shared by all code in this instance. Intended to be started from your Application. """ @spec start_link(SiteMatrix.client_options()) :: GenServer.on_start() def start_link(opts \\ []) do Agent.start_link(fn -> SiteMatrix.new(opts) end, name: __MODULE__) end @doc """ Get the singleton SiteMatrix. """ @spec get_or_start() :: SiteMatrix.sitematrix_state() def get_or_start do case GenServer.whereis(__MODULE__) do nil -> start_link() _ -> :ok end Agent.get(__MODULE__, & &1) end end end