defmodule Wiki.Action.Session do @moduledoc """ This module provides a struct for holding private connection state and accumulated results. ## Fields - `result` - Map with recursively merged values from all requests made using this session. - `state` - Cache for session state and accumulation. """ @type client :: Tesla.Client.t() @type result :: {:ok, t()} | {:error, any} @type state :: keyword @type t :: %__MODULE__{ __client__: client, result: map, state: keyword } defstruct __client__: nil, result: %{}, state: [] end defmodule Wiki.Action do @moduledoc """ Adapter to the MediaWiki [Action API](https://www.mediawiki.org/wiki/Special:MyLanguage/API:Main_page) Most commands return a pipeable Wiki.Action.Session, which can be reused repeatedly. Anonymous requests, ```elixir Wiki.SiteMatrix.new() |> Wiki.SiteMatrix.get!("dewiki") |> Wiki.Action.new() |> Wiki.Action.get!( action: :query, meta: :siteinfo, siprop: :statistics ) # %Wiki.Action.Session{ # ... # result: %{ # "batchcomplete" => true, # "query" => %{ # "statistics" => %{ # "activeusers" => 19393, # "admins" => 188, # "articles" => 2583636, # "edits" => 211249646, # "images" => 130213, # "jobs" => 0, # "pages" => 7164514, # "queued-massmessages" => 0, # "users" => 3716049 # } # } # }, # ... # } ``` Commands can be pipelined while accumulating results, and logged-in user permissions delegated by supplying a [bot password](https://www.mediawiki.org/wiki/Manual:Bot_passwords). ```elixir Wiki.SiteMatrix.new() |> Wiki.SiteMatrix.get!("enwiki") |> Wiki.Action.new( accumulate: true ) |> Wiki.Action.authenticate!( Application.get_env(:example_app, :bot_username), Application.get_env(:example_app, :bot_password) ) |> Wiki.Action.get!( action: :query, meta: :tokens, type: :csrf ) |> (&Wiki.Action.post!(&1, action: :edit, title: "Sandbox", assert: :user, token: &1.result["query"]["tokens"]["csrftoken"], appendtext: "~~~~ was here." )).() |> Map.get(:result) ``` Streaming results from multiple requests using continuation, ```elixir Wiki.SiteMatrix.new() |> Wiki.SiteMatrix.get!("dewiki") |> Wiki.Action.new() |> Wiki.Action.stream( action: :query, list: :recentchanges, rclimit: 5 ) |> Stream.take(10) |> Enum.flat_map(fn response -> response["query"]["recentchanges"] end) |> Enum.map(fn rc -> rc["timestamp"] <> " " <> rc["title"] end) ``` ## Wikibase The [Wikidata](https://www.wikidata.org/) project provides structured data for other wiki projects, and can be accessed through the Action API. Examples: Search for entities called "alphabet", ```elixir Wiki.SiteMatrix.new() |> Wiki.SiteMatrix.get!("wikidatawiki") |> Wiki.Action.new() |> Wiki.Action.get!( action: :wbsearchentities, search: "alphabet", language: :en ) ``` Search for entities with "Frank Zappa" anywhere in the description or contents, ```elixir Wiki.SiteMatrix.new() |> Wiki.SiteMatrix.get!("wikidatawiki") |> Wiki.Action.new() |> Wiki.Action.get!( action: :wbsearchentities, search: "alphabet", language: :en ) ``` Retrieve all data about a specific entity with ID "Q42", ```elixir Wiki.SiteMatrix.new() |> Wiki.SiteMatrix.get!("wikidatawiki") |> Wiki.Action.new() |> Wiki.Action.get!( action: :wbgetentities, ids: "Q42" ) ``` ## Defaults A few parameters are automatically added for convenience, but can be overridden if desired: * The `:format` parameter defaults to `:json`. * `:formatversion` defaults to `2`. Overriding to get pretty-printed JSON and the older response structure, ```elixir Wiki.Action.get!( action: query, meta: siteinfo, siprop: namespaces, format: jsonfm, formatversion: 1 ) ``` """ alias Wiki.{Action.Session, Error, SiteMatrix, Util} @type client_option :: {:accumulate, true} | {:adapter, module()} | {:debug, true} | {:user_agent, binary()} @typedoc """ - `:accumulate` - Merge results from each step of a pipeline, rather than overwriting with the latest response. Useful when carrying along authentication or following continuations. - `:adapter` - Override the HTTP adapter - `:debug` - Turn on verbose logging by setting to `true` - `:disable_compression` - Disable the transparent gzip codec, rather than automatically detecting based on HTTP headers. - `:user_agent` - Override the user-agent header string """ @type client_options :: [client_option()] @default_timeout 60_000 @default_adapter {Tesla.Adapter.Hackney, recv_timeout: @default_timeout} @doc """ Create a new client session ## Arguments - `site` - SiteMatrix.Spec or raw `api.php` endpoint for the wiki you will connect to. For example, "https://en.wikipedia.org/w/api.php". - `opts` - configuration options which modify client behavior """ @spec new(String.t() | SiteMatrix.Spec.t(), client_options()) :: Session.t() def new(site, opts \\ []) def new(%SiteMatrix.Spec{} = site, opts) do site |> SiteMatrix.action_api() |> new(site.opts ++ opts) end def new(url, opts) do %Session{ __client__: client(url, opts) } end @doc """ Make requests to authenticate a client session. This should only be done using a [bot username and password](https://www.mediawiki.org/wiki/Manual:Bot_passwords), which can be created for any normal user account. ## Arguments - `session` - Base session pointing to a wiki. - `username` - Bot username, may be different than the final logged-in username. - `password` - Bot password. Protect this string, it allows others to take on-wiki actions on your behalf. ## Return value Authenticated session object. """ @spec authenticate(Session.t(), String.t(), String.t()) :: Session.result() def authenticate(session, username, password) do with {:ok, login_token} <- get(session, action: :query, meta: :tokens, type: :login ) do post(login_token, action: :login, lgname: username, lgpassword: password, lgtoken: login_token.result["query"]["tokens"]["logintoken"] ) end end @doc """ Assertive variant of `authenticate` """ @spec authenticate!(Session.t(), String.t(), String.t()) :: Session.t() def authenticate!(session, username, password) do case authenticate(session, username, password) do {:ok, session} -> session {:error, error} -> raise error end end @doc """ Make an API GET request ## Arguments - `session` - `Wiki.Action.Session` object. - `params` - Keyword list of query parameters as atoms or strings. ## Return value Session object with its `.result` populated. """ @spec get(Session.t(), keyword) :: Session.result() def get(session, params), do: request(session, :get, query: normalize_params(params)) @doc """ Assertive variant of `get`. """ @spec get!(Session.t(), keyword) :: Session.t() def get!(session, params) do case get(session, params) do {:ok, result} -> result {:error, error} -> raise error end end @doc """ Make an API POST request. ## Arguments - `session` - `Wiki.Action.Session` object. If credentials are required for this action, you should have created this object with the `authenticate/3` function. - `params` - Keyword list of query parameters as atoms or strings. ## Return value Session object with a populated `:result` attribute. """ @spec post(Session.t(), keyword) :: Session.result() def post(session, params), do: request(session, :post, body: normalize_params(params)) @doc """ Assertive variant of `post`. """ @spec post!(Session.t(), keyword) :: Session.t() def post!(session, params) do case post(session, params) do {:ok, result} -> result {:error, error} -> raise error end end @doc """ Make a GET request and follow continuations until exhausted or the stream is closed. ## Arguments - `session` - `Wiki.Action.Session` object. - `params` - Keyword list of query parameters as atoms or strings. ## Return value Enumerable `Stream`, where each returned chunk is a raw result map, possibly containing multiple records. This corresponds to `session.result` from the other entry points. """ @spec stream(Session.t(), keyword) :: Enumerable.t() def stream(session, params) do Stream.resource( fn -> {session, :start} end, fn {prev, :start} -> do_stream_get(prev, params) {prev, :cont} -> get_continuation(prev.result) |> case do nil -> {:halt, nil} continue -> do_stream_get(prev, params ++ continue) end end, fn _ -> nil end ) end defp do_stream_get(session, params) do next = get!(session, params) {[next.result], {next, :cont}} end defp get_continuation(result) do case result do # TODO: Test that a cross between a list and query can be continued # in both dimensions. %{"continue" => continue} -> Map.to_list(continue) %{"query-continue" => continue} -> continue |> Map.values() |> Enum.flat_map(&Map.to_list/1) _ -> nil end end @spec request(Session.t(), :get | :post, keyword) :: Session.result() defp request(session, method, params) do # TODO: This can be extracted into a generic StatefulAdapter now. opts = [opts: session.state] ++ params ++ [method: method] with {:ok, result} <- Tesla.request(session.__client__, opts), {:ok, result} <- validate(result) do {:ok, %Session{ __client__: session.__client__, result: result.body, state: Keyword.delete(result.opts, :opts) }} else {:error, error = %Error{}} -> {:error, error} {:error, error} -> {:error, %Error{message: "#{inspect(error)}"}} end end @spec normalize_params(keyword) :: keyword defp normalize_params(params) do defaults = [ format: :json, formatversion: 2 ] (defaults ++ params) |> remove_boolean_false() |> pipe_lists() |> Enum.sort() |> Enum.dedup() end defp remove_boolean_false(params) do params |> Enum.filter(fn {_, v} -> v not in [false, nil] end) end defp pipe_lists(params) do params |> Enum.map(fn {k, v} when is_list(v) -> {k, pipe_list(v)} entry -> entry end) end defp pipe_list(values) do if Enum.any?(values, fn v -> String.contains?(to_string(v), "|") end) do # Use a special join character because pipe would conflict with the value. unit_separator = "\x1f" Enum.join([""] ++ values, unit_separator) else Enum.join(values, "|") end end defp validate(result) do with nil <- validate_http_status(result.status), nil <- validate_body_type(result.body), nil <- validate_api_errors(result.body) do {:ok, result} end end defp validate_http_status(status) do case status do status when status >= 200 and status < 300 -> nil status -> {:error, %Error{message: "Error received with HTTP status #{status}"}} end end defp validate_body_type(body) do with body when is_map(body) <- body, body when body != %{} <- body do nil else _ -> {:error, %Error{message: "Empty response"}} end end defp validate_api_errors(body) do with nil <- body["error"], nil <- body["errors"] do nil else error when is_map(error) -> {:error, %Error{message: summarize_legacy_error(error)}} errors when is_list(errors) -> {:error, %Error{message: summarize_new_error(errors)}} end end defp summarize_legacy_error(error) do error["info"] || error["code"] || "Unknown error (legacy format)" end defp summarize_new_error(errors) do # TODO: multiple errors case(List.first(errors)) do %{"text" => text} -> text %{"html" => html} -> html %{"key" => key, "params" => params} -> [key, params] |> List.flatten() |> Enum.join("-") %{"code" => code} -> code _ -> "unknown" end end @spec client(binary(), keyword()) :: Tesla.Client.t() defp client(url, opts) do adapter = opts[:adapter] || @default_adapter user_agent = opts[:user_agent] || Util.default_user_agent() default_middleware = [ {Tesla.Middleware.BaseUrl, url}, {Tesla.Middleware.Timeout, timeout: @default_timeout}, Wiki.StatefulClient.CookieJar, Tesla.Middleware.FormUrlencoded, {Tesla.Middleware.Headers, [ {"user-agent", user_agent} ]}, Tesla.Middleware.FollowRedirects, Tesla.Middleware.JSON, Tesla.Middleware.Compression ] Enum.reduce(opts, default_middleware, fn {option, on}, middleware -> cond do on == false -> middleware option == :accumulate -> middleware ++ [Wiki.StatefulClient.CumulativeResult] option == :disable_compression -> middleware -- [Tesla.Middleware.Compression] option == :debug -> middleware ++ [Tesla.Middleware.Logger] true -> middleware end end) |> Tesla.client(adapter) end end defmodule Wiki.StatefulClient.CookieJar do @moduledoc false @behaviour Tesla.Middleware @impl true def call(env, next, _opts) do env = set_cookie_header(env, env.opts[:cookies]) with {:ok, env} <- Tesla.run(env, next) do {:ok, merge_cookies(env, env.opts[:cookies])} end end @spec set_cookie_header(Tesla.Env.t(), nil | map) :: Tesla.Env.t() defp set_cookie_header(env, cookies_opt) defp set_cookie_header(env, nil), do: env defp set_cookie_header(env, cookies) do serialized = Enum.map_join(cookies, "; ", fn {key, value} -> key <> "=" <> value end) Tesla.put_headers(env, [{"cookie", serialized}]) end @spec merge_cookies(Tesla.Env.t(), nil | map) :: Tesla.Env.t() defp merge_cookies(env, old_cookies) defp merge_cookies(env, nil), do: merge_cookies(env, %{}) defp merge_cookies(env, old_cookies) do merged_cookies = Map.merge(old_cookies, extract_cookie_headers(env)) Tesla.put_opt(env, :cookies, merged_cookies) end @spec extract_cookie_headers(Tesla.Env.t()) :: map defp extract_cookie_headers(env) do Tesla.get_headers(env, "set-cookie") |> Enum.map(&SetCookie.parse/1) |> Enum.into(%{}, fn %{key: k, value: v} -> {k, v} end) end end defmodule Wiki.StatefulClient.CumulativeResult do @moduledoc false @behaviour Tesla.Middleware @impl true def call(env, next, _opts) do with {:ok, env} <- Tesla.run(env, next) do accumulated = recursive_merge(env.opts[:accumulated_result] || %{}, env.body) {:ok, Tesla.put_opt(env, :accumulated_result, accumulated) |> Tesla.put_body(accumulated)} end end @spec recursive_merge(map, map) :: map defp recursive_merge(%{} = v1, %{} = v2), do: Map.merge(v1, v2, &recursive_merge/3) # TODO: _key can be dropped @spec recursive_merge(String.t(), map | String.t(), map | String.t()) :: map defp recursive_merge(_key, v1, v2) defp recursive_merge(_key, %{} = v1, %{} = v2), do: recursive_merge(v1, v2) defp recursive_merge(_key, v1, v2) when is_list(v1) and is_list(v2), do: v1 ++ v2 defp recursive_merge(_key, v1, v2) when v1 == v2, do: v1 end