defmodule Mastomation.Notes do @moduledoc """ Download and search private profile notes for accounts you follow or that follow you. """ require Logger alias Mastomation @accounts_page_size 80 @max_relationship_query_chars 6_000 @default_file "notes.json" @type note_entry :: %{ username: String.t(), handle: String.t(), note: String.t() } @doc """ Downloads private notes for all followers and followed accounts, then stores them locally. The generated file contains entries with `username`, `handle`, and `note`. Returns `{:ok, path, entries}`. """ @spec download(String.t(), String.t(), keyword(), keyword()) :: {:ok, String.t(), [note_entry()]} | no_return() def download(instance, token, opts \\ [], req_opts \\ []) do validate_required!(instance, token) Logger.info("notes: resolving authenticated account...") self_id = get_self_id(instance, token, req_opts) Logger.info("notes: fetching following/followers accounts in parallel...") following_task = Task.async(fn -> fetch_accounts(instance, token, self_id, "following", req_opts) end) followers_task = Task.async(fn -> fetch_accounts(instance, token, self_id, "followers", req_opts) end) following = Task.await(following_task, :infinity) Logger.info("notes: fetched #{Enum.count(following)} following accounts") followers = Task.await(followers_task, :infinity) Logger.info("notes: fetched #{Enum.count(followers)} followers accounts") accounts = (following ++ followers) |> uniq_accounts() Logger.info("notes: unique accounts to process: #{Enum.count(accounts)}") Logger.info("notes: fetching private notes from relationships...") relationship_notes = fetch_relationship_notes(instance, token, accounts, req_opts) entries = build_entries(accounts, relationship_notes) path = notes_path(opts) body = JSON.encode!(%{ "downloaded_at" => DateTime.utc_now() |> DateTime.to_iso8601(), "entries" => entries }) Logger.info("notes: writing file to #{path}...") File.mkdir_p!(Path.dirname(path)) File.write!(path, body) Logger.info("notes: done, saved #{Enum.count(entries)} entries") {:ok, path, entries} end @doc """ Searches downloaded notes by terms and returns matching entries. Terms can be a string or list of strings and are matched case-insensitively against `username`, `handle`, and `note`. """ @spec search(String.t() | [String.t()], keyword()) :: {:ok, [note_entry()]} | {:error, :notes_file_not_found} def search(terms, opts \\ []) do path = notes_path(opts) case File.read(path) do {:ok, body} -> terms = normalize_terms(terms) entries = body |> JSON.decode!() |> Map.get("entries", []) |> Enum.map(fn entry -> %{ username: Map.get(entry, "username", ""), handle: Map.get(entry, "handle", ""), note: Map.get(entry, "note", "") } end) |> filter_entries(terms) {:ok, entries} {:error, :enoent} -> {:error, :notes_file_not_found} {:error, _reason} -> {:error, :notes_file_not_found} end end @doc """ Returns the default notes storage path. """ @spec default_notes_path() :: String.t() def default_notes_path do Path.join([Mastomation.xdg_data_home(), "mastomation", @default_file]) end @spec notes_path(keyword()) :: String.t() defp notes_path(opts) do Keyword.get(opts, :path, default_notes_path()) end @spec get_self_id(String.t(), String.t(), keyword()) :: String.t() defp get_self_id(instance, token, req_opts) do %{"id" => id} = Mastomation.Client.get_json!( "#{instance}/api/v1/accounts/verify_credentials", token, req_opts ) id end @type accounts_page_state :: %{ max_id: String.t() | nil, acc: [map()], seen_ids: MapSet.t(String.t()), page: pos_integer() } @spec fetch_accounts(String.t(), String.t(), String.t(), String.t(), keyword()) :: [map()] defp fetch_accounts(instance, token, self_id, relationship, req_opts) do fetch_accounts_page( instance, token, self_id, relationship, req_opts, %{ max_id: nil, acc: [], seen_ids: MapSet.new(), page: 1 } ) end @spec fetch_accounts_page( String.t(), String.t(), String.t(), String.t(), keyword(), accounts_page_state() ) :: [map()] defp fetch_accounts_page(instance, token, self_id, relationship, req_opts, state) do %{max_id: max_id, acc: acc, seen_ids: seen_ids, page: page_number} = state Logger.info("notes: fetching #{relationship} page #{page_number}...") url = accounts_url(instance, self_id, relationship, max_id) page = Mastomation.Client.get_json!(url, token, req_opts) case page do [] -> Logger.info("notes: #{relationship} complete (#{Enum.count(acc)} total accounts)") acc list when is_list(list) -> new_accounts = Enum.reject(list, fn account -> MapSet.member?(seen_ids, Map.get(account, "id")) end) updated_seen_ids = Enum.reduce(new_accounts, seen_ids, fn account, ids -> MapSet.put(ids, Map.get(account, "id")) end) running_total = Enum.count(acc) + Enum.count(new_accounts) Logger.info( "notes: #{relationship} page #{page_number} fetched (#{running_total} collected so far)" ) next_max_id = list |> List.last() |> Map.get("id") cond do new_accounts == [] -> Logger.info("notes: stopping #{relationship} pagination (no new accounts in page)") Logger.info("notes: #{relationship} complete (#{Enum.count(acc)} total accounts)") acc next_max_id in [nil, max_id] -> final = acc ++ new_accounts Logger.info("notes: #{relationship} complete (#{Enum.count(final)} total accounts)") final true -> fetch_accounts_page( instance, token, self_id, relationship, req_opts, %{ max_id: next_max_id, acc: acc ++ new_accounts, seen_ids: updated_seen_ids, page: page_number + 1 } ) end end end @spec accounts_url(String.t(), String.t(), String.t(), String.t() | nil) :: String.t() defp accounts_url(instance, self_id, relationship, max_id) do base = "#{instance}/api/v1/accounts/#{self_id}/#{relationship}" params = [{"limit", Integer.to_string(@accounts_page_size)}] params = if is_nil(max_id), do: params, else: [{"max_id", max_id} | params] "#{base}?#{URI.encode_query(params)}" end @spec uniq_accounts([map()]) :: [map()] defp uniq_accounts(accounts) do accounts |> Enum.reduce(%{}, fn account, acc -> Map.put(acc, Map.get(account, "id"), account) end) |> Map.values() end @spec fetch_relationship_notes(String.t(), String.t(), [map()], keyword()) :: map() defp fetch_relationship_notes(instance, token, accounts, req_opts) do batches = accounts |> Enum.map(&Map.get(&1, "id")) |> relationship_batches() total_batches = Enum.count(batches) batches |> Enum.with_index(1) |> Enum.reduce(%{}, fn {ids, index}, acc -> Logger.info("notes: processing relationship batch #{index}/#{total_batches}") query = URI.encode_query(Enum.map(ids, &{"id[]", &1})) url = "#{instance}/api/v1/accounts/relationships?#{query}" rows = Mastomation.Client.get_json!(url, token, req_opts) |> Enum.reduce(%{}, fn rel, rel_acc -> Map.put(rel_acc, Map.get(rel, "id"), Map.get(rel, "note", "")) end) Map.merge(acc, rows) end) end @spec relationship_batches([String.t()]) :: [[String.t()]] defp relationship_batches(ids) do Enum.reduce(ids, [[]], fn id, [current | rest] = acc -> candidate = current ++ [id] query_len = candidate |> Enum.map(&{"id[]", &1}) |> URI.encode_query() |> String.length() if query_len <= @max_relationship_query_chars do [candidate | rest] else [[id] | acc] end end) |> Enum.reverse() |> Enum.map(&Enum.reject(&1, fn id -> is_nil(id) or id == "" end)) |> Enum.reject(&(&1 == [])) end @spec build_entries([map()], map()) :: [note_entry()] defp build_entries(accounts, relationship_notes) do Enum.map(accounts, fn account -> id = Map.get(account, "id") %{ username: Map.get(account, "username", ""), handle: Map.get(account, "acct", ""), note: Map.get(relationship_notes, id, "") } end) end @spec normalize_terms(String.t() | [String.t()]) :: [String.t()] defp normalize_terms(terms) when is_binary(terms) do terms |> String.split(~r/\s+/, trim: true) |> Enum.map(&String.downcase/1) end defp normalize_terms(terms) when is_list(terms) do terms |> Enum.map(&to_string/1) |> Enum.map(&String.downcase/1) |> Enum.reject(&(&1 == "")) end @spec filter_entries([note_entry()], [String.t()]) :: [note_entry()] defp filter_entries(entries, []), do: entries defp filter_entries(entries, terms) do Enum.filter(entries, fn entry -> haystack = "#{entry.username} #{entry.handle} #{entry.note}" |> String.downcase() Enum.any?(terms, &String.contains?(haystack, &1)) end) end @spec validate_required!(String.t() | nil, String.t() | nil) :: :ok | no_return() defp validate_required!(instance, token) do cond do is_nil(instance) or instance == "" -> raise ArgumentError, "missing instance URL (set --instance-url or MASTODON_INSTANCE_URL)" is_nil(token) or token == "" -> raise ArgumentError, "missing access token (set --access-token or MASTODON_ACCESS_TOKEN)" true -> :ok end end end