defmodule Exoplanet.Parser do @moduledoc false require Logger def parse(%Exoplanet.Config{sources: sources} = config) do sources |> Task.async_stream(__MODULE__, :parse, [config], ordered: false, timeout: to_timeout(second: config.feed_timeout) ) |> Stream.flat_map(fn result -> case result do {:ok, posts} -> posts _ -> [] end end) end def parse({url, %{name: name} = _attrs}, config) do # TODO: Apply filters (e.g., remove images from posts) case Req.get(url, Application.get_env(:exoplanet, :planet_req_options, [])) do {:ok, %{status: 200, body: body}} -> items = if String.contains?(body, " items}} -> parsed_items = Enum.map(items, fn item -> title = item["title"] content = item["description"] authors = List.wrap(item["author"] || name) categories = item["categories"] id = item["link"] || get_in(item, ["guid", "value"]) published = item["pub_date"] && Exoplanet.DateTimeParser.parse!(item["pub_date"]) attrs = %{ authors: authors, title: title, categories: categories, id: id, published: published } {attrs, content} end) parsed_items {:error, reason} -> Logger.error("something went wrong while parsing feed #{url}, reason: #{inspect(reason)}") [] end end @doc false def parse_atom(url, body, name) do case FastRSS.parse_atom(body) do {:ok, %{"entries" => entries}} -> parsed_entries = Enum.map(entries, fn entry -> title = get_in(entry, ["title", "value"]) content = get_in(entry, ["content", "value"]) authors = get_in(entry, ["authors", Access.all(), "name"]) categories = get_in(entry, ["categories", Access.all(), "term"]) id = Map.get(entry, "id") published = entry["published"] && NaiveDateTime.from_iso8601!(entry["published"]) updated = entry["updated"] && NaiveDateTime.from_iso8601!(entry["updated"]) authors = if authors == [], do: [name], else: authors attrs = %{ authors: authors, title: title, categories: categories, id: id, published: published, updated: updated } {attrs, content} end) parsed_entries {:error, reason} -> Logger.error("something went wrong while parsing feed #{url}, reason: #{inspect(reason)}") [] end end end