defmodule ExFussballDeScraper.Scraper do @moduledoc """ Grabs some content from a fussball.de team website. """ @css_defaults %{ team_name: ".stage-team h2", matches: "#id-team-matchplan-table tbody tr", matches_match_id: "td:last-child a", matches_match_headline: "td:first-child", matches_match_headline_splitter: "|", matches_match_club_names: "td.column-club .club-name", current_table: "#team-fixture-league-tables > table", season: "select[name=\"saison\"] option:first-child", season_split_at: "/", season_join_with: "-" } @doc """ Returns the next matches from a fussball.de team website. """ @spec next_matches(String, String) :: {:ok, Map, Integer} | {:error, Atom, Integer} def next_matches(team_rewrite, team_id) do ExFussballDeScraper.GenServer.get(team_rewrite, team_id) |> grab_next_matches() end defp grab_next_matches({:error, reason, created_at}), do: {:error, reason, created_at} defp grab_next_matches({:ok, doc, created_at}) do map = %{html: parse_document(doc), result: %{}} |> find_team_name() |> find_season() |> find_matches() |> get_result() {:ok, map, created_at} end @doc """ Returns the current table from a fussball.de team website. """ @spec current_table(String.t, String.t) :: {:ok, Map.t, Integer.t} | {:error, Atom.t, Integer.t} def current_table(team_rewrite, team_id) do ExFussballDeScraper.GenServer.get(team_rewrite, team_id) |> grab_current_table() end @doc """ Grabs the table html from the given page html. """ @spec grab_table(String.t) :: String.t def grab_table(doc) do doc |> parse_document() |> find_and_edit_table_html() end defp grab_current_table({:error, reason, created_at}), do: {:error, reason, created_at} defp grab_current_table({:ok, doc, created_at}) do map = %{html: parse_document(doc), result: %{}} |> find_team_name() |> find_season() |> find_table() |> get_result() {:ok, map, created_at} end defp get_result(%{result: result}) do result end defp find_team_name(%{html: html, result: result}) do team_name = html |> Floki.find(get_css_path(:team_name)) |> Floki.text() %{html: html, result: Map.put(result, :team_name, team_name)} end defp find_season(%{html: html, result: result}) do season = html |> Floki.find(get_css_path(:season)) |> Enum.map(&Floki.text/1) |> Enum.sort(&(&1 >= &2)) |> List.first() |> to_string() |> String.split(get_css_path(:season_split_at)) |> Enum.join(get_css_path(:season_join_with)) %{html: html, result: Map.put(result, :season, season)} end defp find_matches(%{html: html, result: result}) do matches = html |> Floki.find(get_css_path(:matches)) |> Enum.chunk_every(3) |> Enum.map(&extract_match/1) %{html: html, result: Map.put(result, :matches, matches)} end defp extract_match(markup) do id = markup |> Floki.find(get_css_path(:matches_match_id)) |> Enum.filter(fn({_, _, [first | _rest]}) -> is_binary(first) end) |> Floki.text() |> String.trim() [start_at | [competition]] = markup |> Floki.find(get_css_path(:matches_match_headline)) |> List.first() |> Floki.text() |> String.split(get_css_path(:matches_match_headline_splitter)) |> Enum.map(&String.trim/1) club_names = markup |> Floki.find(get_css_path(:matches_match_club_names)) %{ id: id, start_at: start_at |> datetime_text_to_iso(), competition: competition, home: club_names |> List.first() |> Floki.text() |> String.trim(), guest: club_names |> List.last() |> Floki.text() |> String.trim() } end defp find_table(%{html: page_html, result: result}) do table_html = find_and_edit_table_html(page_html) %{html: page_html, result: Map.put(result, :current_table, table_html)} end defp find_and_edit_table_html(page_html) do page_html |> Floki.find(get_css_path(:current_table)) |> Floki.raw_html() |> String.replace("\n", "") |> String.replace("\t", "") |> remove_images() |> remove_links() |> replace_bootstrap_3_classes() |> replace_bootstrap_3_glyphicons() end defp remove_images(table_html) do table_html = Regex.replace(~r//, table_html, "") Regex.replace(~r/