defmodule ExFussballDeScraper.Scraper do @moduledoc """ Grabs some content from a fussball.de team website. """ @css_defaults %{ team_name: ".stage-team h2", matches: "#id-team-matchplan-table tbody tr", matches_match_id: "td:last-child a", matches_match_id_splitter: " ", matches_match_headline: "tr:first-child td:first-child", matches_match_headline_splitter: "|", matches_match_club_names: "td.column-club .club-name", current_table: "#team-fixture-league-tables > table" } @doc """ Returns the next matches from a fussball.de team website. """ @spec next_matches(String, String) :: {:ok, Map, Integer} | {:error, Atom, Integer} def next_matches(team_rewrite, team_id) do ExFussballDeScraper.GenServer.get(team_rewrite, team_id) |> grab_next_matches() end defp grab_next_matches({:error, reason, created_at}), do: {:error, reason, created_at} defp grab_next_matches({:ok, html, created_at}) do map = html |> find_team_name() |> find_matches() |> get_result() {:ok, map, created_at} end @doc """ Returns the current table from a fussball.de team website. """ def current_table(team_rewrite, team_id) do ExFussballDeScraper.GenServer.get(team_rewrite, team_id) |> grab_current_table() end defp grab_current_table({:error, reason, created_at}), do: {:error, reason, created_at} defp grab_current_table({:ok, html, created_at}) do map = html |> find_team_name() |> find_table() |> remove_images() |> remove_links() |> replace_bootstrap_3_classes() |> replace_bootstrap_3_glyphicons() |> get_result() {:ok, map, created_at} end defp get_result(%{result: result}) do result end defp find_team_name(html) do team_name = html |> Floki.find(get_css_path(:team_name)) |> Floki.text() %{html: html, result: %{team_name: team_name}} end defp find_matches(%{html: html, result: result}) do matches = html |> Floki.find(get_css_path(:matches)) |> Enum.chunk(3) |> Enum.map(&extract_match/1) %{html: html, result: Map.put(result, :matches, matches)} end defp extract_match(markup) do id = Floki.find(markup, get_css_path(:matches_match_id)) |> Enum.filter(fn({_, _, [first | _rest]}) -> is_binary(first) end) |> Floki.text() |> String.trim() |> String.split(get_css_path(:matches_match_id_splitter)) |> List.last() |> String.trim() [start_at | [competition]] = Floki.find(markup, get_css_path(:matches_match_headline)) |> List.first() |> Floki.text() |> String.split(get_css_path(:matches_match_headline_splitter)) |> Enum.map(&String.trim/1) club_names = Floki.find(markup, get_css_path(:matches_match_club_names)) %{ id: id, start_at: start_at |> datetime_text_to_iso(), competition: competition, home: club_names |> List.first() |> Floki.text() |> String.trim(), guest: club_names |> List.last() |> Floki.text() |> String.trim() } end defp find_table(%{html: html, result: result}) do table = html |> Floki.find(get_css_path(:current_table)) |> Floki.raw_html() |> String.replace("\n", "") |> String.replace("\t", "") %{html: html, result: Map.put(result, :current_table, table)} end defp remove_images(%{html: html, result: %{current_table: current_table} = result}) do current_table = Regex.replace(~r//, current_table, "") current_table = Regex.replace(~r/