defmodule Fmcsa do @moduledoc """ Documentation for Fmcsa. """ require Logger alias Marshall alias Fmcsa.Http @states [ "AK", "AL", "AR", "AZ", "CA", "CO", "CT", "DE", "FL", "GA", "HI", "IA", "ID", "IL", "IN", "KS", "KY", "LA", "MA", "MD", "ME", "MI", "MN", "MO", "MS", "MT", "NC", "ND", "NE", "NH", "NJ", "NM", "NV", "NY", "OH", "OK", "OR", "PA", "RI", "SC", "SD", "TN", "TX", "UT", "VA", "VT", "WA", "WI", "WV", "WY" ] def fetch_companies_by_state(state) do {status, response} = Http.fetch_companies_by_state(state) case(status) do :error -> {:error, response.message} :ok -> html = response.body {:ok, html} = Floki.parse_document(html) result = Floki.find(html, "table") c = Floki.find(result, "td.MiddleTDFMCSA") Marshall.company_names(c) end end def fetch_company_names do Enum.map(@states, fn state -> {_, companies} = fetch_companies_by_state(state) companies end) end def fetch_company_profile(url, acc \\ 0) do ## sleep upto 5 seconds so fcma server can calm down and not timeout time = :rand.uniform(5000) :timer.sleep(time) {status, response} = Http.fetch_company_profile(url) case response do :errconnrefused -> if(acc > 5) do time = :rand.uniform(5000) :timer.sleep(time) acc = acc + 1 fetch_company_profile(url,acc) end _-> html = response.body {:ok, html} = Floki.parse_document(html) main = Floki.find(html, "tr.MiddleTDFMCSA") alt = Floki.find(html, "tr.MiddleAltTDFMCSA") Marshall.profile({main, alt}) end end def fetch_company_profile!(url) do {status, response} = Http.fetch_company_profile(url) html = response.body {:ok, html} = Floki.parse_document(html) main = Floki.find(html, "tr.MiddleTDFMCSA") alt = Floki.find(html, "tr.MiddleAltTDFMCSA") {_, profile} = Marshall.profile({main, alt}) profile end def all(state \\ "ALL") do response = case(state) do "SYNC" -> companies = Fmcsa.fetch_company_names() count = Enum.count(companies) Logger.info("Found " <> Integer.to_string(count) <> " companies") Enum.map(companies, fn x -> {_, url} = x Fmcsa.Company.Supervisor.start(x) Fmcsa.Company.Server.fetch_profile(x) end) "ALL" -> companies = Fmcsa.fetch_company_names() count = Enum.count(companies) Logger.info("Found " <> Integer.to_string(count) <> " companies") profiles = Enum.map(companies, fn x -> {_, url} = x Fmcsa.fetch_company_profile(url) end) count = Enum.count(profiles) Logger.info("Fetched " <> Integer.to_string(count) <> " profiles") _ -> {_, response} = Fmcsa.fetch_companies_by_state(state) count = Enum.count(response) Logger.info("Found " <> Integer.to_string(count) <> " companies") profiles = Enum.map(response, fn x -> {company, url} = x Logger.info("Fetching " <> company <> " profile") Fmcsa.fetch_company_profile(url) end) count = Enum.count(profiles) Logger.info("Fetched " <> Integer.to_string(count) <> " profiles") end {:ok, response} end end