defmodule CodeNameRaven.Protocol.Http do @moduledoc """ A standalone, low-level HTTP profiler using `Mint` and `:ssl` to measure connection phases (DNS, TCP, SSL handshakes, TTFB, and download times) and retrieve SSL certificate details. """ require Record # Define records for Erlang public_key to decode SSL certificates Record.defrecord(:otp_certificate, :OTPCertificate, Record.extract(:OTPCertificate, from_lib: "public_key/include/public_key.hrl")) Record.defrecord(:otp_tbs_certificate, :OTPTBSCertificate, Record.extract(:OTPTBSCertificate, from_lib: "public_key/include/public_key.hrl")) Record.defrecord(:otp_validity, :Validity, Record.extract(:Validity, from_lib: "public_key/include/public_key.hrl")) @type timing :: %{ dns_ms: non_neg_integer() | nil, tcp_ms: non_neg_integer() | nil, ssl_ms: non_neg_integer() | nil, ttfb_ms: non_neg_integer() | nil, download_ms: non_neg_integer() | nil, total_ms: non_neg_integer() } @type result :: %{ status: non_neg_integer(), headers: [{String.t(), String.t()}], body: String.t(), timing: timing(), cert_expiry_at: DateTime.t() | nil, redirects: non_neg_integer() } @doc """ Performs an HTTP request and returns connection metrics and certificate details. """ @spec request(atom() | String.t(), String.t(), keyword()) :: {:ok, result()} | {:error, String.t()} def request(method, url, opts \\ []) do max_redirects = case Keyword.get(opts, :follow_redirects, false) do true -> 5 false -> 0 n when is_integer(n) and n >= 0 -> n _ -> 0 end do_request(method, url, opts, max_redirects, 0, %{ dns_ms: 0, tcp_ms: 0, ssl_ms: 0, ttfb_ms: 0, download_ms: 0, total_ms: 0 }) end defp do_request(method, url, opts, max_redirects, redirect_count, accumulated_timing) do total_start = System.monotonic_time() with {:ok, uri} <- parse_url(url), {:ok, scheme, host, port, path} <- validate_uri(uri), {:ok, ip, dns_dur} <- probe_dns(host), timeout_ms = Keyword.get(opts, :timeout_ms, 10_000), tls_verify = Keyword.get(opts, :tls_verify, true), req_headers = normalize_headers(Keyword.get(opts, :headers, [])), req_body = normalize_body(Keyword.get(opts, :body)), mint_start = System.monotonic_time(), {:ok, conn, tcp_dur, ssl_dur, cert_expiry} <- open_connection(scheme, ip, host, port, tls_verify, timeout_ms) do req_info = %{host: host, path: path, method: method} case execute_mint_request(conn, req_info, req_headers, req_body, timeout_ms) do {:ok, resp, ttfb_end, download_end} -> hop_ttfb = System.convert_time_unit(ttfb_end - mint_start, :native, :millisecond) hop_download = calc_hop_download(download_end, ttfb_end) hop_total = System.convert_time_unit(System.monotonic_time() - total_start, :native, :millisecond) current_timing = %{ dns_ms: (accumulated_timing.dns_ms || 0) + dns_dur, tcp_ms: (accumulated_timing.tcp_ms || 0) + tcp_dur, ssl_ms: if(scheme == :https, do: (accumulated_timing.ssl_ms || 0) + ssl_dur, else: nil), ttfb_ms: (accumulated_timing.ttfb_ms || 0) + hop_ttfb, download_ms: (accumulated_timing.download_ms || 0) + hop_download, total_ms: accumulated_timing.total_ms + hop_total } handle_redirect_or_result( resp, current_timing, cert_expiry, max_redirects, redirect_count, url, method, opts ) {:error, reason} -> {:error, "HTTP request failed: #{inspect(reason)}"} end else {:error, reason} -> {:error, reason} end end defp handle_redirect_or_result(resp, timing, cert_expiry, max_redirects, redirect_count, url, method, opts) do status_code = resp[:status] headers = resp[:headers] || [] body = resp[:body] if status_code in [301, 302, 303, 307, 308] and max_redirects > 0 do case find_header(headers, "location") do {:ok, location} -> next_url = URI.merge(url, location) |> to_string() do_request( method, next_url, opts, max_redirects - 1, redirect_count + 1, timing ) _ -> emit_telemetry(timing, url, status_code) {:ok, build_result(status_code, headers, body, timing, cert_expiry, redirect_count)} end else emit_telemetry(timing, url, status_code) {:ok, build_result(status_code, headers, body, timing, cert_expiry, redirect_count)} end end defp build_result(status, headers, body, timing, cert_expiry, redirects) do %{ status: status, headers: headers, body: body, timing: timing, cert_expiry_at: cert_expiry, redirects: redirects } end defp calc_hop_download(nil, _ttfb_end), do: 0 defp calc_hop_download(download_end, ttfb_end) do System.convert_time_unit(download_end - ttfb_end, :native, :millisecond) end defp probe_dns(host) do dns_start = System.monotonic_time() case :inet.getaddr(to_charlist(host), :inet) do {:ok, ip} -> dur = System.convert_time_unit(System.monotonic_time() - dns_start, :native, :millisecond) {:ok, ip, dur} {:error, reason} -> {:error, "DNS resolution for #{host} failed: #{inspect(reason)}"} end end defp open_connection(scheme, ip, host, port, tls_verify, timeout_ms) do tcp_start = System.monotonic_time() case :gen_tcp.connect(ip, port, [:binary, active: false], timeout_ms) do {:ok, tcp_socket} -> tcp_dur = System.convert_time_unit(System.monotonic_time() - tcp_start, :native, :millisecond) upgrade_connection(scheme, tcp_socket, host, port, tls_verify, tcp_dur, timeout_ms) {:error, reason} -> {:error, "TCP connection to #{host}:#{port} failed: #{inspect(reason)}"} end end defp upgrade_connection(:https, tcp_socket, host, port, tls_verify, tcp_dur, timeout_ms) do ssl_start = System.monotonic_time() mint_opts = [ mode: :passive, protocols: [:http1], transport_opts: Keyword.put(ssl_options(host, tls_verify), :timeout, timeout_ms) ] case Mint.HTTP1.upgrade(:http, tcp_socket, :https, host, port, mint_opts) do {:ok, conn} -> ssl_dur = System.convert_time_unit(System.monotonic_time() - ssl_start, :native, :millisecond) {:ok, conn, tcp_dur, ssl_dur, peer_cert_expiry(conn)} {:error, reason} -> :gen_tcp.close(tcp_socket) {:error, "TLS handshake to #{host}:#{port} failed: #{inspect(reason)}"} end end defp upgrade_connection(:http, tcp_socket, host, port, _tls_verify, tcp_dur, _timeout_ms) do case Mint.HTTP1.upgrade(:http, tcp_socket, :http, host, port, mode: :passive, protocols: [:http1]) do {:ok, conn} -> {:ok, conn, tcp_dur, nil, nil} {:error, reason} -> :gen_tcp.close(tcp_socket) {:error, "connection to #{host}:#{port} failed: #{inspect(reason)}"} end end defp peer_cert_expiry(conn) do case conn |> Mint.HTTP.get_socket() |> :ssl.peercert() do {:ok, der_cert} -> extract_cert_expiry(der_cert) _ -> nil end end defp execute_mint_request(conn, req_info, req_headers, req_body, timeout_ms) do method_str = to_string(req_info.method) |> String.upcase() with {:ok, conn, ref} <- Mint.HTTP.request(conn, method_str, req_info.path, req_headers, req_body), {:ok, conn, resp, ttfb_end, download_end} <- receive_response(conn, ref, %{body: ""}, System.monotonic_time(), nil, timeout_ms) do Mint.HTTP.close(conn) {:ok, resp, ttfb_end, download_end} else {:error, conn, reason} -> Mint.HTTP.close(conn) {:error, reason} {:error, reason} -> {:error, reason} end end defp parse_url(url) do {:ok, URI.parse(url)} rescue e -> {:error, "Invalid URL: #{inspect(e)}"} end defp validate_uri(%URI{scheme: s, host: h, port: p, path: path, query: query}) do scheme = parse_scheme(s) cond do is_nil(scheme) -> {:error, "Unsupported scheme: #{inspect(s)}"} is_nil(h) or h == "" -> {:error, "Missing host in URL"} true -> port = p || if(scheme == :https, do: 443, else: 80) req_path = build_request_path(path, query) {:ok, scheme, h, port, req_path} end end defp parse_scheme("http"), do: :http defp parse_scheme("https"), do: :https defp parse_scheme(_), do: nil defp build_request_path(nil, nil), do: "/" defp build_request_path(nil, q), do: "/?#{q}" defp build_request_path(p, nil), do: p defp build_request_path(p, q), do: "#{p}?#{q}" defp ssl_options(host, tls_verify) do base_opts = [ server_name_indication: to_charlist(host), verify: if(tls_verify, do: :verify_peer, else: :verify_none) ] if tls_verify do base_opts ++ [ cacertfile: CAStore.file_path(), depth: 3, customize_hostname_check: [ match_fun: :public_key.pkix_verify_hostname_match_fun(:https) ] ] else base_opts end end defp receive_response(conn, ref, acc, mint_start, ttfb_end, timeout) do case Mint.HTTP.recv(conn, 0, timeout) do {:ok, conn, responses} -> {acc, ttfb_end, _download_start, done} = Enum.reduce(responses, {acc, ttfb_end, nil, false}, &reduce_response_part(&1, &2, ref)) if done or acc[:done] do download_end = System.monotonic_time() {:ok, conn, acc, ttfb_end, download_end} else receive_response(conn, ref, acc, mint_start, ttfb_end, timeout) end {:error, _conn, reason, _responses} -> {:error, reason} end end defp reduce_response_part({:status, ref, status}, {a, t_end, dl_start, _d}, ref) do {Map.put(a, :status, status), t_end || System.monotonic_time(), dl_start, false} end defp reduce_response_part({:headers, ref, headers}, {a, t_end, dl_start, _d}, ref) do {Map.put(a, :headers, headers), t_end || System.monotonic_time(), dl_start, false} end defp reduce_response_part({:data, ref, data}, {a, t_end, dl_start, _d}, ref) do t_end_actual = t_end || System.monotonic_time() {Map.update(a, :body, data, &(&1 <> data)), t_end_actual, dl_start || System.monotonic_time(), false} end defp reduce_response_part({:done, ref}, {a, t_end, dl_start, _}, ref) do t_end_actual = t_end || System.monotonic_time() {Map.put(a, :done, true), t_end_actual, dl_start, true} end defp reduce_response_part(_, acc, _ref), do: acc defp normalize_headers(nil), do: [] defp normalize_headers(headers) when is_list(headers) do Enum.map(headers, fn {name, value} -> {to_string(name), to_string(value)} %{name: name, value: value} -> {to_string(name), to_string(value)} %{"name" => name, "value" => value} -> {to_string(name), to_string(value)} end) end defp normalize_headers(headers) when is_map(headers) do Enum.map(headers, fn {name, value} -> {to_string(name), to_string(value)} end) end defp normalize_body(nil), do: nil defp normalize_body(body), do: to_string(body) defp find_header(headers, name) do target = String.downcase(name) case Enum.find(headers, fn {k, _v} -> String.downcase(k) == target end) do {_k, v} -> {:ok, v} nil -> :error end end defp extract_cert_expiry(der_cert) do decoded = :public_key.pkix_decode_cert(der_cert, :otp) tbs = otp_certificate(decoded, :tbsCertificate) validity = otp_tbs_certificate(tbs, :validity) not_after = otp_validity(validity, :notAfter) parse_asn1_date(not_after) rescue _ -> nil end defp parse_asn1_date({:utcTime, charlist}) do str = List.to_string(charlist) case Regex.run(~r/^(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})Z$/, str) do [_, yy, mm, dd, h, m, s] -> year = String.to_integer(yy) full_year = if year < 50, do: 2000 + year, else: 1900 + year case DateTime.from_iso8601("#{full_year}-#{mm}-#{dd}T#{h}:#{m}:#{s}Z") do {:ok, dt, _} -> dt _ -> nil end _ -> nil end end defp parse_asn1_date({:generalTime, charlist}) do str = List.to_string(charlist) case Regex.run(~r/^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})Z$/, str) do [_, yyyy, mm, dd, h, m, s] -> case DateTime.from_iso8601("#{yyyy}-#{mm}-#{dd}T#{h}:#{m}:#{s}Z") do {:ok, dt, _} -> dt _ -> nil end _ -> nil end end defp parse_asn1_date(_), do: nil defp emit_telemetry(timing, url, status) do :telemetry.execute([:raven, :protocol, :http, :stop], %{ dns_ms: timing.dns_ms, tcp_ms: timing.tcp_ms, ssl_ms: timing.ssl_ms, ttfb_ms: timing.ttfb_ms, download_ms: timing.download_ms, total_ms: timing.total_ms }, %{url: url, status: status}) end end