defmodule HTTPX do @moduledoc ~S""" Simple HTTP(s) client with integrated auth methods. """ use HTTPX.Log alias HTTPX.{Request, RequestError, Response} @type post_body :: binary | {:urlencoded, map | keyword} | {:file, String.t()} | {:json, term} | {:multipart, list} @post_header_urlencoded {"Content-Type", "application/x-www-form-urlencoded"} @post_header_json {"Content-Type", "application/json"} @post_header_file {"Content-Type", "application/octet-stream"} @content_encoding_gzip {"Content-Encoding", "gzip"} @content_encoding_compress {"Content-Encoding", "compress"} @content_encoding_deflate {"Content-Encoding", "deflate"} @content_encoding_br {"Content-Encoding", "br"} @doc ~S""" Performs a get request. For options see: `&request/3`. """ @spec get(String.t(), keyword) :: {:ok, Response.t()} | {:error, term} def get(url, options \\ []), do: :get |> request(url, options) @doc ~S""" Performs a post request, passing the body in the options. For options see: `&request/3`. """ @spec post(String.t(), post_body, keyword) :: {:ok, Response.t()} | {:error, term} def post(url, body, options \\ []) do with {:ok, opts} <- body_encoding(body, options) do request(:post, url, opts) end end @doc ~S""" Performs a patch request, passing the body in the options. For options see: `&request/3`. """ @spec patch(String.t(), post_body, keyword) :: {:ok, Response.t()} | {:error, term} def patch(url, body, options \\ []) do with {:ok, opts} <- body_encoding(body, options) do request(:patch, url, opts) end end @doc ~S""" Performs a put request, passing the body in the options. For options see: `&request/3`. """ @spec put(String.t(), post_body, keyword) :: {:ok, Response.t()} | {:error, term} def put(url, body, options \\ []) do with {:ok, opts} <- body_encoding(body, options) do request(:put, url, opts) end end @doc ~S""" Performs a delete request. For options see: `&request/3`. """ @spec delete(String.t(), keyword) :: {:ok, Response.t()} | {:error, term} def delete(url, options \\ []), do: request(:delete, url, options) @doc ~S""" Performs a request. The given `method` is used and the `url` is called. The following options can be set: * `:body`, the body to send with the request. * `:params`, a map containing query params. * `:headers`, list of header tuples. * `:settings`, options to pass along to `:hackney`. * `:fail`, will error out any request with a non 2xx response code, when set to true. * `:auth`, set authorization options. * `:format`, set to parse. (Like `:json`) * `:retry`, set to retry the request. See the retry options. """ @spec request(term, String.t(), keyword) :: {:ok, Response.t()} | {:error, term} def request(method, url, options \\ []) do method |> Request.prepare(url, options) |> perform() end @doc ~S""" Perform a given request. """ @spec perform(HTTPX.Request.t()) :: {:ok, HTTPX.Response.t()} | {:error, term} def perform(request) do %{ method: m, url: u, body: b, headers: h, settings: s, format: format, fail: fail } = __MODULE__.Process.pre_request(request) Log.log(m, u, h, b) response = :hackney.request(m, u, h, b, s) Log.log(m, u, h, b, response) case response |> __MODULE__.Process.post_request() |> parse_response(format, s) |> handle_response(fail) do {:ok, response} -> __MODULE__.Process.post_parse(response) error -> error end end @doc ~S""" Performs a request on all IPs associated with the host DNS. For more information see: `request/3`. """ @spec multi_request(term, String.t(), keyword) :: %{ok: map, error: map} def multi_request(method, url, opts \\ []) do uri = %{host: host} = URI.parse(url) opts = Keyword.update(opts, :headers, [{"Host", host}], &[{"Host", host} | &1]) host |> String.to_charlist() |> :inet_res.lookup(:in, :a) |> Enum.map(&(&1 |> Tuple.to_list() |> Enum.join("."))) |> Enum.map(&{&1, request(method, to_string(%{uri | host: &1}), opts)}) |> Enum.group_by(&elem(elem(&1, 1), 0)) |> Enum.into(%{}) |> Map.update(:ok, %{}, &Enum.into(&1, %{}, fn {ip, r} -> {ip, elem(r, 1)} end)) |> Map.update(:error, %{}, &Enum.into(&1, %{})) end ### Helpers ### defp parse_response({:ok, status, resp_headers, resp_body}, format, opts) do case parse_body(resp_body, format, opts) do {:ok, body} -> {:ok, %Response{ status: status, headers: resp_headers, body: body }} error -> error end end defp parse_response({:ok, status, resp_headers}, format, opts) do parse_response({:ok, status, resp_headers, ""}, format, opts) end defp parse_response(error, _format, _opts), do: error defp parse_body(body, format, opts) defp parse_body(body, :text, _opts), do: {:ok, body} defp parse_body(body, :json, _opts), do: body |> Jason.decode() defp parse_body(body, :json_atoms, _opts), do: body |> Jason.decode(keys: :atoms) defp parse_body(body, :json_atoms!, _opts), do: body |> Jason.decode(keys: :atoms!) defp parse_body(body, :stream, opts) do stream_opts = opts[:stream] || [] with {:ok, streamer} <- create_stream_splitter(stream_opts[:format] || :chunked, stream_opts) do {:ok, Stream.resource(fn -> {body, <<>>} end, streamer, fn _ref -> :ok end)} end end defp create_stream_splitter(:chunks, _opts) do {:ok, fn {ref, _buffer} -> case :hackney.stream_body(ref) do {:ok, chunk} -> {[chunk], {ref, nil}} :done -> {:halt, ref} {:error, reason} -> raise "Error reading HTTP stream. (#{inspect(reason)})" end end} end defp create_stream_splitter(:newline, opts) do create_stream_splitter(:separated, Keyword.merge(opts, separator: ~r/\r?\n/, ends_with: "\n")) end # credo:disable-for-next-line defp create_stream_splitter(:separated, opts) do if separator = opts[:separator] do ends_with? = cond do e = opts[:ends_with] -> &String.ends_with?(&1, e) is_binary(separator) -> &String.ends_with?(&1, separator) Regex.regex?(separator) -> source = Regex.source(separator) if Regex.escape(source) == source do &String.ends_with?(&1, source) else &Regex.match?(Regex.compile!(source <> "$"), &1) end end {:ok, fn {ref, buffer} -> case :hackney.stream_body(ref) do {:ok, chunk} -> items = String.split(buffer <> chunk, separator, trim: true) if ends_with?.(chunk) do {items, {ref, <<>>}} else {buffer, items} = List.pop_at(items, -1) {items, {ref, buffer}} end :done -> {:halt, ref} {:error, reason} -> raise "Error reading HTTP stream. (#{inspect(reason)})" end end} else {:error, :stream_missing_separator} end end defp create_stream_splitter(_, _), do: {:error, :invalid_stream_format} defp handle_response({:ok, %{status: status}}, true) when status < 200 or status >= 300, do: {:error, :http_status_failure} defp handle_response(response, _), do: response ### Query Encoding ### @doc ~S""" Encode a map as query. """ @spec query_encode(map) :: binary def query_encode(data) do data |> query_encode("") |> query_encode_to_binary() end defp query_encode(data, prefix) do Enum.flat_map(data, fn {field, value} -> key = if prefix == "", do: URI.encode_www_form(to_string(field)), else: [prefix, ?[, URI.encode_www_form(to_string(field)), ?]] if is_map(value) or is_list(value) do query_encode(value, key) else [?&, key, ?=, URI.encode_www_form(to_string(value))] end end) end defp query_encode_to_binary([?& | data]), do: IO.iodata_to_binary(data) defp query_encode_to_binary(data), do: IO.iodata_to_binary(data) defp body_encoding({:urlencoded, body}, options) do {:ok, options |> Keyword.update(:headers, [@post_header_urlencoded], &[@post_header_urlencoded | &1]) |> Keyword.put(:body, query_encode(body)) |> body_maybe_compress(options[:compress])} end defp body_encoding({:file, body}, options) do {:ok, options |> Keyword.update(:headers, [@post_header_file], &[@post_header_file | &1]) |> Keyword.put(:body, body) |> body_maybe_compress(options[:compress])} end defp body_encoding({:json, body}, options) do case Jason.encode(body) do {:ok, encoded} -> {:ok, options |> Keyword.update(:headers, [@post_header_json], &[@post_header_json | &1]) |> Keyword.put(:body, encoded) |> body_maybe_compress(options[:compress])} {:error, _} -> {:error, :body_not_valid_json} end end defp body_encoding({:multipart, data}, options) do body = Enum.map( data, fn {name, {:file, file}} -> {:file, file, name, []} {name, {:file, file, headers}} -> {:file, file, name, headers} {name, {:binfile, data}} -> encode_mp_binfile(name, data) {name, {:binfile, data, opts}} -> encode_mp_binfile(name, data, opts) {name, value} -> {name, value} end ) {:ok, options |> Keyword.put(:body, {:multipart, body}) |> body_maybe_compress(options[:compress])} end defp body_encoding(body, options), do: {:ok, Keyword.put(options, :body, body) |> body_maybe_compress(options[:compress])} defp body_maybe_compress(options, compression) defp body_maybe_compress(options, compression) when compression in [nil, :identity], do: options defp body_maybe_compress(options, :gzip) do options |> Keyword.update(:headers, [@content_encoding_gzip], &[@content_encoding_gzip | &1]) |> Keyword.update!(:body, &:zlib.gzip/1) end defp body_maybe_compress(options, :compress) do options |> Keyword.update(:headers, [@content_encoding_compress], &[@content_encoding_compress | &1]) |> Keyword.update!(:body, &:zlib.compress/1) end defp body_maybe_compress(options, :deflate) do options |> Keyword.update(:headers, [@content_encoding_deflate], &[@content_encoding_deflate | &1]) |> Keyword.update!(:body, &:zlib.compress/1) end defp body_maybe_compress(options, :br) do options |> Keyword.update(:headers, [@content_encoding_br], &[@content_encoding_br | &1]) |> Keyword.update!(:body, &apply(:brotli, :encode, [&1])) rescue UndefinedFunctionError -> reraise "Missing `:brotli` dependency.", __STACKTRACE__ end defp encode_mp_binfile(name, data, opts \\ []) do filename = opts[:filename] || name mime = cond do m = opts[:mime] -> m String.printable?(data) -> "text/plain" :fallback -> "application/octet-stream" end {name, data, {"form-data", [{"name", "\"#{name}\""}, {"filename", "\"#{filename}\""}]}, [{"content-type", mime}]} end ## Bangified ### @doc ~S""" Performs a get request. For options see: `&get/2`. """ @spec get!(String.t(), keyword) :: Response.t() | no_return def get!(url, options \\ []) do case request(:get, url, options) do {:ok, response} -> response {:error, reason} -> context = [ url: url, options: options ] raise RequestError.exception(reason, nil, context) end end @doc ~S""" Performs a post request, passing the body in the options. For options see: `&post/3`. """ @spec post!(String.t(), post_body, keyword) :: Response.t() | no_return def post!(url, body, options \\ []) do case post(url, body, options) do {:ok, response} -> response {:error, reason} -> context = [ url: url, body: body, options: options ] raise RequestError.exception(reason, nil, context) end end @doc ~S""" Performs a patch request, passing the body in the options. For options see: `&patch/3`. """ @spec patch!(String.t(), post_body, keyword) :: Response.t() | no_return def patch!(url, body, options \\ []) do case patch(url, body, options) do {:ok, response} -> response {:error, reason} -> context = [ url: url, body: body, options: options ] raise RequestError.exception(reason, nil, context) end end @doc ~S""" Performs a post request, passing the body in the options. For options see: `&put/3`. """ @spec put!(String.t(), post_body, keyword) :: Response.t() | no_return def put!(url, body, options \\ []) do case put(url, body, options) do {:ok, response} -> response {:error, reason} -> context = [ url: url, body: body, options: options ] raise RequestError.exception(reason, nil, context) end end @doc ~S""" Performs a delete request. For options see: `&delete/2`. """ @spec delete!(String.t(), keyword) :: Response.t() | no_return def delete!(url, options \\ []) do case request(:delete, url, options) do {:ok, response} -> response {:error, reason} -> context = [ url: url, options: options ] raise RequestError.exception(reason, nil, context) end end ## Optimize Process On Load ## @on_load :optimize @doc ~S""" Optimize HTTPX processors. This is automatically called on HTTPX load. So there is no need to call it manually. The function is idempotent, so there is no harm in calling it. """ @spec optimize :: :ok def optimize do # This fix is required to make optimize/0 compatible with Distillery. spawn(fn -> Enum.each(~w(kernel stdlib compiler elixir logger)a, &:application.ensure_started/1) HTTPX.Process.optimize() end) :ok end end