defmodule Cldr.Http do @moduledoc """ Supports securely downloading https content. """ @cldr_unsafe_https "CLDR_UNSAFE_HTTPS" @doc """ Securely download https content from a URL. This function uses the built-in `:httpc` client but enables certificate verification which is not enabled by `:httc` by default. See also https://erlef.github.io/security-wg/secure_coding_and_deployment_hardening/ssl ### Arguments * `url` is a binary URL or a `{url, list_of_headers}` tuple. If provided the headers are a list of `{'header_name', 'header_value'}` tuples. Note that the name and value are both charlists, not strings. * `options` is a keyword list of options ### Options * `:verify_peer` is a boolean value indicating if peer verification should be done for this request. The default is `true`. ### Returns * `{:ok, body}` if the return is successful * `{:not_modified, headers}` if the request would result in returning the same results as one matching an etag * `{:error, error}` if the download is unsuccessful. An error will also be logged in these cases. ### Unsafe HTTPS If the environment variable `CLDR_UNSAFE_HTTPS` is set to anything other than `FALSE`, `false`, `nil` or `NIL` then no peer verification of certificates is performed. Setting this variable is not recommended but may be required is where peer verification for unidentified reasons. Please [open an issue](https://github.com/elixir-cldr/cldr/issues) if this occurs. ### Certificate stores In order to keep dependencies to a minimum, `get/1` attempts to locate an already installed certificate store. It will try to locate a store in the following order which is intended to satisfy most host systems. The certificate store is expected to be a path name on the host system. ```elixir # A certificate store configured by the # developer Application.get_env(:ex_cldr, :cacertfile) # Populated if hex package `CAStore` is configured CAStore.file_path() # Populated if hex package `certfi` is configured :certifi.cacertfile() # Debian/Ubuntu/Gentoo etc. "/etc/ssl/certs/ca-certificates.crt", # Fedora/RHEL 6 "/etc/pki/tls/certs/ca-bundle.crt", # OpenSUSE "/etc/ssl/ca-bundle.pem", # OpenELEC "/etc/pki/tls/cacert.pem", # CentOS/RHEL 7 "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", # Open SSL on MacOS "/usr/local/etc/openssl/cert.pem", # MacOS & Alpine Linux "/etc/ssl/cert.pem" ``` """ @spec get(String.t | {String.t, list()}, options :: Keyword.t) :: {:ok, binary} | {:not_modified, any()} | {:error, any} def get(url, options \\ []) def get(url, options) when is_binary(url) and is_list(options) do case get_with_headers(url, options) do {:ok, _headers, body} -> {:ok, body} other -> other end end def get({url, headers}, options) when is_binary(url) and is_list(headers) and is_list(options) do case get_with_headers({url, headers}, options) do {:ok, _headers, body} -> {:ok, body} other -> other end end @doc """ Securely download https content from a URL. This function uses the built-in `:httpc` client but enables certificate verification which is not enabled by `:httc` by default. See also https://erlef.github.io/security-wg/secure_coding_and_deployment_hardening/ssl ### Arguments * `url` is a binary URL or a `{url, list_of_headers}` tuple. If provided the headers are a list of `{'header_name', 'header_value'}` tuples. Note that the name and value are both charlists, not strings. * `options` is a keyword list of options ### Options * `:verify_peer` is a boolean value indicating if peer verification should be done for this request. The default is `true`. ### Returns * `{:ok, body, headers}` if the return is successful * `{:not_modified, headers}` if the request would result in returning the same results as one matching an etag * `{:error, error}` if the download is unsuccessful. An error will also be logged in these cases. ### Unsafe HTTPS If the environment variable `CLDR_UNSAFE_HTTPS` is set to anything other than `FALSE`, `false`, `nil` or `NIL` then no peer verification of certificates is performed. Setting this variable is not recommended but may be required is where peer verification for unidentified reasons. Please [open an issue](https://github.com/elixir-cldr/cldr/issues) if this occurs. ### Certificate stores In order to keep dependencies to a minimum, `get/1` attempts to locate an already installed certificate store. It will try to locate a store in the following order which is intended to satisfy most host systems. The certificate store is expected to be a path name on the host system. ```elixir # A certificate store configured by the # developer Application.get_env(:ex_cldr, :cacertfile) # Populated if hex package `CAStore` is configured CAStore.file_path() # Populated if hex package `certfi` is configured :certifi.cacertfile() # Debian/Ubuntu/Gentoo etc. "/etc/ssl/certs/ca-certificates.crt", # Fedora/RHEL 6 "/etc/pki/tls/certs/ca-bundle.crt", # OpenSUSE "/etc/ssl/ca-bundle.pem", # OpenELEC "/etc/pki/tls/cacert.pem", # CentOS/RHEL 7 "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", # Open SSL on MacOS "/usr/local/etc/openssl/cert.pem", # MacOS & Alpine Linux "/etc/ssl/cert.pem" ``` """ @doc since: "2.21.0" @spec get_with_headers(String.t | {String.t, list()}, options :: Keyword.t) :: {:ok, list(), binary} | {:not_modified, any()} | {:error, any} def get_with_headers(request, options \\ []) def get_with_headers(url, options) when is_binary(url) do get_with_headers({url, []}, options) end def get_with_headers({url, headers}, options) when is_binary(url) and is_list(headers) and is_list(options) do require Logger hostname = String.to_charlist(URI.parse(url).host) url = String.to_charlist(url) verify_peer? = Keyword.get(options, :verify_peer, true) case :httpc.request(:get, {url, headers}, https_opts(hostname, verify_peer?), []) do {:ok, {{_version, 200, _}, headers, body}} -> {:ok, headers, body} {:ok, {{_version, 304, _}, headers, _body}} -> {:not_modified, headers} {_, {{_version, code, message}, _headers, _body}} -> Logger.bare_log( :error, "Failed to download #{url}. " <> "HTTP Error: (#{code}) #{inspect(message)}" ) {:error, code} {:error, {:failed_connect, [{_, {host, _port}}, {_, _, sys_message}]}} -> Logger.bare_log( :error, "Failed to connect to #{inspect(host)} to download #{inspect url}" ) {:error, sys_message} {:error, {other}} -> Logger.bare_log( :error, "Failed to download #{inspect url}. Error #{inspect other}" ) {:error, other} end end @certificate_locations [ # Configured cacertfile Application.compile_env(:ex_cldr, :cacertfile), # Populated if hex package CAStore is configured if(Code.ensure_loaded?(CAStore), do: CAStore.file_path()), # Populated if hex package certfi is configured if(Code.ensure_loaded?(:certifi), do: :certifi.cacertfile() |> List.to_string), # Debian/Ubuntu/Gentoo etc. "/etc/ssl/certs/ca-certificates.crt", # Fedora/RHEL 6 "/etc/pki/tls/certs/ca-bundle.crt", # OpenSUSE "/etc/ssl/ca-bundle.pem", # OpenELEC "/etc/pki/tls/cacert.pem", # CentOS/RHEL 7 "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", # Open SSL on MacOS "/usr/local/etc/openssl/cert.pem", # MacOS & Alpine Linux "/etc/ssl/cert.pem" ] |> Enum.reject(&is_nil/1) @doc false def certificate_store do @certificate_locations |> Enum.find(&File.exists?/1) |> raise_if_no_cacertfile! |> :erlang.binary_to_list end defp raise_if_no_cacertfile!(nil) do raise RuntimeError, """ No certificate trust store was found. Tried looking for: #{inspect @certificate_locations} A certificate trust store is required in order to download locales for your configuration. Since ex_cldr could not detect a system installed certificate trust store one of the following actions may be taken: 1. Install the hex package `castore`. It will be automatically detected after recompilation. 2. Install the hex package `certifi`. It will be automatically detected after recomilation. 3. Specify the location of a certificate trust store by configuring it in `config.exs`: config :ex_cldr, cacertfile: "/path/to/cacertfile", ... """ end defp raise_if_no_cacertfile!(file) do file end defp https_opts(hostname, verify_peer?) do if secure_ssl?() and verify_peer? do [ssl: [ verify: :verify_peer, cacertfile: certificate_store(), depth: 4, ciphers: preferred_ciphers(), versions: protocol_versions(), eccs: preferred_eccs(), reuse_sessions: true, server_name_indication: hostname, secure_renegotiate: true, customize_hostname_check: [ match_fun: :public_key.pkix_verify_hostname_match_fun(:https) ] ] ] else [ssl: [ verify: :verify_none, server_name_indication: hostname, secure_renegotiate: true, reuse_sessions: true, versions: protocol_versions(), ciphers: preferred_ciphers(), versions: protocol_versions(), ] ] end end def preferred_ciphers do preferred_ciphers = [ # Cipher suites (TLS 1.3): TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 %{cipher: :aes_128_gcm, key_exchange: :any, mac: :aead, prf: :sha256}, %{cipher: :aes_256_gcm, key_exchange: :any, mac: :aead, prf: :sha384}, %{cipher: :chacha20_poly1305, key_exchange: :any, mac: :aead, prf: :sha256}, # Cipher suites (TLS 1.2): ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256: # ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305: # ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 %{cipher: :aes_128_gcm, key_exchange: :ecdhe_ecdsa, mac: :aead, prf: :sha256}, %{cipher: :aes_128_gcm, key_exchange: :ecdhe_rsa, mac: :aead, prf: :sha256}, %{cipher: :aes_256_gcm, key_exchange: :ecdh_ecdsa, mac: :aead, prf: :sha384}, %{cipher: :aes_256_gcm, key_exchange: :ecdh_rsa, mac: :aead, prf: :sha384}, %{cipher: :chacha20_poly1305, key_exchange: :ecdhe_ecdsa, mac: :aead, prf: :sha256}, %{cipher: :chacha20_poly1305, key_exchange: :ecdhe_rsa, mac: :aead, prf: :sha256}, %{cipher: :aes_128_gcm, key_exchange: :dhe_rsa, mac: :aead, prf: :sha256}, %{cipher: :aes_256_gcm, key_exchange: :dhe_rsa, mac: :aead, prf: :sha384} ] :ssl.filter_cipher_suites(preferred_ciphers, []) end def protocol_versions do if otp_version() < 25 do [:"tlsv1.2"] else [:"tlsv1.2", :"tlsv1.3"] end end def preferred_eccs do # TLS curves: X25519, prime256v1, secp384r1 preferred_eccs = [:secp256r1, :secp384r1] :ssl.eccs() -- (:ssl.eccs() -- preferred_eccs) end def secure_ssl? do case System.get_env(@cldr_unsafe_https) do nil -> true "FALSE" -> false "false" -> false "nil" -> false "NIL" -> false _other -> true end end def otp_version do :erlang.system_info(:otp_release) |> List.to_integer end end