defmodule PhoenixApiToolkit.Ecto.Validators do @moduledoc """ Generic validators and helper functions for validating Ecto changesets. ## Examples The examples in this module use the following basic schema and changeset: @schema %{ first_name: :string, last_name: :string, last_name_prefix: :string, order_by: :string, file: :string, mime_type: :string } def changeset(changes \\\\ %{}) do {%{}, @schema} |> cast(changes, [:first_name, :last_name, :order_by, :file]) end """ alias Ecto.Changeset import Ecto.Changeset require Logger @order_by_format ~r/^(asc|desc|asc_nulls_last|desc_nulls_last|asc_nulls_first|desc_nulls_first):(\w{1,20})$/ @doc """ If the changeset does not contain a change for `field` - even if the field already has a value in the changeset data - set it to `change`. Useful for setting default changes. ## Examples For the implementation of `changeset/1`, see `#{__MODULE__}`. iex> changeset() |> put_change_if_unchanged(:first_name, "Peter") #Ecto.Changeset iex> changeset(%{first_name: "Jason"}) |> put_change_if_unchanged(:first_name, "Peter") #Ecto.Changeset """ @spec put_change_if_unchanged(Changeset.t(), atom(), any()) :: Changeset.t() @deprecated "renamed to default_change/3" def put_change_if_unchanged(changeset, field, value) def put_change_if_unchanged(cs, fld, val), do: default_change(cs, fld, val) @doc """ If the changeset does not contain a change for `field` - even if the field already has a value in the changeset data - set it to `change`. Useful for setting default changes. ## Examples For the implementation of `changeset/1`, see `#{__MODULE__}`. iex> changeset() |> default_change(:first_name, "Peter") #Ecto.Changeset iex> changeset(%{first_name: "Jason"}) |> default_change(:first_name, "Peter") #Ecto.Changeset """ @spec default_change(Ecto.Changeset.t(), atom, any) :: Ecto.Changeset.t() def default_change(changeset, field, value) def default_change(cs, fld, val), do: put_change(cs, fld, get_change(cs, fld, val)) @doc """ Validates that `field` is a suitable parameter for an (i)like query. User input for (i)like queries should not contain metacharacters because this creates a denial-of-service attack vector: introducing a lot of metacharacters rapidly increases the performance costs of such queries. The metacharacters for (i)like queries are '_', '%' and the escape character of the database, which defaults to '\\'. ## Examples For the implementation of `changeset/1`, see `#{__MODULE__}`. iex> changeset(%{first_name: "Peter", last_name: "Pan"}) |> validate_ilike_safe([:first_name, :last_name]) #Ecto.Changeset iex> changeset(%{first_name: "Peter%"}) |> validate_ilike_safe(:first_name) #Ecto.Changeset iex> changeset(%{first_name: "Pet_er"}) |> validate_ilike_safe(:first_name) #Ecto.Changeset iex> changeset(%{first_name: "Pet\\\\er"}) |> validate_ilike_safe(:first_name) #Ecto.Changeset """ @spec validate_ilike_safe(Changeset.t(), atom | [atom]) :: Changeset.t() def validate_ilike_safe(changeset, fields) when is_list(fields), do: Enum.reduce(fields, changeset, &validate_ilike_safe(&2, &1)) def validate_ilike_safe(changeset, field), do: validate_format(changeset, field, ~r/^[^\%_\\]*$/, message: "may not contain _ % or \\") @doc """ Validates that `field` (or multiple fields) contains plaintext. ## Examples For the implementation of `changeset/1`, see `#{__MODULE__}`. iex> changeset(%{first_name: "Peter", last_name: "Pan"}) |> validate_plaintext([:first_name, :last_name]) #Ecto.Changeset iex> changeset(%{first_name: "Peter{}"}) |> validate_plaintext(:first_name) #Ecto.Changeset """ @spec validate_plaintext(Changeset.t(), atom() | [atom()]) :: Changeset.t() def validate_plaintext(changeset, fields) when is_list(fields), do: Enum.reduce(fields, changeset, &validate_plaintext(&2, &1)) def validate_plaintext(changeset, field) do changeset |> validate_format(field, ~r/^[\w\s\.\,\-\!\?]*$/, message: "can only contain a-Z 0-9 _ . , - ! ? and whitespace" ) end @doc """ Validate the value of an `order_by` query parameter. The format of the parameter is expected to match `#{@order_by_format |> inspect()}` (may be repeated, comma-separated). The supported fields should be passed as a list or `MapSet` (which performs better) to `orderables`. If the change is valid, the original change is replaced with a keyword list of `{:field, :direction}`, which is supported by `PhoenixApiToolkit.Ecto.DynamicFilters.standard_filters/4`. ## Examples For the implementation of `changeset/1`, see `#{__MODULE__}`. @orderables ~w(first_name last_name) |> MapSet.new() iex> changeset(%{order_by: "asc:last_name"}) |> validate_order_by(@orderables) #Ecto.Changeset iex> changeset(%{order_by: "invalid"}) |> validate_order_by(@orderables) #Ecto.Changeset iex> changeset(%{order_by: "asc:eye_count"}) |> validate_order_by(@orderables) #Ecto.Changeset iex> changeset(%{order_by: nil}) |> validate_order_by(@orderables) #Ecto.Changeset iex> changeset(%{order_by: "asc:last_name,desc_nulls_last:first_name"}) |> validate_order_by(@orderables) #Ecto.Changeset """ @spec validate_order_by(Changeset.t(), Enum.t()) :: Changeset.t() def validate_order_by(changeset, orderable_fields) do with order_by when is_binary(order_by) <- get_change(changeset, :order_by) do order_by |> String.split(",", trim: true) |> Enum.reduce(put_change(changeset, :order_by, []), fn string, cs -> with [dir, field] <- Regex.run(@order_by_format, string, capture: :all_but_first), {true, field} <- {field in orderable_fields, field} do update_change(cs, :order_by, &[{String.to_atom(dir), String.to_atom(field)} | &1]) else nil -> add_error(cs, :order_by, "format is asc|desc:field") {false, field} -> add_error(cs, :order_by, "unknown field " <> field) end end) |> update_change(:order_by, &Enum.reverse/1) else _ -> delete_change(changeset, :order_by) end end @doc """ Move a change to another field in the changeset (if its value is not nil). Like `Ecto.Changeset.put_change/3`, the change is moved without additional validation. Optionally, the value can be mapped using `value_mapper`, which defaults to the identity function. ## Examples For the implementation of `changeset/1`, see `#{__MODULE__}`. # there is no effect when there is no change to the field iex> changeset() |> move_change(:first_name, :last_name) #Ecto.Changeset # a change is moved to another field name as-is by default iex> changeset(%{first_name: "Pan"}) |> move_change(:first_name, :last_name) #Ecto.Changeset # an optional value_mapper can be passed to do some processing on the change along the way iex> changeset(%{first_name: "Pan"}) |> move_change(:first_name, :last_name, & String.upcase(&1)) #Ecto.Changeset """ @spec move_change(Changeset.t(), atom(), atom(), (any -> any)) :: Changeset.t() def move_change(%{changes: changes} = changeset, field, new_field, value_mapper \\ & &1) do case Map.get(changes, field) do nil -> changeset value -> new_changes = changes |> Map.delete(field) |> Map.put(new_field, value_mapper.(value)) Map.put(changeset, :changes, new_changes) end end @doc """ Validate a searchable field. If the value of `field` is postfixed with '\\*', a fuzzy search instead of a equal_to match is considered to be intended. In this case, the value must be at least 4 characters long and must be (i)like safe (as per `validate_ilike_safe/2`), and is moved to `search_field`. The postfix '\\*' is stripped from the search string. The purpose is to pass the changes along to a `list`-query which supports searching by `search_field`, and equal_to filtering by `field`. See `PhoenixApiToolkit.Ecto.DynamicFilters` for more info on dynamic filtering. The minimum length for a search string can be overridden with option `:min_length`. ## Examples For the implementation of `changeset/1`, see `#{__MODULE__}`. # a last_name value postfixed with '*' is search query iex> changeset(%{last_name: "Smit*"}) |> validate_searchable(:last_name, :last_name_prefix) #Ecto.Changeset # values without postfix '*' are passed through iex> changeset(%{last_name: "Smit"}) |> validate_searchable(:last_name, :last_name_prefix) #Ecto.Changeset # to prevent too-broad, expensive ilike queries, search parameters must be >=4 characters long iex> changeset(%{last_name: "Smi*"}) |> validate_searchable(:last_name, :last_name_prefix) #Ecto.Changeset # the min_length value can be overridden iex> changeset(%{last_name: "Smi*"}) |> validate_searchable(:last_name, :last_name_prefix, min_length: 5) #Ecto.Changeset # additionally, search parameters must be ilike safe, as per validate_ilike_safe/2 iex> changeset(%{last_name: "Sm_it*"}) |> validate_searchable(:last_name, :last_name_prefix) #Ecto.Changeset """ @spec validate_searchable(Changeset.t(), atom(), atom(), keyword) :: Changeset.t() def validate_searchable(changeset, field, search_field, opts \\ []) do with value when not is_nil(value) <- get_change(changeset, field), true <- String.ends_with?(value, "*") do changeset |> put_change(field, String.trim_trailing(value, "*")) # to prevent useless, expensive lookups |> validate_length(field, min: opts[:min_length] || 4) |> validate_ilike_safe(field) |> map_if_valid(&move_change(&1, field, search_field)) else _ -> changeset end end @doc """ For verifying files uploaded as base64-encoded binaries. Attempts to decode `field` and validate its file signature. The file signature, also known as a file's "magic bytes", can be looked up on the internet (for example [here](https://en.wikipedia.org/wiki/List_of_file_signatures)) and may be a list of allowed magic byte types. ## Examples For the implementation of `changeset/1`, see `#{__MODULE__}`. @pdf_signature "255044462D" |> Base.decode16!() @png_signature "89504E470D0A1A0A" |> Base.decode16!() @png_file "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" @gif_file "R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==" @txt_file "some text" |> Base.encode64() # if the signature checks out, the uploaded file is decoded and the changeset valid iex> cs = changeset(%{file: @png_file}) |> validate_upload(:file, @png_signature) iex> {cs.valid?, cs.changes.file} {true, @png_file |> Base.decode64!()} # multiple signatures can be provided iex> cs = changeset(%{file: @png_file}) |> validate_upload(:file, [@pdf_signature, @png_signature]) iex> cs.valid? true # if the signature does not check out, an error is added to the changeset and the decoded file is discarded iex> cs = changeset(%{file: @gif_file}) |> validate_upload(:file, [@pdf_signature, @png_signature]) iex> {cs.valid?, cs.errors, cs.changes.file} {false, [file: {"invalid file type", []}], @gif_file} # decoding errors are handled gracefully iex> cs = changeset(%{file: "a"}) |> validate_upload(:file, @pdf_signature) iex> {cs.valid?, cs.errors} {false, [file: {"invalid base64 encoding", []}]} # it is possible to provide a custom validator function (which should return `true` if valid) iex> cs = changeset(%{file: @txt_file}) |> validate_upload(:file, &String.starts_with?(&1, "some")) iex> cs.valid? true """ @spec validate_upload(Changeset.t(), atom, binary | [binary]) :: Changeset.t() def validate_upload(changeset, field, file_signature) when not is_list(file_signature), do: validate_upload(changeset, field, [file_signature]) def validate_upload(changeset, field, file_signatures) do with change when not is_nil(change) <- get_change(changeset, field), {:base64, {:ok, binary}} <- {:base64, Base.decode64(change)}, {:valid, true} <- {:valid, Enum.find_value(file_signatures, &file_signature_matches?(binary, &1))} do put_change(changeset, field, binary) else nil -> changeset {:base64, :error} -> add_error(changeset, field, "invalid base64 encoding") {:valid, _} -> add_error(changeset, field, "invalid file type") _too_short_binary -> add_error(changeset, field, "invalid file") end end @doc """ Like `validate_upload/3`, but sets the mime type to `mime_field`. Parameter `file_sig_mime_map` should be a map of file type signatures to mime types. ## Examples For the implementation of `changeset/1`, see `#{__MODULE__}`. def is_valid_text(binary), do: String.starts_with?(binary, "some") @signatures %{ ("89504E470D0A1A0A" |> Base.decode16!()) => "image/png", &__MODULE__.is_valid_text/1 => "text/plain" } @png_file "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" @gif_file "R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==" @txt_file "some text" |> Base.encode64() iex> cs = changeset(%{file: @png_file}) |> validate_upload(:file, :mime_type, @signatures) iex> {cs.valid?, cs.changes.file, cs.changes.mime_type} {true, @png_file |> Base.decode64!(), "image/png"} iex> cs = changeset(%{file: @txt_file}) |> validate_upload(:file, :mime_type, @signatures) iex> {cs.valid?, cs.changes.file, cs.changes.mime_type} {true, @txt_file |> Base.decode64!(), "text/plain"} iex> cs = changeset(%{file: @gif_file}) |> validate_upload(:file, :mime_type, @signatures) iex> {cs.valid?, cs.errors, cs.changes.file, cs.changes[:mime_type]} {false, [file: {"invalid file type", []}], @gif_file, nil} """ @spec validate_upload(Ecto.Changeset.t(), atom, atom, map) :: Ecto.Changeset.t() def validate_upload(changeset, field, mime_field, file_sig_mime_map) do with base64_data when not is_nil(base64_data) <- get_change(changeset, field), {:base64, {:ok, decoded}} <- {:base64, Base.decode64(base64_data)}, %{valid?: true} = changeset <- validate_file_signature(changeset, field, mime_field, file_sig_mime_map, decoded) do put_change(changeset, field, decoded) else nil -> changeset {:base64, _} -> add_error(changeset, field, "invalid base64 encoding") %Ecto.Changeset{} = changeset -> changeset end end @doc """ If `changeset` is valid, apply the first function `then_do` to it, else apply the second function `else_do` to it, which defaults to the identity function. ## Examples # function then_do is applied to the changeset if it is valid iex> %Ecto.Changeset{valid?: true} |> map_if_valid(& &1.changes) %{} # if the changeset is invalid and else_do is provided, apply it to the changeset iex> %Ecto.Changeset{valid?: false} |> map_if_valid(& &1.changes, & &1.errors) [] # else_do defaults to identity, returning the changeset iex> %Ecto.Changeset{valid?: false} |> map_if_valid(& &1.changes) #Ecto.Changeset """ @spec map_if_valid(Changeset.t(), (Changeset.t() -> any), (Changeset.t() -> any)) :: Changeset.t() def map_if_valid(changeset, then_do, else_do \\ & &1) def map_if_valid(%{valid?: true} = cs, then_do, _else_do), do: then_do.(cs) def map_if_valid(cs, _then_do, else_do), do: else_do.(cs) @doc """ Apply `function` to multiple fields of the changeset. Convenience wrapper for validators that don't support multiple fields. ## Examples / doctests For the implementation of `changeset/1`, see `#{__MODULE__}`. iex> changeset(%{first_name: "Luke", last_name: "Skywalker"}) ...> |> multifield_apply([:first_name, :last_name], &validate_length(&1, &2, max: 3)) ...> |> Map.take([:valid?, :errors]) %{valid?: false, errors: [ {:last_name, {"should be at most %{count} character(s)", [count: 3, validation: :length, kind: :max, type: :string]}}, {:first_name, {"should be at most %{count} character(s)", [count: 3, validation: :length, kind: :max, type: :string]}} ]} """ @spec multifield_apply(Changeset.t(), [atom], (Changeset.t(), atom -> Changeset.t())) :: Changeset.t() def multifield_apply(changeset, fields, function) do Enum.reduce(fields, changeset, &function.(&2, &1)) end ########### # Private # ########### defp file_signature_matches?(binary, file_signature) when is_function(file_signature), do: file_signature.(binary) defp file_signature_matches?(binary, file_signature) do byte_count = byte_size(file_signature) with <> <- binary do head == file_signature else other -> other end end # # validate that the uploaded file starts with known magic bytes # field is used for errors # sets :image_mime to mime type if recognized # defp validate_file_signature(changeset, field, mime_field, allowed_mimes, data) do Enum.find_value( allowed_mimes, add_error(changeset, field, "invalid file type"), fn {signature, mime_type} -> case file_signature_matches?(data, signature) do true -> put_change(changeset, mime_field, mime_type) _ -> false end end ) end end