defmodule Unicode.String do @moduledoc """ This module provides functions that implement some of the [Unicode](https://unicode.org) standards: * The [Unicode Case Folding](https://www.unicode.org/versions/Unicode13.0.0/ch03.pdf) algorithm to provide case-independent equality checking irrespective of language or script. * The [Unicode Segmentation](https://unicode.org/reports/tr29/) algorithm to detect, break or splut strings into grapheme clusters, works and sentences. * The [Unicode Line Breaking](https://www.unicode.org/reports/tr14/) algorithm to determine line breaks (as in word-wrapping). """ alias Unicode.Property alias Unicode.String.Break alias Unicode.String.Segment defdelegate fold(string), to: Unicode.String.Case.Folding defdelegate fold(string, type), to: Unicode.String.Case.Folding @type string_interval :: {String.t(), String.t()} @type break_type :: :grapheme | :word | :line | :sentence @type error_return :: {:error, String.t()} @type options :: [ {:locale, String.t()}, {:break, break_type}, {:suppressions, boolean} ] @type split_options :: [ {:locale, String.t()}, {:break, break_type}, {:suppressions, boolean}, {:trim, boolean} ] @type break_or_no_break :: :break | :no_break @type break_match :: {break_or_no_break, {String.t(), {String.t(), String.t()}}} | {break_or_no_break, {String.t(), String.t()}} @default_locale "root" @default_break :word @doc """ Compares two strings in a case insensitive manner. Case folding is applied to the two string arguments which are then compared with the `==` operator. ## Arguments * `string_a` and `string_b` are two strings to be compared * `type` is the case folding type to be applied. The alternatives are `:full`, `:simple` and `:turkic`. The default is `:full`. ## Returns * `true` or `false` ## Notes * This function applies the [Unicode Case Folding algorithm](https://www.unicode.org/versions/Unicode13.0.0/ch03.pdf) * The algorithm does not apply any treatment to diacritical marks hence "compare strings without accents" is not part of this function. ## Examples iex> Unicode.String.equals_ignoring_case? "ABC", "abc" true iex> Unicode.String.equals_ignoring_case? "beißen", "beissen" true iex> Unicode.String.equals_ignoring_case? "grüßen", "grussen" false """ @spec equals_ignoring_case?(String.t(), String.t(), atom()) :: boolean def equals_ignoring_case?(string_a, string_b, type \\ :full) do fold(string_a, type) == fold(string_b, type) end @doc """ Returns a boolean indicating if the requested break is applicable at the point between the two string segments represented by `{string_before, string_after}`. ## Arguments * `string` is any `t:String.t/0`. * `options` is a keyword list of options. ## Returns * `true` or `false` or * raises an exception if there is an error ## Options * `:locale` is any locale returned by `Unicode.String.Segment.known_locales/0`. The default is #{inspect(@default_locale)} which corresponds to the break rules defined by the [Unicode Segmentation](https://unicode.org/reports/tr29/) rules. * `:break` is the type of break. It is one of `:grapheme`, `:word`, `:line` or `:sentence`. The default is `#{inspect(@default_break)}`. * `:suppressions` is a boolean which, if `true`, will suppress breaks for common abbreviations defined for the `locale`. The default is `true`. ## Examples iex> Unicode.String.break? {"This is ", "some words"} true iex> Unicode.String.break? {"This is ", "some words"}, break: :sentence false iex> Unicode.String.break? {"This is one. ", "This is some words."}, break: :sentence true """ @spec break?(string_interval, options) :: boolean def break?({string_before, string_after}, options \\ []) do case break({string_before, string_after}, options) do {:break, _} -> true {:no_break, _} -> false {:error, reason} -> raise ArgumentError, reason end end @doc """ Returns match data indicating if the requested break is applicable at the point between the two string segments represented by `{string_before, string_after}`. ## Arguments * `string` is any `t:String.t/0`. * `options` is a keyword list of options. ## Returns A tuple indicating if a break would be applicable at this point between `string_before` and `string_after`. * `{:break, {string_before, {matched_string, remaining_string}}}` or * `{:no_break, {string_before, {matched_string, remaining_string}}}` or * `{:error, reason}` ## Options * `:locale` is any locale returned by `Unicode.String.Segment.known_locales/0`. The default is #{inspect(@default_locale)} which corresponds to the break rules defined by the [Unicode Segmentation](https://unicode.org/reports/tr29/) rules. * `:break` is the type of break. It is one of `:grapheme`, `:word`, `:line` or `:sentence`. The default is `#{inspect(@default_break)}`. * `:suppressions` is a boolean which, if `true`, will suppress breaks for common abbreviations defined for the `locale`. The default is `true`. ## Examples iex> Unicode.String.break {"This is ", "some words"} {:break, {"This is ", {"s", "ome words"}}} iex> Unicode.String.break {"This is ", "some words"}, break: :sentence {:no_break, {"This is ", {"s", "ome words"}}} iex> Unicode.String.break {"This is one. ", "This is some words."}, break: :sentence {:break, {"This is one. ", {"T", "his is some words."}}} """ @spec break(string_interval, options) :: break_match | error_return def break({string_before, string_after}, options \\ []) do locale = Keyword.get(options, :locale, @default_locale) break = Keyword.get(options, :break, @default_break) with {:ok, break} <- validate(:break, break), {:ok, locale} <- validate(:locale, locale) do Break.break({string_before, string_after}, locale, break, options) end end @doc """ Returns an enumerable that splits a string on demand. ## Arguments * `string` is any `t:String.t/0`. * `options` is a keyword list of options. ## Returns * A function that implements the enumerable protocol or * `{:error, reason}` ## Options * `:locale` is any locale returned by `Unicode.String.Segment.known_locales/0`. The default is #{inspect(@default_locale)} which corresponds to the break rules defined by the [Unicode Segmentation](https://unicode.org/reports/tr29/) rules. * `:break` is the type of break. It is one of `:grapheme`, `:word`, `:line` or `:sentence`. The default is `#{inspect(@default_break)}`. * `:suppressions` is a boolean which, if `true`, will suppress breaks for common abbreviations defined for the `locale`. The default is `true`. * `:trim` is a boolean indicating if segments the are comprised of only white space are to be excluded from the returned list. The default is `false`. ## Examples iex> enum = Unicode.String.splitter "This is a sentence. And another.", break: :word, trim: true iex> Enum.take enum, 3 ["This", "is", "a"] """ @spec splitter(String.t(), split_options) :: function | error_return def splitter(string, options) when is_binary(string) do locale = Keyword.get(options, :locale, @default_locale) break = Keyword.get(options, :break, @default_break) with {:ok, break} <- validate(:break, break), {:ok, locale} <- validate(:locale, locale) do Stream.unfold(string, &Break.next(&1, locale, break, options)) end end @doc """ Returns next segment in a string. ## Arguments * `string` is any `t:String.t/0`. * `options` is a keyword list of options. ## Returns A tuple with the segment and the remainder of the string or `""` in case the String reached its end. * `{next_string, rest_of_the_string}` or * `{:error, reason}` ## Options * `:locale` is any locale returned by `Unicode.String.Segment.known_locales/0`. The default is #{inspect(@default_locale)} which corresponds to the break rules defined by the [Unicode Segmentation](https://unicode.org/reports/tr29/) rules. * `:break` is the type of break. It is one of `:grapheme`, `:word`, `:line` or `:sentence`. The default is `#{inspect(@default_break)}`. * `:suppressions` is a boolean which, if `true`, will suppress breaks for common abbreviations defined for the `locale`. The default is `true`. ## Examples iex> Unicode.String.next "This is a sentence. And another.", break: :word {"This", " is a sentence. And another."} iex> Unicode.String.next "This is a sentence. And another.", break: :sentence {"This is a sentence. ", "And another."} """ @spec next(String.t(), split_options) :: String.t() | nil | error_return def next(string, options \\ []) when is_binary(string) do locale = Keyword.get(options, :locale, @default_locale) break = Keyword.get(options, :break, @default_break) with {:ok, break} <- validate(:break, break), {:ok, locale} <- validate(:locale, locale) do Break.next(string, locale, break, options) end end @doc """ Splits a string according to the specified break type. ## Arguments * `string` is any `t:String.t/0`. * `options` is a keyword list of options. ## Returns * A list of strings after applying the specified break rules or * `{:error, reason}` ## Options * `:locale` is any locale returned by `Unicode.String.Segment.known_locales/0`. The default is #{inspect(@default_locale)} which corresponds to the break rules defined by the [Unicode Segmentation](https://unicode.org/reports/tr29/) rules. * `:break` is the type of break. It is one of `:grapheme`, `:word`, `:line` or `:sentence`. The default is `#{inspect(@default_break)}`. * `:suppressions` is a boolean which, if `true`, will suppress breaks for common abbreviations defined for the `locale`. The default is `true`. * `:trim` is a boolean indicating if segments the are comprised of only white space are to be excluded from the returned list. The default is `false`. ## Examples iex> Unicode.String.split "This is a sentence. And another.", break: :word ["This", " ", "is", " ", "a", " ", "sentence", ".", " ", "And", " ", "another", "."] iex> Unicode.String.split "This is a sentence. And another.", break: :word, trim: true ["This", "is", "a", "sentence", ".", "And", "another", "."] iex> Unicode.String.split "This is a sentence. And another.", break: :sentence ["This is a sentence. ", "And another."] """ @spec split(String.t(), split_options) :: [String.t(), ...] | error_return def split(string, options \\ []) when is_binary(string) do locale = Keyword.get(options, :locale, @default_locale) break = Keyword.get(options, :break, @default_break) with {:ok, break} <- validate(:break, break), {:ok, locale} <- validate(:locale, locale) do Break.split(string, locale, break, options) end |> maybe_trim(options[:trim]) end defp maybe_trim(list, true) when is_list(list) do Enum.reject(list, &Property.white_space?/1) end defp maybe_trim(list, _) do list end @doc """ Return a stream that breaks a string into graphemes, words, sentences or line breaks. ## Arguments * `string` is any `t:String.t/0`. * `options` is a keyword list of options. ## Returns * A stream that is an `t:Enumerable.t/0` that can be used with the functions in the `Stream` or `Enum` modules. * `{:error, reason}` ## Options * `:locale` is any locale returned by `Unicode.String.Segment.known_locales/0`. The default is #{inspect(@default_locale)} which corresponds to the break rules defined by the [Unicode Segmentation](https://unicode.org/reports/tr29/) rules. * `:break` is the type of break. It is one of `:grapheme`, `:word`, `:line` or `:sentence`. The default is `#{inspect(@default_break)}`. * `:suppressions` is a boolean which, if `true`, will suppress breaks for common abbreviations defined for the `locale`. The default is `true`. * `:trim` is a boolean indicating if segments the are comprised of only white space are to be excluded from the returned list. The default is `false`. ## Examples iex> Enum.to_list Unicode.String.stream("this is a set of words", trim: true) ["this", "is", "a", "set", "of", "words"] iex> Enum.to_list Unicode.String.stream("this is a set of words", break: :sentence, trim: true) ["this is a set of words"] """ @doc since: "1.2.0" @spec stream(String.t(), Keyword.t()) :: Enumerable.t() | {:error, String.t()} def stream(string, options \\ []) do locale = Keyword.get(options, :locale, @default_locale) break = Keyword.get(options, :break, @default_break) with {:ok, break} <- validate(:break, break), {:ok, locale} <- validate(:locale, locale) do Stream.resource( fn -> string end, fn string -> case Break.next(string, locale, break, options) do nil -> {:halt, ""} {break, rest} -> {[break], rest} end end, fn _ -> :ok end ) end end # # Helpers # defp validate(:locale, locale) when is_binary(locale) do if locale in Segment.known_locales() do {:ok, locale} else {:error, Segment.unknown_locale_error(locale)} end end defp validate(:locale, locale) when is_atom(locale) do locale = Atom.to_string(locale) validate(:locale, locale) end @breaks [:word, :grapheme, :line, :sentence] defp validate(:break, break) do if break in @breaks do {:ok, break} else {:error, "Unknown break #{inspect(break)}. Valid breaks are #{inspect(@breaks)}"} end end end