%% Copyright (c) 2017-2022 Guilherme Andrade %% %% Permission is hereby granted, free of charge, to any person obtaining a %% copy of this software and associated documentation files (the "Software"), %% to deal in the Software without restriction, including without limitation %% the rights to use, copy, modify, merge, publish, distribute, sublicense, %% and/or sell copies of the Software, and to permit persons to whom the %% Software is furnished to do so, subject to the following conditions: %% %% The above copyright notice and this permission notice shall be included in %% all copies or substantial portions of the Software. %% %% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR %% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, %% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE %% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER %% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING %% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER %% DEALINGS IN THE SOFTWARE. %% %% locus is an independent project and has not been authorized, sponsored, %% or otherwise approved by MaxMind. %% @private -module(locus_util). -include_lib("kernel/include/file.hrl"). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). -endif. %% ------------------------------------------------------------------ %% API Function Exports %% ------------------------------------------------------------------ -export( [parse_ip_address/1, lists_anymap/2, lists_take/2, bin_to_hex_str/1, expect_linked_process_termination/1, url_query_encode/1, filesystem_safe_name/1, is_utf8_binary/1, is_unicode_string/1, is_date/1, purge_term_of_very_large_binaries/1, resolve_http_location/2, censor_url_query/2, parse_absolute_http_url/1, is_termination_reason_harmless/1 ]). -ifndef(TEST). -hank([{unnecessary_function_arguments, [{is_shutdown_subreason_harmless, 1, 1}]}]). -endif. %% ------------------------------------------------------------------ %% Macro Definitions %% ------------------------------------------------------------------ -define(is_uint8(V), ((V) band 16#FF =:= (V))). -define(is_uint16(V), ((V) band 16#FFFF =:= (V))). -define(TERM_PURGE_LARGE_BINARY_THRESHOLD, (1024 * 1024)). %% ------------------------------------------------------------------ %% API Function Definitions %% ------------------------------------------------------------------ -spec parse_ip_address(binary() | string() | inet:ip_address()) -> {ok, inet:ip_address()} | {error, einval}. parse_ip_address({A, B, C, D} = Address) when ?is_uint8(A), ?is_uint8(B), ?is_uint8(C), ?is_uint8(D) -> {ok, Address}; parse_ip_address({A, B, C, D, E, F, G, H} = Address) when ?is_uint16(A), ?is_uint16(B), ?is_uint16(C), ?is_uint16(D), ?is_uint16(E), ?is_uint16(F), ?is_uint16(G), ?is_uint16(H) -> {ok, Address}; parse_ip_address(Binary) when is_binary(Binary) -> String = binary_to_list(Binary), parse_ip_address(String); parse_ip_address(String) when length(String) >= 0 -> case string:tokens(String, "/") of [StrAddress] -> inet:parse_address(StrAddress); [StrAddress, _PrefixLenStr] -> inet:parse_address(StrAddress); _ -> {error, einval} end; parse_ip_address(_Invalid) -> {error, einval}. -spec lists_anymap(fun ((term()) -> boolean() | {true, term()}), list()) -> {true, term()} | false. lists_anymap(Fun, [H|T]) -> case Fun(H) of {true, Mapped} -> {true, Mapped}; true -> {true, H}; false -> lists_anymap(Fun, T) end; lists_anymap(_Fun, []) -> false. -spec lists_take(term(), list()) -> {ok, list()} | error. lists_take(Elem, List) -> lists_take_recur(Elem, List, []). -spec bin_to_hex_str(binary()) -> [48..57 | 97..102]. bin_to_hex_str(Bin) -> bin_to_hex_str_recur(Bin, []). -spec bin_to_hex_str_recur(bitstring(), [48..57 | 97..102]) -> [48..57 | 97..102]. bin_to_hex_str_recur(<>, Acc) when Nibble < 10 -> bin_to_hex_str_recur(Rest, [$0 + Nibble | Acc]); bin_to_hex_str_recur(<>, Acc) -> bin_to_hex_str_recur(Rest, [$a + Nibble - 10 | Acc]); bin_to_hex_str_recur(<<>>, Acc) -> lists:reverse(Acc). -spec expect_linked_process_termination(pid()) -> boolean(). expect_linked_process_termination(Pid) -> case flush_link_exit(Pid, 5000) of true -> true; false -> exit(Pid, kill), flush_link_exit(Pid, 1000) end. -spec url_query_encode(unicode:chardata()) -> binary(). url_query_encode(Chardata) -> <> = unicode:characters_to_binary(Chardata), << <<(url_query_encode_codepoint(Codepoint))/bytes>> || <> <= Binary >>. -spec filesystem_safe_name(binary()) -> binary(). filesystem_safe_name(Name) -> OnlyWordsAndSpaces = re:replace(Name, "[^\\w\\s-]+", "-", [global, unicode, ucp]), re:replace(OnlyWordsAndSpaces, "[-\\s]+", "-", [global, unicode, ucp, {return, binary}]). -spec is_utf8_binary(term()) -> boolean(). is_utf8_binary(<>) -> case unicode:characters_to_binary(Bytes) of <<_/bytes>> -> true; _ -> false end; is_utf8_binary(_) -> false. -spec is_unicode_string(term()) -> boolean(). is_unicode_string(Value) when length(Value) >= 0 -> try unicode:characters_to_binary(Value) of <<_/bytes>> -> true; _ -> false catch _:_ -> false end; is_unicode_string(_) -> false. -spec is_date(term()) -> boolean(). is_date(Date) -> try calendar:valid_date(Date) of IsValidDate -> IsValidDate catch _:_ -> false end. -spec purge_term_of_very_large_binaries(T) -> T. purge_term_of_very_large_binaries([H|T]) -> MappedH = purge_term_of_very_large_binaries(H), MappedT = purge_term_of_very_large_binaries(T), [MappedH | MappedT]; purge_term_of_very_large_binaries(Tuple) when is_tuple(Tuple) -> List = tuple_to_list(Tuple), MappedList = purge_term_of_very_large_binaries(List), list_to_tuple(MappedList); purge_term_of_very_large_binaries(Map) when is_map(Map) -> List = maps:to_list(Map), MappedList = purge_term_of_very_large_binaries(List), maps:from_list(MappedList); purge_term_of_very_large_binaries(Binary) when is_binary(Binary) -> Size = byte_size(Binary), ReferencedSize = binary:referenced_byte_size(Binary), if Size >= ?TERM_PURGE_LARGE_BINARY_THRESHOLD -> {'__$VERY_LARGE_BINARY', #{size => Size}}; ReferencedSize >= ?TERM_PURGE_LARGE_BINARY_THRESHOLD -> binary:copy(Binary); true -> Binary end; purge_term_of_very_large_binaries(Other) -> Other. -spec resolve_http_location(string(), string()) -> {ok, string()} | {error, term()}. resolve_http_location(BaseURL, Location) -> % See: % * https://tools.ietf.org/html/rfc7231#page-68 % * https://tools.ietf.org/html/rfc3986#section-4.2 case parse_absolute_http_url(Location) of {ok, _} -> % location is an absolute URL {ok, Location}; {error, not_absolute_http_url} -> % location is a relative URL BaseURLParseSuccess = parse_absolute_http_url(BaseURL), % we ignore URL fragments as they wouldn't be sent to the server anyway. {ok, {Scheme, UserInfo, Host, Port, _Path, _Query, _Fragment}} = BaseURLParseSuccess, case Location of "//" ++ _ -> % network-path reference NewURL = format_string("~s:~s", [Scheme, Location]), {ok, NewURL}; "/" ++ _ -> % we should consider the colon character rule here NewURL = build_url(Scheme, UserInfo, Host, Port, Location, ""), {ok, NewURL}; _ -> {error, bad_relative_location} end; {error, Reason} -> {error, Reason} end. -spec censor_url_query(string(), [unicode:charlist()]) -> string(). censor_url_query(URL, KeysToCensor) -> % we ignore URL fragments as they wouldn't have been sent to the server anyway. {ok, {Scheme, UserInfo, Host, Port, Path, Query, _Fragment}} = parse_absolute_http_url(URL), case Query of "?" ++ QueryBody -> BinEncodedKeysToCensor = [url_query_encode(Key) || Key <- KeysToCensor], PrefixesForWhichToCensorSuffix = [binary_to_list(BinEncodedKey) ++ "=" || BinEncodedKey <- BinEncodedKeysToCensor], QueryPairs = string:tokens(QueryBody, [$&]), CensoredQueryPairs = [maybe_censor_url_query_pair(Pair, PrefixesForWhichToCensorSuffix) || Pair <- QueryPairs], CensoredQueryBody = lists:join($&, CensoredQueryPairs), CensoredQuery = [$? | CensoredQueryBody], build_url(Scheme, UserInfo, Host, Port, Path, CensoredQuery); "" -> URL end. -spec parse_absolute_http_url(string()) -> {ok, {atom(), string(), string(), inet:port_number(), string(), string(), string()}} | {error, not_absolute_http_url} | {error, {atom(), term()}}. parse_absolute_http_url(URI) -> case uri_string:parse(URI) of #{scheme := SchemeStr, host := Host} = ParsedURI when (SchemeStr =:= "http" orelse SchemeStr =:= "https"), length(Host) > 0 -> Scheme = list_to_existing_atom(SchemeStr), DefaultPort = case Scheme of http -> 80; https -> 443 end, Query = case ParsedURI of #{query := QueryStr} when length(QueryStr) > 0 -> [$? | QueryStr]; % simulate http_uri:parse/2 behaviour #{} -> "" end, {ok, {Scheme, maps:get(userinfo, ParsedURI, ""), Host, maps:get(port, ParsedURI, DefaultPort), maps:get(path, ParsedURI, "/"), Query, maps:get(fragment, ParsedURI, "") }}; #{} -> {error, not_absolute_http_url}; {error, Reason, Context} -> {error, {Reason, Context}} end. -spec is_termination_reason_harmless(term()) -> boolean(). is_termination_reason_harmless(Reason) -> case Reason of normal -> true; shutdown -> true; {shutdown, SubReason} -> is_shutdown_subreason_harmless(SubReason); _ -> false end. %% ------------------------------------------------------------------ %% Internal Function Definitions %% ------------------------------------------------------------------ -spec lists_take_recur(term(), list(), list()) -> {ok, list()} | error. lists_take_recur(Elem, [H|T], Acc) when Elem =:= H -> {ok, lists:reverse(Acc, T)}; lists_take_recur(Elem, [H|T], Acc) -> lists_take_recur(Elem, T, [H|Acc]); lists_take_recur(_, [], _) -> error. -spec flush_link_exit(pid(), timeout()) -> boolean(). flush_link_exit(Pid, Timeout) -> receive {'EXIT', Pid, _} -> true after Timeout -> false end. url_query_encode_codepoint(Codepoint) when Codepoint >= $A, Codepoint =< $Z; Codepoint >= $a, Codepoint =< $z; Codepoint >= $0, Codepoint =< $9; Codepoint =:= $*; Codepoint =:= $-; Codepoint =:= $.; Codepoint =:= $_ -> <>; url_query_encode_codepoint($\ ) -> <<$+>>; url_query_encode_codepoint(Codepoint) -> UTF8Bytes = <>, << <<$%, (integer_to_binary(Byte, 16))/bytes>> || <> <= UTF8Bytes >>. -spec build_url(atom(), string(), iodata(), inet:port_number(), iodata(), iodata()) -> string(). build_url(Scheme, UserInfo, Host, Port, Path, Query) when UserInfo =:= "", ((Scheme =:= http andalso Port =:= 80) orelse (Scheme =:= https andalso Port =:= 443)) -> format_string("~s://~s~s~s", [Scheme, Host, Path, Query]); build_url(Scheme, UserInfo, Host, Port, Path, Query) when UserInfo =:= "" -> format_string("~s://~s:~b~s", [Scheme, Host, Port, Path, Query]); build_url(Scheme, UserInfo, Host, Port, Path, Query) -> format_string("~s://~s@~s:~b~s", [Scheme, UserInfo, Host, Port, Path, Query]). %-spec format_string(string(), list()) -> string(). format_string(Fmt, Args) -> IoData = io_lib:format(Fmt, Args), Binary = iolist_to_binary(IoData), binary_to_list(Binary). maybe_censor_url_query_pair(Pair, PrefixesForWhichToCensorSuffix) -> case lists_anymap( fun (Prefix) -> lists:prefix(Prefix, Pair) end, PrefixesForWhichToCensorSuffix) of {true, Prefix} -> Prefix ++ "XXXXXXXXXXXXXXXX"; false -> Pair end. -ifdef(TEST). is_shutdown_subreason_harmless(simulated_crash) -> % A way of simulating crash reactions without spamming the console false; is_shutdown_subreason_harmless(_) -> true. -else. is_shutdown_subreason_harmless(_SubReason) -> true. -endif. %% ------------------------------------------------------------------ %% Internal Function Definitions - Unit Tests %% ------------------------------------------------------------------ -ifdef(TEST). resolve_http_location_test() -> BaseURL = "http://www.foo.bar", AbsoluteLocation = "http://www.example.com/whatnot", NetworkPathReference = "//google.com/thing", RelativeReference = "/etc", ?assertEqual({ok, AbsoluteLocation}, resolve_http_location(BaseURL, AbsoluteLocation)), ?assertEqual({ok, "http:" ++ NetworkPathReference}, resolve_http_location(BaseURL, NetworkPathReference)), ?assertEqual({ok, BaseURL ++ RelativeReference}, resolve_http_location(BaseURL, RelativeReference)). -endif.