%% @doc %% Functions for creating and unpacking Hex tarballs. -module(hex_tarball). -export([ create/2, create/3, create_docs/1, create_docs/2, unpack/2, unpack/3, unpack_docs/2, unpack_docs/3, format_checksum/1, format_error/1 ]). -ifdef(TEST). -export([do_decode_metadata/1, do_decode_metadata/2, gzip/1, normalize_requirements/1]). -endif. -define(VERSION, <<"3">>). -define(HASH_CHUNK_SIZE, 65536). -define(MAX_VERSION_SIZE, 32). -define(MAX_CHECKSUM_SIZE, 128). -define(MAX_METADATA_SIZE, 1024 * 1024). -define(METADATA_CHUNK_SIZE, 4096). -define(BUILD_TOOL_FILES, [ {<<"mix.exs">>, <<"mix">>}, {<<"rebar.config">>, <<"rebar3">>}, {<<"rebar">>, <<"rebar3">>}, {<<"Makefile">>, <<"make">>}, {<<"Makefile.win">>, <<"make">>} ]). -include_lib("kernel/include/file.hrl"). -type checksum() :: binary(). -type contents() :: [{filename(), binary()}]. -type filename() :: string(). -type files() :: [{filename(), filename() | binary()}]. -type metadata() :: map(). -type tarball() :: binary(). %%==================================================================== %% API functions %%==================================================================== %% @doc %% Creates a package tarball. %% %% Returns the binary of the tarball the "inner checksum" and "outer checksum". %% The inner checksum is deprecated in favor of the outer checksum. %% %% Examples: %% %% ``` %% > Metadata = #{<<"name">> => <<"foo">>, <<"version">> => <<"1.0.0">>}, %% > Files = [{"src/foo.erl", <<"-module(foo).">>}], %% > hex_tarball:create(Metadata, Files). %% {ok, #{tarball => <<86,69,...>>, %% outer_checksum => <<40,32,...>>, %% inner_checksum => <<178,12,...>>}} %% ''' %% @end -spec create(metadata(), files(), hex_core:config()) -> {ok, #{ tarball => tarball(), outer_checksum => checksum(), inner_checksum => tarball() }} | {error, term()}. create(Metadata, Files, Config) -> #{ tarball_max_size := TarballMaxSize, tarball_max_uncompressed_size := TarballMaxUncompressedSize } = Config, FilesRoot = maps:get(tarball_files_root, Config, "."), MetadataBinary = encode_metadata(Metadata), case valid_size(MetadataBinary, ?MAX_METADATA_SIZE) of false -> {error, {tarball, {file_too_big, "metadata.config"}}}; true -> case validate_create_files(Files, FilesRoot) of {ok, ValidatedFiles} -> ContentsTarball = create_memory_tarball(ValidatedFiles), ContentsTarballCompressed = gzip(ContentsTarball), InnerChecksum = inner_checksum( ?VERSION, MetadataBinary, ContentsTarballCompressed ), InnerChecksumBase16 = encode_base16(InnerChecksum), OuterFiles = [ {"VERSION", ?VERSION}, {"CHECKSUM", InnerChecksumBase16}, {"metadata.config", MetadataBinary}, {"contents.tar.gz", ContentsTarballCompressed} ], case valid_size(ContentsTarball, TarballMaxUncompressedSize) of true -> Tarball = create_memory_tarball(OuterFiles), OuterChecksum = checksum(Tarball), case valid_size(Tarball, TarballMaxSize) of true -> {ok, #{ tarball => Tarball, outer_checksum => OuterChecksum, inner_checksum => InnerChecksum }}; false -> {error, {tarball, {too_big_compressed, TarballMaxSize}}} end; false -> {error, {tarball, {too_big_uncompressed, TarballMaxUncompressedSize}}} end; {error, _} = Error -> Error end end. -spec create(metadata(), files()) -> {ok, #{ tarball => tarball(), outer_checksum => checksum(), inner_checksum => tarball() }} | {error, term()}. create(Metadata, Files) -> create(Metadata, Files, hex_core:default_config()). %% @doc %% Creates a docs tarball. %% %% Examples: %% %% ``` %% > Files = [{"doc/index.html", <<"Docs">>}], %% > hex_tarball:create_docs(Files). %% {ok, <<86,69,...>>} %% ''' %% @end -spec create_docs(files(), hex_core:config()) -> {ok, tarball()} | {error, term()}. create_docs(Files, Config) -> #{ docs_tarball_max_size := TarballMaxSize, docs_tarball_max_uncompressed_size := TarballMaxUncompressedSize } = Config, FilesRoot = maps:get(tarball_files_root, Config, "."), case validate_create_files(Files, FilesRoot) of {ok, ValidatedFiles} -> UncompressedTarball = create_memory_tarball(ValidatedFiles), case valid_size(UncompressedTarball, TarballMaxUncompressedSize) of true -> Tarball = gzip(UncompressedTarball), case valid_size(Tarball, TarballMaxSize) of true -> {ok, Tarball}; false -> {error, {tarball, {too_big_compressed, TarballMaxSize}}} end; false -> {error, {tarball, {too_big_uncompressed, TarballMaxUncompressedSize}}} end; {error, _} = Error -> Error end. -spec create_docs(files()) -> {ok, tarball()} | {error, term()}. create_docs(Files) -> create_docs(Files, hex_core:default_config()). %% @doc %% Unpacks a package tarball. %% %% Remember to verify the outer tarball checksum against the registry checksum %% returned from `hex_repo:get_package(Config, Package)'. %% %% The first argument is the tarball, either as a binary or `{file, Path}' %% to read from a file on disk. Using `{file, Path}' avoids loading the %% tarball into memory. %% %% The second argument controls the output: %% %% - `memory' - unpack contents into memory and return them %% - `none' - only extract metadata and checksums, skip contents %% - A path string - extract contents to the given directory %% %% Examples: %% %% ``` %% > hex_tarball:unpack(Tarball, memory). %% {ok,#{outer_checksum => <<...>>, %% contents => [{"src/foo.erl",<<"-module(foo).">>}], %% metadata => #{<<"name">> => <<"foo">>, ...}}} %% %% > hex_tarball:unpack(Tarball, none). %% {ok,#{outer_checksum => <<...>>, %% metadata => #{<<"name">> => <<"foo">>, ...}}} %% %% > hex_tarball:unpack(Tarball, "path/to/unpack"). %% {ok,#{outer_checksum => <<...>>, %% metadata => #{<<"name">> => <<"foo">>, ...}}} %% ''' -spec unpack (tarball() | {file, filename()}, memory, hex_core:config()) -> {ok, #{ outer_checksum => checksum(), inner_checksum => checksum(), metadata => metadata(), contents => contents() }} | {error, term()}; (tarball() | {file, filename()}, none, hex_core:config()) -> {ok, #{ outer_checksum => checksum(), inner_checksum => checksum(), metadata => metadata() }} | {error, term()}; (tarball() | {file, filename()}, filename(), hex_core:config()) -> {ok, #{ outer_checksum => checksum(), inner_checksum => checksum(), metadata => metadata() }} | {error, term()}. unpack(Input, memory, Config) -> case check_input_size(Input, Config) of true -> OuterChecksum = outer_checksum(Input), Source = tar_source(Input), case hex_erl_tar:extract(Source, [memory]) of {ok, []} -> {error, {tarball, empty}}; {ok, FileList} -> case validate_outer_file_sizes(maps:from_list(FileList)) of {ok, Files} -> do_unpack(Files, OuterChecksum, memory, Config); {error, _} = Error -> Error end; {error, Reason} -> {error, {tarball, Reason}} end; false -> {error, {tarball, too_big}} end; unpack(Input, Output, Config) -> case check_input_size(Input, Config) of true -> OuterChecksum = outer_checksum(Input), Source = tar_source(Input), TmpDir = tmp_path(), ok = file:make_dir(TmpDir), try case hex_erl_tar:extract(Source, [{cwd, TmpDir}]) of ok -> case read_outer_files(TmpDir) of {ok, Files} -> do_unpack(Files, OuterChecksum, Output, Config); {error, _} = Error -> Error end; {error, Reason} -> {error, {tarball, Reason}} end after remove_dir(TmpDir) end; false -> {error, {tarball, too_big}} end. %% @doc %% Unpacks a package tarball. %% %% @see unpack/3 -spec unpack (tarball() | {file, filename()}, memory) -> {ok, #{ outer_checksum => checksum(), inner_checksum => checksum(), metadata => metadata(), contents => contents() }} | {error, term()}; (tarball() | {file, filename()}, none) -> {ok, #{ outer_checksum => checksum(), inner_checksum => checksum(), metadata => metadata() }} | {error, term()}; (tarball() | {file, filename()}, filename()) -> {ok, #{ outer_checksum => checksum(), inner_checksum => checksum(), metadata => metadata() }} | {error, term()}. unpack(Tarball, Output) -> unpack(Tarball, Output, hex_core:default_config()). %% @doc %% Unpacks a documentation tarball. %% %% The first argument is the tarball, either as a binary or `{file, Path}' %% to read from a file on disk. Using `{file, Path}' avoids loading the %% tarball into memory. %% %% Examples: %% %% ``` %% > hex_tarball:unpack_docs(Tarball, memory). %% {ok, [{"index.html", <<"">>}, ...]} %% %% > hex_tarball:unpack_docs(Tarball, "path/to/unpack"). %% ok %% ''' -spec unpack_docs (tarball() | {file, filename()}, memory, hex_core:config()) -> {ok, contents()} | {error, term()}; (tarball() | {file, filename()}, filename(), hex_core:config()) -> ok | {error, term()}. unpack_docs(Input, Output, Config) -> case check_docs_input_size(Input, Config) of true -> MaxSize = maps:get(docs_tarball_max_uncompressed_size, Config), unpack_tarball(tar_source(Input), Output, MaxSize); false -> {error, {tarball, too_big}} end. -spec unpack_docs (tarball() | {file, filename()}, memory) -> {ok, contents()} | {error, term()}; (tarball() | {file, filename()}, filename()) -> ok | {error, term()}. unpack_docs(Input, Output) -> unpack_docs(Input, Output, hex_core:default_config()). %% @doc %% Returns base16-encoded representation of checksum. -spec format_checksum(checksum()) -> binary(). format_checksum(Checksum) -> encode_base16(Checksum). %% @doc %% Converts an error reason term to a human-readable error message string. -spec format_error(term()) -> string(). format_error({tarball, {file_too_big, Name}}) -> io_lib:format("file too big: ~s", [Name]); format_error({tarball, empty}) -> "empty tarball"; format_error({tarball, {too_big_uncompressed, Size}}) -> io_lib:format("package exceeds max uncompressed size ~w ~s", [format_byte_size(Size), "MB"]); format_error({tarball, {too_big_compressed, Size}}) -> io_lib:format("package exceeds max compressed size ~w ~s", [format_byte_size(Size), "MB"]); format_error({tarball, {missing_files, Files}}) -> io_lib:format("missing files: ~p", [Files]); format_error({tarball, {bad_version, Vsn}}) -> io_lib:format("unsupported version: ~p", [Vsn]); format_error({tarball, invalid_checksum}) -> "invalid tarball checksum"; format_error({tarball, {unsafe_path, Name}}) -> io_lib:format("unsafe path in tarball: ~s", [Name]); format_error({tarball, {unsafe_symlink, Name, LinkTarget}}) -> io_lib:format("unsafe symlink in tarball: ~s -> ~s", [Name, LinkTarget]); format_error({tarball, {unsupported_file_type, Name, Type}}) -> io_lib:format("unsupported file type in tarball: ~s (~p)", [Name, Type]); format_error({tarball, Reason}) -> "tarball error, " ++ hex_erl_tar:format_error(Reason); format_error({inner_tarball, Reason}) -> "inner tarball error, " ++ hex_erl_tar:format_error(Reason); format_error({metadata, invalid_terms}) -> "error reading package metadata: invalid terms"; format_error({metadata, not_key_value}) -> "error reading package metadata: not in key-value format"; format_error({metadata, Reason}) -> "error reading package metadata" ++ safe_erl_term:format_error(Reason); format_error({checksum_mismatch, ExpectedChecksum, ActualChecksum}) -> io_lib:format( "tarball checksum mismatch~n~n" ++ "Expected (base16-encoded): ~s~n" ++ "Actual (base16-encoded): ~s", [encode_base16(ExpectedChecksum), encode_base16(ActualChecksum)] ). format_byte_size(Size) -> Size / 1000000. %%==================================================================== %% Internal functions %%==================================================================== %% @private inner_checksum(Version, MetadataBinary, {path, ContentsPath}) -> HashState0 = crypto:hash_init(sha256), HashState1 = crypto:hash_update(HashState0, Version), HashState2 = crypto:hash_update(HashState1, MetadataBinary), HashState3 = stream_file_hash(HashState2, ContentsPath), crypto:hash_final(HashState3); inner_checksum(Version, MetadataBinary, ContentsBinary) -> Blob = <>, crypto:hash(sha256, Blob). %% @private checksum(ContentsBinary) when is_binary(ContentsBinary) -> crypto:hash(sha256, ContentsBinary). %% @private tar_source({file, Path}) -> Path; tar_source(Tarball) -> {binary, Tarball}. %% @private outer_checksum({file, Path}) -> file_checksum(Path); outer_checksum(Tarball) -> crypto:hash(sha256, Tarball). %% @private check_input_size({file, Path}, Config) -> valid_file_size(Path, maps:get(tarball_max_size, Config)); check_input_size(Tarball, Config) -> valid_size(Tarball, maps:get(tarball_max_size, Config)). %% @private check_docs_input_size({file, Path}, Config) -> valid_file_size(Path, maps:get(docs_tarball_max_size, Config)); check_docs_input_size(Tarball, Config) -> valid_size(Tarball, maps:get(docs_tarball_max_size, Config)). %% @private encode_metadata(Meta) -> Data = lists:map( fun(MetaPair) -> String = io_lib_pretty:print(binarify(MetaPair), [{encoding, utf8}]), unicode:characters_to_binary([String, ".\n"]) end, maps:to_list(Meta) ), iolist_to_binary(Data). %% @private do_unpack(Files, OuterChecksum, Output, Config) -> State = #{ inner_checksum => undefined, outer_checksum => OuterChecksum, contents => undefined, files => Files, metadata => undefined, output => Output, config => Config }, State1 = check_files(State), State2 = check_version(State1), State3 = check_inner_checksum(State2), State4 = decode_metadata(State3), finish_unpack(State4). %% @private finish_unpack({error, _} = Error) -> Error; finish_unpack(#{ metadata := Metadata, files := Files, inner_checksum := InnerChecksum, outer_checksum := OuterChecksum, output := Output, config := Config }) -> _ = maps:get("VERSION", Files), Contents = maps:get("contents.tar.gz", Files), MaxUncompressedSize = maps:get(tarball_max_uncompressed_size, Config), Result = #{ inner_checksum => InnerChecksum, outer_checksum => OuterChecksum, metadata => Metadata }, case Output of none -> {ok, Result}; memory -> case unpack_contents(Contents, memory, MaxUncompressedSize) of {ok, UnpackedContents} -> {ok, Result#{contents => UnpackedContents}}; {error, Reason} -> {error, {inner_tarball, Reason}} end; _ -> filelib:ensure_dir(filename:join(Output, "*")), case unpack_contents(Contents, Output, MaxUncompressedSize) of ok -> [ try_updating_mtime(filename:join(Output, P)) || P <- filelib:wildcard("**", Output) ], copy_metadata_config(Output, maps:get("metadata.config", Files)), {ok, Result}; {error, Reason} -> {error, {inner_tarball, Reason}} end end. %% @private unpack_contents(Contents, Output, MaxSize) -> Opts = case Output of memory -> [memory, compressed]; _ -> [{cwd, Output}, compressed] end, Source = case Contents of {path, ContentsPath} -> ContentsPath; ContentsBinary -> {binary, ContentsBinary} end, case hex_erl_tar:extract(Source, [{max_size, MaxSize} | Opts]) of {error, too_big} -> {error, {too_big_uncompressed, MaxSize}}; Other -> Other end. %% @private copy_metadata_config(Output, MetadataBinary) -> ok = file:write_file(filename:join(Output, "hex_metadata.config"), MetadataBinary). %% @private check_files(#{files := Files} = State) -> RequiredFiles = ["VERSION", "CHECKSUM", "metadata.config", "contents.tar.gz"], case diff_keys(Files, RequiredFiles, []) of ok -> State; {error, {missing_keys, Keys}} -> {error, {tarball, {missing_files, Keys}}} end. %% @private check_version({error, _} = Error) -> Error; check_version(#{files := Files} = State) -> case maps:get("VERSION", Files) of <<"3">> -> State; Version -> {error, {tarball, {bad_version, Version}}} end. %% @private % Note: This checksum is deprecated check_inner_checksum({error, _} = Error) -> Error; check_inner_checksum(#{files := Files} = State) -> ChecksumBase16 = maps:get("CHECKSUM", Files), ExpectedChecksum = decode_base16(ChecksumBase16), Version = maps:get("VERSION", Files), MetadataBinary = maps:get("metadata.config", Files), Contents = maps:get("contents.tar.gz", Files), ActualChecksum = inner_checksum(Version, MetadataBinary, Contents), if byte_size(ExpectedChecksum) /= 32 -> {error, {tarball, invalid_inner_checksum}}; ExpectedChecksum == ActualChecksum -> maps:put(inner_checksum, ExpectedChecksum, State); true -> {error, {tarball, {inner_checksum_mismatch, ExpectedChecksum, ActualChecksum}}} end. %% @private decode_metadata({error, _} = Error) -> Error; decode_metadata(#{files := #{"metadata.config" := Binary}, config := Config} = State) when is_binary(Binary) -> Fields = maps:get(metadata_fields, Config, all), case do_decode_metadata(Binary, Fields) of #{} = Metadata -> maps:put(metadata, normalize_metadata(Metadata), State); Other -> Other end. -ifdef(TEST). do_decode_metadata(Binary) -> do_decode_metadata(Binary, all). -endif. %% @private do_decode_metadata(Binary, all) when is_binary(Binary) -> case decode_metadata_chunked(utf8, Binary, <<>>, [], "", []) of latin1_fallback -> decode_metadata_chunked(latin1, Binary, <<>>, [], "", []); Other -> Other end; do_decode_metadata(Binary, Fields) when is_binary(Binary), is_list(Fields) -> case decode_metadata_streaming(utf8, Binary, <<>>, [], "", [], Fields, start) of latin1_fallback -> decode_metadata_streaming(latin1, Binary, <<>>, [], "", [], Fields, start); Other -> Other end. %% @private %% Streams the metadata.config binary through safe_erl_term:tokens/2 in %% small chunks so we never materialize the whole binary as a char list. %% Each accepted dot-terminated form is parsed and accumulated immediately, %% keeping peak memory at roughly one chunk + one term's tokens + AST. decode_metadata_chunked(Encoding, Binary, IncTail, Cont, Chars, Acc) -> case Chars of [] when Binary =:= <<>>, IncTail =:= <<>> -> flush_metadata_eof(Cont, Acc); [] when Binary =:= <<>>, Encoding =:= utf8 -> %% Trailing bytes that can never form a complete UTF-8 sequence — %% restart the whole decode in latin1 mode rather than spin. latin1_fallback; [] -> case decode_metadata_chunk(Encoding, Binary, IncTail) of {ok, NewChars, NewBinary, NewTail} -> feed_metadata(Encoding, Cont, NewChars, NewBinary, NewTail, Acc); latin1_fallback -> latin1_fallback end; _ -> feed_metadata(Encoding, Cont, Chars, Binary, IncTail, Acc) end. %% @private feed_metadata(Encoding, Cont, Chars, Binary, IncTail, Acc) -> case safe_erl_term:tokens(Cont, Chars) of {more, NewCont} -> decode_metadata_chunked(Encoding, Binary, IncTail, NewCont, "", Acc); {done, {ok, Tokens, _}, RestChars} -> case parse_metadata_term(Tokens) of {ok, Term} -> decode_metadata_chunked( Encoding, Binary, IncTail, [], normalize_rest_chars(RestChars), [Term | Acc] ); {error, _} = Err -> Err end; {done, {eof, _}, _} -> finalize_metadata(Acc); {done, {error, {_, safe_erl_term, Reason}, _}, _} -> {error, {metadata, Reason}} end. %% @private flush_metadata_eof([], Acc) -> finalize_metadata(Acc); flush_metadata_eof(Cont, Acc) -> case safe_erl_term:tokens(Cont, eof) of {done, {eof, _}, _} -> finalize_metadata(Acc); {done, {ok, _Tokens, _}, _} -> {error, {metadata, invalid_terms}}; {done, {error, {_, safe_erl_term, Reason}, _}, _} -> {error, {metadata, Reason}} end. %% @private finalize_metadata([]) -> {error, {metadata, invalid_terms}}; finalize_metadata(Acc) -> try maps:from_list(lists:reverse(Acc)) of Map -> Map catch error:badarg -> {error, {metadata, not_key_value}} end. %% @private parse_metadata_term(Tokens) -> case erl_parse:parse_term(Tokens) of {ok, Term} -> {ok, Term}; {error, _} -> {error, {metadata, invalid_terms}} end. %% @private decode_metadata_chunk(utf8, Binary, IncTail) -> {Chunk, Rest} = take_metadata_chunk(Binary), Combined = case IncTail of <<>> -> Chunk; _ -> <> end, case unicode:characters_to_list(Combined, utf8) of L when is_list(L) -> {ok, L, Rest, <<>>}; {incomplete, L, NewTail} -> {ok, L, Rest, NewTail}; {error, _, _} -> latin1_fallback end; decode_metadata_chunk(latin1, Binary, _IncTail) -> {Chunk, Rest} = take_metadata_chunk(Binary), {ok, binary_to_list(Chunk), Rest, <<>>}. %% @private take_metadata_chunk(Binary) when byte_size(Binary) > ?METADATA_CHUNK_SIZE -> <> = Binary, {Chunk, Rest}; take_metadata_chunk(Binary) -> {Binary, <<>>}. %% @private normalize_rest_chars(eof) -> ""; normalize_rest_chars(L) when is_list(L) -> L. %% @private %% Streams the metadata.config binary through safe_erl_term:token/2 one token %% at a time. Forms whose key is in Fields are accumulated and parsed; forms %% whose key is not in Fields are discarded with only a depth counter held in %% state, so peak memory stays bounded regardless of the unwanted form's size. decode_metadata_streaming(Encoding, Binary, IncTail, Cont, Chars, Acc, Fields, State) -> case Chars of [] when Binary =:= <<>>, IncTail =:= <<>> -> flush_metadata_streaming_eof(Cont, Acc, Fields, State); [] when Binary =:= <<>>, Encoding =:= utf8 -> latin1_fallback; [] -> case decode_metadata_chunk(Encoding, Binary, IncTail) of {ok, NewChars, NewBinary, NewTail} -> feed_metadata_streaming( Encoding, Cont, NewChars, NewBinary, NewTail, Acc, Fields, State ); latin1_fallback -> latin1_fallback end; _ -> feed_metadata_streaming(Encoding, Cont, Chars, Binary, IncTail, Acc, Fields, State) end. %% @private feed_metadata_streaming(Encoding, Cont, Chars, Binary, IncTail, Acc, Fields, State) -> case safe_erl_term:token(Cont, Chars) of {more, NewCont} -> decode_metadata_streaming(Encoding, Binary, IncTail, NewCont, "", Acc, Fields, State); {done, {ok, Token, _}, RestChars} -> case advance_metadata_state(State, Acc, Fields, Token) of {next, NewState, NewAcc} -> decode_metadata_streaming( Encoding, Binary, IncTail, [], normalize_rest_chars(RestChars), NewAcc, Fields, NewState ); {error, _} = Err -> Err end; {done, {eof, _}, _} -> finalize_metadata_streaming(Acc, State); {done, {error, {_, safe_erl_term, Reason}, _}, _} -> {error, {metadata, Reason}} end. %% @private flush_metadata_streaming_eof([], Acc, _Fields, State) -> finalize_metadata_streaming(Acc, State); flush_metadata_streaming_eof(Cont, Acc, Fields, State) -> case safe_erl_term:token(Cont, eof) of {done, {ok, Token, _}, _} -> case advance_metadata_state(State, Acc, Fields, Token) of {next, NewState, NewAcc} -> flush_metadata_streaming_eof([], NewAcc, Fields, NewState); {error, _} = Err -> Err end; {done, {eof, _}, _} -> finalize_metadata_streaming(Acc, State); {done, {error, {_, safe_erl_term, Reason}, _}, _} -> {error, {metadata, Reason}} end. %% @private finalize_metadata_streaming(Acc, start) -> finalize_metadata(Acc); finalize_metadata_streaming([], between) -> #{}; finalize_metadata_streaming(Acc, between) -> finalize_metadata(Acc); finalize_metadata_streaming(_Acc, _State) -> {error, {metadata, invalid_terms}}. %% @private %% State machine for streaming the metadata.config schema. Forms are required %% to be `{<<"key">>, value}.` — anything else is rejected as invalid. %% %% States: start | between | {after_open, Prefix} | {after_left_binary, Prefix} %% | {after_key, KeyChars, Prefix} | {after_right_binary, KeyChars, Prefix} %% | {accumulate, Prefix, Depth} | {skip, Depth} %% %% `start` is the initial position; `between` is the position after a form has %% been completed. Distinguishing them lets empty input return the same %% invalid_terms error as the non-streaming path while a stream that %% successfully skipped every form returns an empty map. advance_metadata_state(Open, Acc, _Fields, {'{', _} = T) when Open =:= start; Open =:= between -> {next, {after_open, [T]}, Acc}; advance_metadata_state({after_open, Prefix}, Acc, _Fields, {'<<', _} = T) -> {next, {after_left_binary, [T | Prefix]}, Acc}; advance_metadata_state({after_left_binary, Prefix}, Acc, _Fields, {string, _, KeyChars} = T) -> {next, {after_key, KeyChars, [T | Prefix]}, Acc}; advance_metadata_state({after_key, KeyChars, Prefix}, Acc, _Fields, {'>>', _} = T) -> {next, {after_right_binary, KeyChars, [T | Prefix]}, Acc}; advance_metadata_state({after_right_binary, KeyChars, Prefix}, Acc, Fields, {',', _} = T) -> case extract_metadata_key(KeyChars) of {ok, Key} -> case lists:member(Key, Fields) of true -> {next, {accumulate, [T | Prefix], 1}, Acc}; false -> {next, {skip, 1}, Acc} end; error -> {error, {metadata, not_key_value}} end; advance_metadata_state({accumulate, Prefix, 0}, Acc, _Fields, {dot, _} = T) -> Tokens = lists:reverse([T | Prefix]), case parse_metadata_term(Tokens) of {ok, Term} -> {next, between, [Term | Acc]}; {error, _} = Err -> Err end; advance_metadata_state({accumulate, _, _}, _Acc, _Fields, {dot, _}) -> {error, {metadata, invalid_terms}}; advance_metadata_state({accumulate, Prefix, Depth}, Acc, _Fields, {Open, _} = T) when Open =:= '{'; Open =:= '[' -> {next, {accumulate, [T | Prefix], Depth + 1}, Acc}; advance_metadata_state({accumulate, Prefix, Depth}, Acc, _Fields, {Close, _} = T) when Close =:= '}'; Close =:= ']' -> {next, {accumulate, [T | Prefix], Depth - 1}, Acc}; advance_metadata_state({accumulate, Prefix, Depth}, Acc, _Fields, T) -> {next, {accumulate, [T | Prefix], Depth}, Acc}; advance_metadata_state({skip, 0}, Acc, _Fields, {dot, _}) -> {next, between, Acc}; advance_metadata_state({skip, _}, _Acc, _Fields, {dot, _}) -> {error, {metadata, invalid_terms}}; advance_metadata_state({skip, Depth}, Acc, _Fields, {Open, _}) when Open =:= '{'; Open =:= '[' -> {next, {skip, Depth + 1}, Acc}; advance_metadata_state({skip, Depth}, Acc, _Fields, {Close, _}) when Close =:= '}'; Close =:= ']' -> {next, {skip, Depth - 1}, Acc}; advance_metadata_state({skip, Depth}, Acc, _Fields, _Token) -> {next, {skip, Depth}, Acc}; advance_metadata_state(_State, _Acc, _Fields, _Token) -> {error, {metadata, not_key_value}}. %% @private extract_metadata_key(KeyChars) -> try list_to_binary(KeyChars) of Key -> {ok, Key} catch error:badarg -> error end. %% @private normalize_metadata(Metadata1) -> Metadata2 = maybe_update_with(<<"requirements">>, fun normalize_requirements/1, Metadata1), Metadata3 = maybe_update_with(<<"links">>, fun try_into_map/1, Metadata2), Metadata4 = maybe_update_with(<<"extra">>, fun try_into_nested_map/1, Metadata3), guess_build_tools(Metadata4). %% @private normalize_requirements(Requirements) -> case is_list(Requirements) andalso (Requirements /= []) andalso is_list(hd(Requirements)) of true -> maps:from_list(lists:map(fun normalize_legacy_requirement/1, Requirements)); false -> try_into_map(fun normalize_normal_requirement/1, Requirements) end. %% @private normalize_normal_requirement({Name, Requirement}) -> {Name, try_into_map(Requirement)}. %% @private normalize_legacy_requirement(Requirement) -> Map = maps:from_list(Requirement), Name = maps:get(<<"name">>, Map), {Name, maps:without([<<"name">>], Map)}. %% @private guess_build_tools(#{<<"build_tools">> := BuildTools} = Metadata) when is_list(BuildTools) -> Metadata; guess_build_tools(#{<<"files">> := Filenames} = Metadata) -> BaseFiles = [ Filename || Filename <- Filenames, filename:dirname(binary_to_list(Filename)) == "." ], BuildTools = lists:usort([ Tool || {Filename, Tool} <- ?BUILD_TOOL_FILES, lists:member(Filename, BaseFiles) ]), Metadata#{<<"build_tools">> => BuildTools}; guess_build_tools(Metadata) -> Metadata. %%==================================================================== %% Tar Helpers %%==================================================================== %% @private validate_create_files(Files, FilesRoot) when is_list(Files) -> validate_create_files(Files, FilesRoot, []). validate_create_files([], _FilesRoot, Acc) -> {ok, lists:reverse(Acc)}; validate_create_files([File | Rest], FilesRoot, Acc) -> case validate_create_file(File, FilesRoot) of {ok, ValidatedFile} -> validate_create_files(Rest, FilesRoot, [ValidatedFile | Acc]); {error, _} = Error -> Error end. validate_create_file({Filename, Contents}, _FilesRoot) when is_list(Filename), is_binary(Contents) -> case validate_archive_path(Filename) of ok -> {ok, {Filename, Contents}}; {error, _} = Error -> Error end; validate_create_file(Filename, FilesRoot) when is_list(Filename) -> validate_create_file({Filename, Filename}, FilesRoot); validate_create_file({Filename, AbsFilename}, FilesRoot) when is_list(Filename), is_list(AbsFilename) -> case validate_archive_path(Filename) of ok -> validate_source_file(Filename, AbsFilename, FilesRoot); {error, _} = Error -> Error end. validate_archive_path(Filename) -> case safe_relative_archive_path(Filename) of false -> {error, {tarball, {unsafe_path, Filename}}}; true -> ok end. validate_source_file(ArchiveName, SourcePath, FilesRoot) -> case source_file_paths(SourcePath, FilesRoot) of {ok, DiskPath, RelativePath, Root} -> validate_source_file_root(ArchiveName, DiskPath, RelativePath, Root); outside_root -> {error, {tarball, {unsafe_path, ArchiveName}}} end. source_file_paths(SourcePath, FilesRoot) -> Root = normalize_root(filename:absname(FilesRoot)), case source_relative_path(SourcePath, Root) of {ok, RelativePath} -> {ok, source_disk_path(SourcePath, Root), RelativePath, Root}; outside_root -> outside_root end. normalize_root(Path) -> filename:join(normalize_root_parts(filename:split(Path), [])). normalize_root_parts([], Acc) -> lists:reverse(Acc); normalize_root_parts(["." | Parts], Acc) -> normalize_root_parts(Parts, Acc); normalize_root_parts([".." | Parts], Acc) -> normalize_root_parent(Parts, Acc); normalize_root_parts([Part | Parts], Acc) -> normalize_root_parts(Parts, [Part | Acc]). normalize_root_parent(Parts, [Root] = Acc) -> case filename:pathtype(Root) of relative -> normalize_root_parts(Parts, []); _ -> normalize_root_parts(Parts, Acc) end; normalize_root_parent(Parts, [_Part | Acc]) -> normalize_root_parts(Parts, Acc); normalize_root_parent(Parts, []) -> normalize_root_parts(Parts, []). source_disk_path(SourcePath, Root) -> case filename:pathtype(SourcePath) of absolute -> SourcePath; _ -> filename:join(Root, SourcePath) end. source_relative_path(SourcePath, Root) -> case filename:pathtype(SourcePath) of absolute -> strip_root_path(filename:split(SourcePath), filename:split(Root)); _ -> {ok, SourcePath} end. strip_root_path([], []) -> {ok, "."}; strip_root_path(PathParts, []) -> {ok, filename:join(PathParts)}; strip_root_path([Part | PathParts], [Part | RootParts]) -> strip_root_path(PathParts, RootParts); strip_root_path(_PathParts, _RootParts) -> outside_root. validate_source_file_root(ArchiveName, DiskPath, RelativePath, Root) -> case file:read_link_info(DiskPath, []) of {ok, #file_info{type = Type}} when Type =:= regular; Type =:= directory -> case validate_source_root(ArchiveName, RelativePath, Root) of ok -> {ok, {ArchiveName, DiskPath}}; {error, _} = Error -> Error end; {ok, #file_info{type = symlink}} -> {ok, LinkTarget} = file:read_link(DiskPath), ResolvedTarget = archive_join(archive_dirname(ArchiveName), LinkTarget), case safe_relative_archive_path(ResolvedTarget) of false -> {error, {tarball, {unsafe_symlink, ArchiveName, LinkTarget}}}; true -> case validate_source_root(ArchiveName, RelativePath, Root) of ok -> {ok, {ArchiveName, DiskPath}}; {error, _} = Error -> Error end end; {ok, #file_info{type = Type}} -> {error, {tarball, {unsupported_file_type, ArchiveName, Type}}}; _ -> case validate_source_root(ArchiveName, RelativePath, Root) of ok -> {ok, {ArchiveName, DiskPath}}; {error, _} = Error -> Error end end. validate_source_root(ArchiveName, SourcePath, FilesRoot) -> case filelib:safe_relative_path(SourcePath, FilesRoot) of unsafe -> {error, {tarball, {unsafe_path, ArchiveName}}}; _ -> ok end. safe_relative_archive_path(Path) -> case archive_path_absolute(Path) orelse archive_path_drive(Path) of true -> false; false -> safe_relative_archive_path(archive_path_split(Path), []) end. safe_relative_archive_path([], _Acc) -> true; safe_relative_archive_path(["." | Rest], Acc) -> safe_relative_archive_path(Rest, Acc); safe_relative_archive_path([".." | _Rest], []) -> false; safe_relative_archive_path([".." | Rest], [_ | Acc]) -> safe_relative_archive_path(Rest, Acc); safe_relative_archive_path([_Part | Rest], Acc) -> safe_relative_archive_path(Rest, [ok | Acc]). archive_path_absolute([$/ | _Rest]) -> true; archive_path_absolute([$\\ | _Rest]) -> true; archive_path_absolute(_Path) -> false. archive_path_drive([Drive, $: | _Rest]) when Drive >= $a, Drive =< $z; Drive >= $A, Drive =< $Z -> true; archive_path_drive(_Path) -> false. archive_path_split(Path) -> string:tokens(Path, "/\\"). archive_dirname(Path) -> case archive_path_split(Path) of [] -> "."; [_Name] -> "."; Parts -> string:join(lists:droplast(Parts), "/") end. archive_join(_Dir, Path) when Path =:= [] -> Path; archive_join(_Dir, Path = [$/ | _Rest]) -> Path; archive_join(_Dir, Path = [$\\ | _Rest]) -> Path; archive_join(_Dir, Path = [Drive, $: | _Rest]) when Drive >= $a, Drive =< $z; Drive >= $A, Drive =< $Z -> Path; archive_join(".", Path) -> Path; archive_join(Dir, Path) -> Dir ++ "/" ++ Path. %% @private unpack_tarball(Source, memory, MaxSize) -> case hex_erl_tar:extract(Source, [memory, compressed, {max_size, MaxSize}]) of {error, too_big} -> {error, {tarball, {too_big_uncompressed, MaxSize}}}; Other -> Other end; unpack_tarball(Source, Output, MaxSize) -> filelib:ensure_dir(filename:join(Output, "*")), case hex_erl_tar:extract(Source, [{cwd, Output}, compressed, {max_size, MaxSize}]) of ok -> [ try_updating_mtime(filename:join(Output, Path)) || Path <- filelib:wildcard("**", Output) ], ok; {error, too_big} -> {error, {tarball, {too_big_uncompressed, MaxSize}}}; Other -> Other end. %% @private %% let it silently fail for bad symlinks try_updating_mtime(Path) -> Time = calendar:universal_time(), _ = file:write_file_info(Path, #file_info{mtime = Time}, [{time, universal}]), ok. %% @private create_memory_tarball(Files) -> Path = tmp_path(), {ok, Tar} = hex_erl_tar:open(Path, [write]), try add_files(Tar, Files) after ok = hex_erl_tar:close(Tar) end, {ok, Tarball} = file:read_file(Path), ok = file:delete(Path), Tarball. %% @private tmp_path() -> "tmp_" ++ binary_to_list(encode_base16(crypto:strong_rand_bytes(32))). %% @private add_files(Tar, Files) when is_list(Files) -> lists:map(fun(File) -> add_file(Tar, File) end, Files). %% @private add_file(Tar, {Filename, Contents}) when is_list(Filename) and is_binary(Contents) -> ok = hex_erl_tar:add(Tar, Contents, Filename, tar_opts()); add_file(Tar, Filename) when is_list(Filename) -> add_file(Tar, {Filename, Filename}); add_file(Tar, {Filename, AbsFilename}) when is_list(Filename), is_list(AbsFilename) -> {ok, FileInfo} = file:read_link_info(AbsFilename, []), case FileInfo#file_info.type of symlink -> ok = hex_erl_tar:add(Tar, {Filename, AbsFilename}, tar_opts()); directory -> case file:list_dir(AbsFilename) of {ok, []} -> hex_erl_tar:add(Tar, {Filename, AbsFilename}, tar_opts()); {ok, _} -> ok end; _ -> Mode = FileInfo#file_info.mode, {ok, Contents} = file:read_file(AbsFilename), ok = hex_erl_tar:add(Tar, Contents, Filename, [{mode, Mode} | tar_opts()]) end. %% @private tar_opts() -> NixEpoch = calendar:datetime_to_gregorian_seconds({{1970, 1, 1}, {0, 0, 0}}), Y2kEpoch = calendar:datetime_to_gregorian_seconds({{2000, 1, 1}, {0, 0, 0}}), Epoch = Y2kEpoch - NixEpoch, [{atime, Epoch}, {mtime, Epoch}, {ctime, Epoch}, {uid, 0}, {gid, 0}]. %% @private %% Reproducible gzip by not setting mtime and OS %% %% From https://tools.ietf.org/html/rfc1952 %% %% +---+---+---+---+---+---+---+---+---+---+ %% |ID1|ID2|CM |FLG| MTIME |XFL|OS | (more-->) %% +---+---+---+---+---+---+---+---+---+---+ %% %% +=======================+ %% |...compressed blocks...| (more-->) %% +=======================+ %% %% +---+---+---+---+---+---+---+---+ %% | CRC32 | ISIZE | %% +---+---+---+---+---+---+---+---+ gzip(Uncompressed) -> Compressed = gzip_no_header(Uncompressed), Header = <<31, 139, 8, 0, 0, 0, 0, 0, 0, 0>>, Crc = erlang:crc32(Uncompressed), Size = byte_size(Uncompressed), Trailer = <>, iolist_to_binary([Header, Compressed, Trailer]). %% @private gzip_no_header(Uncompressed) -> Zstream = zlib:open(), try zlib:deflateInit(Zstream, default, deflated, -15, 8, default), Compressed = zlib:deflate(Zstream, Uncompressed, finish), zlib:deflateEnd(Zstream), iolist_to_binary(Compressed) after zlib:close(Zstream) end. %%==================================================================== %% Helpers %%==================================================================== %% @private valid_size(Binary, infinity) when is_binary(Binary) -> true; valid_size(Binary, Limit) when is_binary(Binary) and is_integer(Limit) -> byte_size(Binary) =< Limit. %% @private valid_file_size(_Path, infinity) -> true; valid_file_size(Path, Limit) when is_integer(Limit) -> case file:read_file_info(Path) of {ok, #file_info{size = Size}} -> Size =< Limit; {error, _} -> false end. %% @private file_checksum(Path) -> {ok, Fd} = file:open(Path, [read, raw, binary]), try file_checksum_loop(Fd, crypto:hash_init(sha256)) after file:close(Fd) end. %% @private file_checksum_loop(Fd, HashState) -> case file:read(Fd, ?HASH_CHUNK_SIZE) of {ok, Data} -> file_checksum_loop(Fd, crypto:hash_update(HashState, Data)); eof -> crypto:hash_final(HashState) end. %% @private stream_file_hash(HashState, Path) -> {ok, Fd} = file:open(Path, [read, raw, binary]), try stream_file_hash_loop(Fd, HashState) after file:close(Fd) end. %% @private stream_file_hash_loop(Fd, HashState) -> case file:read(Fd, ?HASH_CHUNK_SIZE) of {ok, Data} -> stream_file_hash_loop(Fd, crypto:hash_update(HashState, Data)); eof -> HashState end. %% @private %% Reads outer tar files from a directory after extraction. %% Small files (VERSION, CHECKSUM, metadata.config) are read into memory. %% contents.tar.gz is referenced by path. read_outer_files(Dir) -> RequiredFiles = ["VERSION", "CHECKSUM", "metadata.config", "contents.tar.gz"], case read_outer_files(Dir, RequiredFiles, #{}) of {ok, Files} -> validate_outer_file_sizes(Files); {error, _} = Error -> Error end. read_outer_files(_Dir, [], Acc) -> {ok, Acc}; read_outer_files(Dir, ["contents.tar.gz" | Rest], Acc) -> Path = filename:join(Dir, "contents.tar.gz"), case filelib:is_regular(Path) of true -> read_outer_files(Dir, Rest, Acc#{"contents.tar.gz" => {path, Path}}); false -> {error, {tarball, {missing_files, ["contents.tar.gz"]}}} end; read_outer_files(Dir, [Name | Rest], Acc) -> Path = filename:join(Dir, Name), case file:read_file(Path) of {ok, Data} -> read_outer_files(Dir, Rest, Acc#{Name => Data}); {error, _} -> {error, {tarball, {missing_files, [Name]}}} end. %% @private validate_outer_file_sizes(Files) -> case byte_size(maps:get("VERSION", Files, <<>>)) > ?MAX_VERSION_SIZE of true -> {error, {tarball, {file_too_big, "VERSION"}}}; false -> case byte_size(maps:get("CHECKSUM", Files, <<>>)) > ?MAX_CHECKSUM_SIZE of true -> {error, {tarball, {file_too_big, "CHECKSUM"}}}; false -> case byte_size(maps:get("metadata.config", Files, <<>>)) > ?MAX_METADATA_SIZE of true -> {error, {tarball, {file_too_big, "metadata.config"}}}; false -> {ok, Files} end end end. %% @private remove_dir(Dir) -> case file:list_dir(Dir) of {ok, Entries} -> lists:foreach( fun(Entry) -> Path = filename:join(Dir, Entry), case filelib:is_dir(Path) of true -> remove_dir(Path); false -> file:delete(Path) end end, Entries ), file:del_dir(Dir); {error, _} -> ok end. %% @private binarify(Binary) when is_binary(Binary) -> Binary; binarify(Number) when is_number(Number) -> Number; binarify(Atom) when Atom == undefined orelse is_boolean(Atom) -> Atom; binarify(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); binarify(List) when is_list(List) -> [binarify(E) || E <- List]; binarify({Key, Value}) -> {binarify(Key), binarify(Value)}; binarify(Map) when is_map(Map) -> List = maps:to_list(Map), lists:map(fun({K, V}) -> binarify({K, V}) end, List). %% @private diff_keys(Map, RequiredKeys, OptionalKeys) -> Keys = maps:keys(Map), MissingKeys = RequiredKeys -- Keys, UnknownKeys = Keys -- (RequiredKeys ++ OptionalKeys), case {MissingKeys, UnknownKeys} of {[], []} -> ok; % Server should validate this but clients should not % {_, [_ | _]} -> % {error, {unknown_keys, UnknownKeys}}; _ -> {error, {missing_keys, MissingKeys}} end. %% @private maybe_update_with(Key, Fun, Map) -> case maps:find(Key, Map) of {ok, Value} -> maps:put(Key, Fun(Value), Map); error -> Map end. %% @private try_into_map(List) -> try_into_map(fun(X) -> X end, List). %% @private try_into_map(Fun, Input) -> case has_map_shape(Input) of true -> maps:from_list(lists:map(Fun, Input)); false -> Input end. %% @private try_into_nested_map(List) -> try_into_nested_map(fun(X) -> X end, List). %% @private try_into_nested_map(Fun, Input) -> case has_map_shape(Input) of true -> maps:from_list( lists:map( fun({Key, Value}) -> Fun({Key, try_into_nested_map(Fun, Value)}) end, Input ) ); false -> Input end. %% @private has_map_shape(Input) -> is_list(Input) andalso lists:all(fun(E) -> is_tuple(E) andalso (tuple_size(E) == 2) end, Input). %% @private encode_base16(Binary) -> <> = Binary, String = string:to_upper(lists:flatten(io_lib:format("~64.16.0b", [X]))), list_to_binary(String). %% Based on https://github.com/goj/base16/blob/master/src/base16.erl %% (C) 2012, Erlang Solutions Ltd. %% @private decode_base16(Base16) -> <<<<(unhex(H) bsl 4 + unhex(L))>> || <> <= Base16>>. %% @private unhex(D) when $0 =< D andalso D =< $9 -> D - $0; unhex(D) when $a =< D andalso D =< $f -> 10 + D - $a; unhex(D) when $A =< D andalso D =< $F -> 10 + D - $A.