-module(packkit@internal@huf). -compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch, inline]). -define(FILEPATH, "src/packkit/internal/huf.gleam"). -export([decode_stream/3, decode_four_streams/3, read_tree/1]). -export_type([huf_error/0, tree/0, fwd_bit_reader/0]). -if(?OTP_RELEASE >= 27). -define(MODULEDOC(Str), -moduledoc(Str)). -define(DOC(Str), -doc(Str)). -else. -define(MODULEDOC(Str), -compile([])). -define(DOC(Str), -compile([])). -endif. ?MODULEDOC(false). -type huf_error() :: {huf_truncated, binary()} | {huf_invalid_weights, binary()} | {huf_bitstream_error, packkit@internal@fse:fse_error()} | {huf_unsupported, binary()}. -type tree() :: {tree, integer(), gleam@dict:dict(integer(), {integer(), integer()})}. -type fwd_bit_reader() :: {fwd_bit_reader, bitstring(), integer(), integer(), integer()}. -file("src/packkit/internal/huf.gleam", 77). ?DOC(false). -spec unpack_direct_weights(bitstring(), integer(), list(integer())) -> list(integer()). unpack_direct_weights(Bytes, Remaining, Acc) -> case {Bytes, Remaining} of {_, 0} -> lists:reverse(Acc); {<>, 1} -> High = erlang:'bsr'(Byte, 4), lists:reverse([High | Acc]); {<>, _} -> High@1 = erlang:'bsr'(Byte@1, 4), Low = erlang:'band'(Byte@1, 16#0F), unpack_direct_weights(Rest, Remaining - 2, [Low, High@1 | Acc]); {_, _} -> lists:reverse(Acc) end. -file("src/packkit/internal/huf.gleam", 129). ?DOC(false). -spec sum_powers(list(integer()), integer()) -> integer(). sum_powers(Weights, Acc) -> case Weights of [] -> Acc; [0 | Rest] -> sum_powers(Rest, Acc); [W | Rest@1] -> sum_powers(Rest@1, Acc + erlang:'bsl'(1, W - 1)) end. -file("src/packkit/internal/huf.gleam", 141). ?DOC(false). -spec high_bit_loop(integer(), integer()) -> integer(). high_bit_loop(Value, Acc) -> case Value of 0 -> Acc; _ -> high_bit_loop(erlang:'bsr'(Value, 1), Acc + 1) end. -file("src/packkit/internal/huf.gleam", 137). ?DOC(false). -spec high_bit(integer()) -> integer(). high_bit(Value) -> high_bit_loop(Value, -1). -file("src/packkit/internal/huf.gleam", 108). ?DOC(false). -spec weights_with_implied_last(list(integer())) -> {ok, {list(integer()), integer()}} | {error, nil}. weights_with_implied_last(Weights) -> Weight_total = sum_powers(Weights, 0), case Weight_total of 0 -> {error, nil}; _ -> Max_bits = high_bit(Weight_total) + 1, Total = erlang:'bsl'(1, Max_bits), Rest = Total - Weight_total, case (Rest > 0) andalso (erlang:'bsl'(1, high_bit(Rest)) =:= Rest) of false -> {error, nil}; true -> Last_weight = high_bit(Rest) + 1, {ok, {lists:append(Weights, [Last_weight]), Max_bits}} end end. -file("src/packkit/internal/huf.gleam", 200). ?DOC(false). -spec fill_span( gleam@dict:dict(integer(), {integer(), integer()}), integer(), integer(), integer(), integer() ) -> gleam@dict:dict(integer(), {integer(), integer()}). fill_span(Acc, Start, Span, Sym, Bits) -> case Span of 0 -> Acc; _ -> fill_span( gleam@dict:insert(Acc, Start, {Sym, Bits}), Start + 1, Span - 1, Sym, Bits ) end. -file("src/packkit/internal/huf.gleam", 184). ?DOC(false). -spec fill_lookup( list({integer(), integer()}), integer(), integer(), gleam@dict:dict(integer(), {integer(), integer()}) ) -> gleam@dict:dict(integer(), {integer(), integer()}). fill_lookup(Sorted, Max_bits, Next_code, Acc) -> case Sorted of [] -> Acc; [{Sym, Bits} | Rest] -> Span = erlang:'bsl'(1, Max_bits - Bits), Updated = fill_span(Acc, Next_code, Span, Sym, Bits), fill_lookup(Rest, Max_bits, Next_code + Span, Updated) end. -file("src/packkit/internal/huf.gleam", 148). ?DOC(false). -spec build_lookup_table(list(integer()), integer()) -> gleam@dict:dict(integer(), {integer(), integer()}). build_lookup_table(Weights, Max_bits) -> With_bits = begin _pipe = gleam@list:index_fold( Weights, [], fun(Acc, Weight, Symbol) -> case Weight of 0 -> Acc; _ -> [{Symbol, (Max_bits + 1) - Weight} | Acc] end end ), lists:reverse(_pipe) end, Sorted = gleam@list:sort( With_bits, fun(A, B) -> {Sym_a, Bits_a} = A, {Sym_b, Bits_b} = B, case gleam@int:compare(Bits_b, Bits_a) of eq -> gleam@int:compare(Sym_a, Sym_b); Other -> Other end end ), fill_lookup(Sorted, Max_bits, 0, maps:new()). -file("src/packkit/internal/huf.gleam", 98). ?DOC(false). -spec build_tree(list(integer())) -> {ok, tree()} | {error, huf_error()}. build_tree(Weights) -> case weights_with_implied_last(Weights) of {error, _} -> {error, {huf_invalid_weights, <<"huf: invalid weight stream"/utf8>>}}; {ok, {All_weights, Max_bits}} -> Lookup = build_lookup_table(All_weights, Max_bits), {ok, {tree, Max_bits, Lookup}} end. -file("src/packkit/internal/huf.gleam", 60). ?DOC(false). -spec read_direct_weights(integer(), bitstring()) -> {ok, {tree(), integer()}} | {error, huf_error()}. read_direct_weights(Header_byte, Rest) -> Num_symbols = Header_byte - 127, Weights_byte_count = (Num_symbols + 1) div 2, case gleam_stdlib:bit_array_slice(Rest, 0, Weights_byte_count) of {error, _} -> {error, {huf_truncated, <<"huf: direct-weight body truncated"/utf8>>}}; {ok, Weights_bits} -> Weights = unpack_direct_weights(Weights_bits, Num_symbols, []), gleam@result:'try'( build_tree(Weights), fun(Tree) -> {ok, {Tree, 1 + Weights_byte_count}} end ) end. -file("src/packkit/internal/huf.gleam", 247). ?DOC(false). -spec decode_symbols_loop( tree(), packkit@internal@fse:backward_reader(), integer(), bitstring() ) -> {ok, bitstring()} | {error, huf_error()}. decode_symbols_loop(Tree, Reader, Remaining, Acc) -> case Remaining of 0 -> {ok, Acc}; _ -> case begin _pipe = packkit@internal@fse:read_backward_bits_padded( Reader, erlang:element(2, Tree) ), gleam@result:map_error( _pipe, fun(Field@0) -> {huf_bitstream_error, Field@0} end ) end of {error, E} -> {error, E}; {ok, {Index, _}} -> case gleam_stdlib:map_get(erlang:element(3, Tree), Index) of {error, _} -> {error, {huf_invalid_weights, <<"huf: bitstream index out of table"/utf8>>}}; {ok, {Sym, Used_bits}} -> case begin _pipe@1 = packkit@internal@fse:read_backward_bits_padded( Reader, Used_bits ), gleam@result:map_error( _pipe@1, fun(Field@0) -> {huf_bitstream_error, Field@0} end ) end of {error, E@1} -> {error, E@1}; {ok, {_, Reader_consumed}} -> decode_symbols_loop( Tree, Reader_consumed, Remaining - 1, <> ) end end end end. -file("src/packkit/internal/huf.gleam", 224). ?DOC(false). -spec decode_stream(tree(), bitstring(), integer()) -> {ok, bitstring()} | {error, huf_error()}. decode_stream(Tree, Bytes, Num_symbols) -> gleam@result:'try'( begin _pipe = packkit@internal@fse:new_backward_reader(Bytes), gleam@result:map_error( _pipe, fun(Field@0) -> {huf_bitstream_error, Field@0} end ) end, fun(Reader) -> decode_symbols_loop(Tree, Reader, Num_symbols, <<>>) end ). -file("src/packkit/internal/huf.gleam", 354). ?DOC(false). -spec slice_stream(bitstring(), integer(), integer()) -> {ok, bitstring()} | {error, huf_error()}. slice_stream(Bytes, Offset, Length) -> case gleam_stdlib:bit_array_slice(Bytes, Offset, Length) of {ok, Value} -> {ok, Value}; {error, _} -> {error, {huf_truncated, <<"huf: stream slice out of range"/utf8>>}} end. -file("src/packkit/internal/huf.gleam", 335). ?DOC(false). -spec decode_split_streams( tree(), bitstring(), {integer(), integer(), integer(), integer()}, {integer(), integer(), integer(), integer()} ) -> {ok, bitstring()} | {error, huf_error()}. decode_split_streams(Tree, Streams, Byte_sizes, Symbol_counts) -> {B1, B2, B3, B4} = Byte_sizes, {C1, C2, C3, C4} = Symbol_counts, gleam@result:'try'( slice_stream(Streams, 0, B1), fun(S1) -> gleam@result:'try'( slice_stream(Streams, B1, B2), fun(S2) -> gleam@result:'try'( slice_stream(Streams, B1 + B2, B3), fun(S3) -> gleam@result:'try'( slice_stream(Streams, (B1 + B2) + B3, B4), fun(S4) -> gleam@result:'try'( decode_stream(Tree, S1, C1), fun(O1) -> gleam@result:'try'( decode_stream(Tree, S2, C2), fun(O2) -> gleam@result:'try'( decode_stream( Tree, S3, C3 ), fun(O3) -> gleam@result:'try'( decode_stream( Tree, S4, C4 ), fun(O4) -> {ok, gleam_stdlib:bit_array_concat( [O1, O2, O3, O4] )} end ) end ) end ) end ) end ) end ) end ) end ). -file("src/packkit/internal/huf.gleam", 294). ?DOC(false). -spec decode_four_streams(tree(), bitstring(), integer()) -> {ok, bitstring()} | {error, huf_error()}. decode_four_streams(Tree, Bytes, Regenerated_size) -> Total = erlang:byte_size(Bytes), case Total < 6 of true -> {error, {huf_truncated, <<"huf: 4-stream jump table truncated"/utf8>>}}; false -> case Bytes of <> -> Body_size = Total - 6, Size4 = ((Body_size - Size1) - Size2) - Size3, gleam@bool:guard( Size4 < 0, {error, {huf_invalid_weights, <<"huf: 4-stream jump table sums past body"/utf8>>}}, fun() -> Per_stream = (Regenerated_size + 3) div 4, Stream4_size = Regenerated_size - (Per_stream * 3), decode_split_streams( Tree, Streams, {Size1, Size2, Size3, Size4}, {Per_stream, Per_stream, Per_stream, Stream4_size} ) end ); _ -> {error, {huf_truncated, <<"huf: 4-stream header malformed"/utf8>>}} end end. -file("src/packkit/internal/huf.gleam", 471). ?DOC(false). -spec lookup_symbol( gleam@dict:dict(integer(), packkit@internal@fse:state_entry()), integer() ) -> integer(). lookup_symbol(Table, State) -> _pipe = gleam_stdlib:map_get(Table, State), _pipe@1 = gleam@result:map( _pipe, fun(Entry) -> erlang:element(2, Entry) end ), gleam@result:unwrap(_pipe@1, 0). -file("src/packkit/internal/huf.gleam", 493). ?DOC(false). -spec new_fwd_reader(bitstring()) -> fwd_bit_reader(). new_fwd_reader(Bytes) -> {fwd_bit_reader, Bytes, 0, 0, 0}. -file("src/packkit/internal/huf.gleam", 497). ?DOC(false). -spec fwd_refill(fwd_bit_reader(), integer()) -> fwd_bit_reader(). fwd_refill(Reader, Needed) -> gleam@bool:guard( erlang:element(4, Reader) >= Needed, Reader, fun() -> case erlang:element(2, Reader) of <> -> fwd_refill( {fwd_bit_reader, Rest, erlang:'bor'( erlang:element(3, Reader), erlang:'bsl'(Byte, erlang:element(4, Reader)) ), erlang:element(4, Reader) + 8, erlang:element(5, Reader)}, Needed ); _ -> Reader end end ). -file("src/packkit/internal/huf.gleam", 517). ?DOC(false). -spec fwd_peek(fwd_bit_reader(), integer()) -> {integer(), fwd_bit_reader()}. fwd_peek(Reader, Count) -> Refilled = fwd_refill(Reader, Count), Mask = erlang:'bsl'(1, Count) - 1, {erlang:'band'(erlang:element(3, Refilled), Mask), Refilled}. -file("src/packkit/internal/huf.gleam", 523). ?DOC(false). -spec fwd_drop(fwd_bit_reader(), integer()) -> fwd_bit_reader(). fwd_drop(Reader, Count) -> Refilled = fwd_refill(Reader, Count), case erlang:element(4, Refilled) >= Count of true -> {fwd_bit_reader, erlang:element(2, Refilled), erlang:'bsr'(erlang:element(3, Refilled), Count), erlang:element(4, Refilled) - Count, erlang:element(5, Refilled) + Count}; false -> Refilled end. -file("src/packkit/internal/huf.gleam", 537). ?DOC(false). -spec fwd_read(fwd_bit_reader(), integer()) -> {ok, {integer(), fwd_bit_reader()}} | {error, huf_error()}. fwd_read(Reader, Count) -> Refilled = fwd_refill(Reader, Count), case erlang:element(4, Refilled) >= Count of false -> {error, {huf_truncated, <<"huf: FSE distribution truncated"/utf8>>}}; true -> {Value, Peeked} = fwd_peek(Refilled, Count), {ok, {Value, fwd_drop(Peeked, Count)}} end. -file("src/packkit/internal/huf.gleam", 643). ?DOC(false). -spec split_distribution_count(integer(), integer(), integer(), integer()) -> {integer(), integer()}. split_distribution_count(Value, Threshold, Remaining, Bit_count) -> Max_val = ((2 * Threshold) - 1) - Remaining, case erlang:'band'(Value, Threshold - 1) < Max_val of true -> {erlang:'band'(Value, Threshold - 1), Bit_count - 1}; false -> Raw = erlang:'band'(Value, (2 * Threshold) - 1), Adjusted = case Raw >= Threshold of true -> Raw - Max_val; false -> Raw end, {Adjusted, Bit_count} end. -file("src/packkit/internal/huf.gleam", 663). ?DOC(false). -spec shrink_threshold(integer(), integer(), integer()) -> {integer(), integer()}. shrink_threshold(Threshold, Bit_count, Remaining) -> gleam@bool:guard( Remaining >= Threshold, {Threshold, Bit_count}, fun() -> gleam@bool:guard( Threshold =< 1, {Threshold, Bit_count}, fun() -> shrink_threshold(Threshold div 2, Bit_count - 1, Remaining) end ) end ). -file("src/packkit/internal/huf.gleam", 676). ?DOC(false). -spec read_zero_repeat(fwd_bit_reader(), integer()) -> {ok, {integer(), fwd_bit_reader()}} | {error, huf_error()}. read_zero_repeat(Reader, Acc) -> gleam@result:'try'( fwd_read(Reader, 2), fun(_use0) -> {Chunk, Reader@1} = _use0, case Chunk of 3 -> read_zero_repeat(Reader@1, Acc + 3); N -> {ok, {Acc + N, Reader@1}} end end ). -file("src/packkit/internal/huf.gleam", 687). ?DOC(false). -spec prepend_zeros(integer(), list(integer())) -> list(integer()). prepend_zeros(Count, Acc) -> case Count of 0 -> Acc; _ -> prepend_zeros(Count - 1, [0 | Acc]) end. -file("src/packkit/internal/huf.gleam", 590). ?DOC(false). -spec decode_distribution_counts( fwd_bit_reader(), integer(), integer(), integer(), boolean(), integer(), integer(), list(integer()) ) -> {ok, {list(integer()), fwd_bit_reader()}} | {error, huf_error()}. decode_distribution_counts( Reader, Remaining, Threshold, Bit_count, Previous_is_zero, Charnum, Max_symbol, Acc ) -> gleam@bool:guard( not ((Remaining > 1) andalso (Charnum =< Max_symbol)), {ok, {lists:reverse(Acc), Reader}}, fun() -> case Previous_is_zero of true -> gleam@result:'try'( read_zero_repeat(Reader, 0), fun(_use0) -> {Extra, Reader@1} = _use0, Acc@1 = prepend_zeros(Extra, Acc), decode_distribution_counts( Reader@1, Remaining, Threshold, Bit_count, false, Charnum + Extra, Max_symbol, Acc@1 ) end ); false -> {Value, Reader@2} = fwd_peek(Reader, Bit_count), {Count, Bits_consumed} = split_distribution_count( Value, Threshold, Remaining, Bit_count ), Reader@3 = fwd_drop(Reader@2, Bits_consumed), Probability = Count - 1, Abs_prob = gleam@int:absolute_value(Probability), Next_remaining = Remaining - Abs_prob, {Next_threshold, Next_bit_count} = shrink_threshold( Threshold, Bit_count, Next_remaining ), decode_distribution_counts( Reader@3, Next_remaining, Next_threshold, Next_bit_count, Probability =:= 0, Charnum + 1, Max_symbol, [Probability | Acc] ) end end ). -file("src/packkit/internal/huf.gleam", 702). ?DOC(false). -spec repeat_zeros(integer(), list(integer())) -> list(integer()). repeat_zeros(N, Acc) -> case N of 0 -> Acc; _ -> repeat_zeros(N - 1, [0 | Acc]) end. -file("src/packkit/internal/huf.gleam", 694). ?DOC(false). -spec pad_distribution(list(integer()), integer()) -> list(integer()). pad_distribution(Counts, Target) -> Current = erlang:length(Counts), case Current >= Target of true -> Counts; false -> lists:append(Counts, repeat_zeros(Target - Current, [])) end. -file("src/packkit/internal/huf.gleam", 551). ?DOC(false). -spec read_distribution(bitstring(), integer(), integer()) -> {ok, {list(integer()), integer(), bitstring()}} | {error, huf_error()}. read_distribution(Bytes, Max_accuracy_log, Max_symbol) -> Reader = new_fwd_reader(Bytes), gleam@result:'try'( fwd_read(Reader, 4), fun(_use0) -> {Low4, Reader@1} = _use0, Accuracy_log = Low4 + 5, case Accuracy_log > Max_accuracy_log of true -> {error, {huf_invalid_weights, <<"huf: FSE distribution accuracy_log too high"/utf8>>}}; false -> Table_size = erlang:'bsl'(1, Accuracy_log), gleam@result:'try'( decode_distribution_counts( Reader@1, Table_size + 1, Table_size, Accuracy_log + 1, false, 0, Max_symbol, [] ), fun(_use0@1) -> {Counts, Reader@2} = _use0@1, Bytes_consumed = (erlang:element(5, Reader@2) + 7) div 8, Total = erlang:byte_size(Bytes), case gleam_stdlib:bit_array_slice( Bytes, Bytes_consumed, Total - Bytes_consumed ) of {ok, Rest} -> {ok, {pad_distribution( Counts, Max_symbol + 1 ), Accuracy_log, Rest}}; {error, _} -> {error, {huf_truncated, <<"huf: FSE distribution tail truncated"/utf8>>}} end end ) end end ). -file("src/packkit/internal/huf.gleam", 433). ?DOC(false). -spec finish_or_continue_weight_pair( {ok, {integer(), integer(), packkit@internal@fse:backward_reader()}} | {error, packkit@internal@fse:fse_error()}, gleam@dict:dict(integer(), packkit@internal@fse:state_entry()), integer(), integer(), boolean(), list(integer()), integer() ) -> {ok, list(integer())} | {error, huf_error()}. finish_or_continue_weight_pair( Decode_outcome, Table, State_a, State_b, Use_a, Acc, Active_state ) -> case Decode_outcome of {error, fse_truncated} -> Partner = case Use_a of true -> State_b; false -> State_a end, Active_symbol = lookup_symbol(Table, Active_state), Partner_symbol = lookup_symbol(Table, Partner), {ok, lists:reverse([Partner_symbol, Active_symbol | Acc])}; {error, fse_empty_bitstream} -> Partner = case Use_a of true -> State_b; false -> State_a end, Active_symbol = lookup_symbol(Table, Active_state), Partner_symbol = lookup_symbol(Table, Partner), {ok, lists:reverse([Partner_symbol, Active_symbol | Acc])}; {ok, {Symbol, Next_state, New_reader}} -> {Next_a, Next_b} = case Use_a of true -> {Next_state, State_b}; false -> {State_a, Next_state} end, decode_fse_weight_pairs( Table, New_reader, Next_a, Next_b, not Use_a, [Symbol | Acc] ) end. -file("src/packkit/internal/huf.gleam", 410). ?DOC(false). -spec decode_fse_weight_pairs( gleam@dict:dict(integer(), packkit@internal@fse:state_entry()), packkit@internal@fse:backward_reader(), integer(), integer(), boolean(), list(integer()) ) -> {ok, list(integer())} | {error, huf_error()}. decode_fse_weight_pairs(Table, Reader, State_a, State_b, Use_a, Acc) -> Active_state = case Use_a of true -> State_a; false -> State_b end, _pipe = packkit@internal@fse:decode_state(Table, Active_state, Reader), finish_or_continue_weight_pair( _pipe, Table, State_a, State_b, Use_a, Acc, Active_state ). -file("src/packkit/internal/huf.gleam", 386). ?DOC(false). -spec decode_fse_weight_stream(bitstring()) -> {ok, list(integer())} | {error, huf_error()}. decode_fse_weight_stream(Bytes) -> gleam@result:'try'( read_distribution(Bytes, 6, 12), fun(_use0) -> {Counts, Accuracy_log, After_header} = _use0, Table = packkit@internal@fse:build_state_table(Counts, Accuracy_log), gleam@result:'try'( begin _pipe = packkit@internal@fse:new_backward_reader( After_header ), gleam@result:map_error( _pipe, fun(Field@0) -> {huf_bitstream_error, Field@0} end ) end, fun(Reader) -> gleam@result:'try'( begin _pipe@1 = packkit@internal@fse:read_backward_bits( Reader, Accuracy_log ), gleam@result:map_error( _pipe@1, fun(Field@0) -> {huf_bitstream_error, Field@0} end ) end, fun(_use0@1) -> {State_a, Reader@1} = _use0@1, gleam@result:'try'( begin _pipe@2 = packkit@internal@fse:read_backward_bits( Reader@1, Accuracy_log ), gleam@result:map_error( _pipe@2, fun(Field@0) -> {huf_bitstream_error, Field@0} end ) end, fun(_use0@2) -> {State_b, Reader@2} = _use0@2, decode_fse_weight_pairs( Table, Reader@2, State_a, State_b, true, [] ) end ) end ) end ) end ). -file("src/packkit/internal/huf.gleam", 365). ?DOC(false). -spec read_fse_weights(integer(), bitstring()) -> {ok, {tree(), integer()}} | {error, huf_error()}. read_fse_weights(I_size, Rest) -> case gleam_stdlib:bit_array_slice(Rest, 0, I_size) of {error, _} -> {error, {huf_truncated, <<"huf: FSE-weight body truncated"/utf8>>}}; {ok, Fse_bytes} -> gleam@result:'try'( decode_fse_weight_stream(Fse_bytes), fun(Weights) -> gleam@result:'try'( build_tree(Weights), fun(Tree) -> {ok, {Tree, 1 + I_size}} end ) end ) end. -file("src/packkit/internal/huf.gleam", 49). ?DOC(false). -spec read_tree(bitstring()) -> {ok, {tree(), integer()}} | {error, huf_error()}. read_tree(Bytes) -> case Bytes of <> -> case Header_byte >= 128 of true -> read_direct_weights(Header_byte, Rest); false -> read_fse_weights(Header_byte, Rest) end; _ -> {error, {huf_truncated, <<"huf: tree description header missing"/utf8>>}} end.