%%%------------------------------------------------------------------- %%% @doc loki_xml_splitter - Splits XML into parseable chunks %%% @end %%%------------------------------------------------------------------- -module(loki_xml_splitter). -export([ split_by_tag/2, split_fixed/2, validate_chunk/1 ]). %%%=================================================================== %%% API Functions %%%=================================================================== %% @doc Split XML by specific tag using regex -spec split_by_tag(binary(), binary()) -> {ok, [binary()]} | {error, term()}. split_by_tag(Binary, Tag) -> %% Build regex pattern: ]*>.*? %% Use simple pattern without capturing groups for better compatibility OpenTag = <<"<", Tag/binary>>, CloseTag = <<"">>, case find_tag_chunks(Binary, OpenTag, CloseTag, []) of [] -> %% Fallback to fixed-size splitting io:format("No matches for tag ~p, falling back to fixed-size~n", [Tag]), split_fixed(Binary, 1048576); Chunks -> ValidChunks = lists:filter(fun validate_chunk/1, Chunks), {ok, ValidChunks} end. %% Helper to find chunks by tag find_tag_chunks(<<>>, _OpenTag, _CloseTag, Acc) -> lists:reverse(Acc); find_tag_chunks(Binary, OpenTag, CloseTag, Acc) -> case binary:split(Binary, OpenTag) of [_Before, Rest] -> case binary:split(Rest, CloseTag) of [Content, Remaining] -> Chunk = <>, find_tag_chunks(Remaining, OpenTag, CloseTag, [Chunk | Acc]); [_] -> lists:reverse(Acc) end; [_] -> lists:reverse(Acc) end. %% @doc Split XML into fixed-size chunks with boundary reconstruction -spec split_fixed(binary(), pos_integer()) -> {ok, [binary()]}. split_fixed(Binary, ChunkSize) when byte_size(Binary) =< ChunkSize -> {ok, [Binary]}; split_fixed(Binary, ChunkSize) -> Chunks = split_fixed_impl(Binary, ChunkSize, []), ReconstructedChunks = reconstruct_boundaries(Chunks), {ok, ReconstructedChunks}. %% @doc Validate that a chunk is well-formed XML -spec validate_chunk(binary()) -> boolean(). validate_chunk(<<>>) -> false; validate_chunk(Chunk) -> %% Simple validation: check for balanced opening and closing tags OpenCount = count_open_tags(Chunk), CloseCount = count_close_tags(Chunk), OpenCount == CloseCount andalso OpenCount > 0. count_open_tags(Binary) -> %% Count opening tags, excluding self-closing and special tags Parts = binary:split(Binary, <<"<">>, [global]), lists:foldl(fun(Part, Count) -> case Part of <<>> -> Count; <<"/", _/binary>> -> Count; %% Closing tag <<"!", _/binary>> -> Count; %% Comment/CDATA <<"?", _/binary>> -> Count; %% Processing instruction _ -> %% Check if it's a self-closing tag (ends with />) case binary:match(Part, <<"/>">>) of {Pos, _} when Pos < byte_size(Part) -> Count; %% Self-closing _ -> Count + 1 %% Regular opening tag end end end, 0, Parts). count_close_tags(Binary) -> Matches = binary:matches(Binary, <<">), length(Matches). %%%=================================================================== %%% Internal Functions %%%=================================================================== split_fixed_impl(<<>>, _, Acc) -> lists:reverse(Acc); split_fixed_impl(Binary, ChunkSize, Acc) when byte_size(Binary) =< ChunkSize -> lists:reverse([Binary | Acc]); split_fixed_impl(Binary, ChunkSize, Acc) -> %% Try to split at tag boundary near ChunkSize case find_tag_boundary(Binary, ChunkSize) of {ok, SplitPos} -> <> = Binary, split_fixed_impl(Rest, ChunkSize, [Chunk | Acc]); not_found -> %% Force split at ChunkSize <> = Binary, split_fixed_impl(Rest, ChunkSize, [Chunk | Acc]) end. %% @doc Find nearest tag boundary near target position find_tag_boundary(Binary, TargetPos) -> SearchStart = max(0, TargetPos - 1000), SearchEnd = min(byte_size(Binary), TargetPos + 1000), SearchSize = SearchEnd - SearchStart, <<_:SearchStart/binary, SearchRegion:SearchSize/binary, _/binary>> = Binary, %% Look for closing tag pattern case binary:matches(SearchRegion, [<<">">>]) of [] -> not_found; Matches -> %% Find closest match to target Closest = find_closest_match(Matches, TargetPos - SearchStart), {ok, SearchStart + Closest + 1} end. find_closest_match(Matches, Target) -> [{Pos, _} | _] = lists:sort(fun({P1, _}, {P2, _}) -> abs(P1 - Target) =< abs(P2 - Target) end, Matches), Pos. %% @doc Reconstruct tag boundaries across chunks reconstruct_boundaries([]) -> []; reconstruct_boundaries([Chunk]) -> [Chunk]; reconstruct_boundaries([C1, C2 | Rest]) -> case has_open_tag_at_end(C1) of true -> %% Merge with next chunk reconstruct_boundaries([<> | Rest]); false -> [C1 | reconstruct_boundaries([C2 | Rest])] end. has_open_tag_at_end(Binary) -> Size = byte_size(Binary), CheckSize = min(100, Size), <<_:(Size-CheckSize)/binary, Tail:CheckSize/binary>> = Binary, %% Count < and > in tail OpenCount = length(binary:matches(Tail, <<"<">>)), CloseCount = length(binary:matches(Tail, <<">">>)), OpenCount > CloseCount.