defmodule ConcatJSON.ParseError do @type t :: %__MODULE__{position: integer, line: integer} defexception [:position, :line, :token] def message( %{position: position, line: line, token: token} ) do "Unexpected token at position #{position}, line #{line}: " <> inspect_char(token) end defp inspect_char( char ) do str = <> if String.printable?(str) do "#{inspect char, base: :hex} ('#{str}')" else "#{inspect char, base: :hex}" end end end defmodule ConcatJSON do @moduledoc """ Documentation for ConcatJSON. """ require Logger alias ConcatJSON.{ParseError} @default_state %{pos: 0, line: 1, obj: 0, arr: 0, current: nil } @type state :: %{ pos: integer, line: integer, obj: integer, arr: integer, current: atom() | nil } @type input :: String.t() | list @type error :: {:error, ParseError.t() } @doc """ Returns all iex> ConcatJSON.parse('{"ok":true}') {:ok, ['{"ok":true}'] } """ @spec parse( input ) :: {:ok, list} | {:continue, list, String.t(), list, state } | error def parse( value ) do parse( value, [], @default_state ) end @doc """ ## Examples iex> ConcatJSON.parse!('[ "foo", "bar" ]') ['["foo","bar"]'] iex> ConcatJSON.parse!('not valid ]') ** (ConcatJSON.ParseError) Unexpected token at position 10, line 1: 0x5D (']') """ def parse!( value ) do case parse( value ) do {:ok, results} -> results {:continue, results, _input, _buffer, _state } -> results {:error, error} -> raise error end end @doc """ """ def parse( input, buffer, state, results \\ [] ) def parse( input, buffer, state, results ) when is_list(input), do: parse( to_string(input), buffer, state, results ) def parse( input, buffer, state, results ) do # Logger.debug("begin parse input #{String.length(input)} (#{input}) buffer (#{buffer})") parse_loop( input, Enum.reverse(buffer), state, results ) end defp parse_loop( input, buffer, state, results ) do case walk_parse(input, buffer, state) do {:ok, _input, [], _state } -> {:ok, results } {:ok, "", value, _state } -> {:ok, results ++ [ Enum.reverse(value)] } {:ok, input, value, state} -> # Logger.debug(":ok parse input #{String.length(input)} (#{input}) buffer (#{buffer})") parse_loop( input, [], state, results ++ [Enum.reverse(value)] ) {:continue, input, value, state } -> {:continue, results, input, Enum.reverse(value), state } other -> other end end @doc """ inspiration from https://stackoverflow.com/a/29408700/2377677 """ def stream(enum) do enum |> Stream.transform( {}, fn(val,acc) -> # Logger.debug "val: (#{val})" # IO.puts "acc: #{Jason.encode!(acc)}" # call parse depending on whether we are continuing parse_result = case acc do {:continue, buffer, opts} -> # Logger.debug "continue with (#{val}) (#{buffer})" ConcatJSON.parse(val, buffer, opts) _ -> ConcatJSON.parse( val ) end case parse_result do {:ok, results} -> # Logger.warn ":ok with #{length(results)} (#{results})" # parse came back with results, so return them {results,{}} {:continue, [], _input, buffer, opts } -> # Logger.warn "continue with empty results true" # the parse was incomplete, so store state in accum for continuing # on next read {[], {:continue, buffer, opts} } {:continue, results, _input, buffer, opts } -> # Logger.warn "continue with (#{results}) #{results == []}" # parse was incomplete, but came back with results, so emit # those results and keep the parse state in the buffer {results, {:continue, buffer, opts} } # {:error, _buffer, _input, _opts, msg } -> # for now we just let errors continue # Logger.warn "error: #{msg}" # {[], {:error, msg}} end end) end @doc """ Takes an enumeration and parses all of the values """ def reduce(enum) do { result, _acc } = Enum.reduce(enum, { [], {} }, fn value, { result, acc } -> parse_result = case acc do {:continue, buffer, opts } -> ConcatJSON.parse(value, buffer, opts) _ -> ConcatJSON.parse(value) end case parse_result do {:ok, results} -> { result ++ results, {} } {:continue, [], _input, buffer, opts } -> {result, {:continue, buffer, opts} } {:continue, results, _input, buffer, opts } -> {result ++ results, {:continue, buffer, opts} } end end) result end defp walk_parse(binary, acc, state) defp walk_parse(binary, acc, state) when is_list(binary) do walk_parse(to_string(binary), acc, state) end defp walk_parse(binary, acc, state) when binary == "" do # Logger.debug "walk_parse 4 - binary end (#{binary}) (#{acc})" # raise "STAHP" {:ok, binary, acc, state} end # defp walk_parse(binary, acc, state) do # walk_parse(binary, Enum.reverse(to_charlist(acc)), state) # end defp walk_parse(<<"#", rest::binary>>, acc, %{pos: position} = state) do case walk_parse_comment(rest, acc, %{state | pos: position + 1, current: :comment}) do {:ok, rest, acc, state} -> walk_parse(rest, acc, state) {:continue, rest, acc, state} -> {:continue, rest, acc, state} end end defp walk_parse(<<"//", rest::binary>>, acc, %{pos: position} = state) do case walk_parse_comment(rest, acc, %{state | pos: position + 2, current: :comment}) do {:ok, rest, acc, state} -> # Logger.debug "walk_parse 5 // :ok - (#{rest}) (#{acc})" walk_parse(rest, acc, state) {:continue, rest, acc, state} -> # Logger.debug "walk_parse 5 // :continue" {:continue, rest, acc, state} end end defp walk_parse(<<"[", rest::binary>>, acc, %{pos: pos} = state) do # Logger.debug("> walk_parse 6 '#{acc}' '[#{rest}'") case walk_parse_array(rest, '[' ++ acc, %{state | pos: pos + 1, arr: 1, current: :array}) do {:ok, rest, acc, state} -> {:ok, rest, acc, %{state|current: nil}} {:continue, rest, acc, state} -> {:continue, rest, acc, state} end end defp walk_parse(<<"{", rest::binary>>, acc, %{pos: pos} = state) do # Logger.debug("walk_parse 7 (#{'{' ++ acc}) ({#{rest})") case walk_parse_obj(rest, '{' ++ acc, %{state | pos: pos + 1, obj: 1, current: :obj}) do {:ok, rest, acc, state} -> {:ok, rest, acc, %{state|current: nil}} {:continue, rest, acc, state} -> {:continue, rest, acc, state} end end # continues a partial parse of an obj, array, or string defp walk_parse(<>, acc, %{pos: pos, obj: obj, arr: arr, current: current} = state) when obj > 0 or arr > 0 or current == :string or current == :value do acc = Enum.reverse(acc) # Logger.debug("> walk_parse 7a (#{to_string(acc) <> rest})") case walk_parse(to_string(acc) <> rest, [], %{ state | pos: pos - length(acc), obj: 0, arr: 0, current: nil }) do {:ok, rest, acc, state} -> {:ok, rest, acc, state} {:continue, rest, acc, state} -> {:continue, rest, acc, state} end end # continues a partial parse of a comment defp walk_parse(<>, acc, %{pos: pos, current: current} = state) when current == :comment do acc = Enum.reverse(acc) # Logger.debug("> walk_parse 7b '#{to_string(acc) <> rest}'") case walk_parse(to_string(acc) <> "#" <> rest, [], %{ state | pos: pos - length(acc), current: nil }) do {:ok, rest, acc, state} -> {:ok, rest, acc, state} {:continue, rest, acc, state} -> {:continue, rest, acc, state} end end defp walk_parse(<>, _acc, %{pos: pos, line: line}) when ch == ?] or ch == ?} do {:error, %ParseError{position: pos, line: line, token: ch}} end # begin parsing quote defp walk_parse(<<"\"", rest::binary>>, acc, %{pos: pos} = state) do case walk_parse_quote(rest, '\"' ++ acc, %{state | pos: pos + 1, current: :string}) do {:ok, rest, acc, state} -> {:ok, rest, acc, %{state|current: nil}} {code, rest, acc, state} -> {code, rest, acc, state} end end # newline - inc line number defp walk_parse(<<"\n", rest::binary>>, acc, %{line: line} = state) do # Logger.debug "walk_parse 5 (#{acc}) rest:(#{rest})" walk_parse(rest, acc, %{state | pos: 0, line: line + 1}) end defp walk_parse(<<" ", rest::binary>>, acc, %{pos: pos} = state) do # Logger.debug "walk_parse space #{acc}" walk_parse(rest, acc, %{state | pos: pos + 1}) end # pass everything defp walk_parse(<>, acc, %{pos: pos} = state) do # Logger.debug "walk_parse char:(#{to_string([char])})" case walk_parse_value(rest, [char] ++ acc, %{state| pos: pos + 1, current: :value}) do {:ok, rest, acc, state} -> {:ok, rest, acc, %{state|current: nil}} {code, rest, acc, state} -> {code, rest, acc, state} end # walk_parse(rest, [char] ++ acc, %{state | pos: pos + 1}) end defp walk_parse_comment(<<"\n", rest::binary>>, acc, %{line: line} = state) do # Logger.debug("walk_parse_comment 1 - :ok") {:ok, rest, acc, %{state | line: line + 1, pos: 0, current: nil}} end defp walk_parse_comment(<<_char, rest::binary>>, acc, %{pos: pos} = state) do # Logger.debug("walk_parse_comment 2 ") walk_parse_comment(rest, acc, %{state | pos: pos + 1}) end defp walk_parse_comment(binary, acc, state) when binary == "" do # Logger.debug("walk_parse_comment 3 - :ok") {:continue, binary, acc, state} end defp walk_parse_quote(<<(~s(\\")), rest::binary>>, acc, %{pos: pos} = state) do # Logger.debug("walk_parse_quote 1 #{acc} char:(#{~s(\\")})") walk_parse_quote(rest, '\"' ++ acc, %{state | pos: pos + 1}) end defp walk_parse_quote(<<(~s(")), rest::binary>>, acc, %{pos: pos} = state) do # Logger.debug("walk_parse_quote 2 #{acc} - fin") {:ok, rest, '"' ++ acc, %{state | pos: pos + 1}} end defp walk_parse_quote(<>, acc, %{pos: pos} = state) do # Logger.debug("walk_parse_quote 3 (#{acc}) ++ (#{[char]})") walk_parse_quote(rest, [char] ++ acc, %{state | pos: pos + 1}) end defp walk_parse_quote(binary, acc, state) when binary == "" do # Logger.debug("walk_parse_quote 4 (#{acc}) (#{binary})") {:continue, binary, acc, state} end # close array - finish if count ends, continue otherwise defp walk_parse_array(<<"]", rest::binary>>, acc, %{pos: pos, arr: arr} = state) do # Logger.debug("walk_parse_array 2 #{acc} arr:#{arr}") acc = ']' ++ acc state = %{state | pos: pos + 1, arr: arr - 1} case arr do 1 -> {:ok, rest, acc, state} _ -> walk_parse_array(rest, acc, state) end end defp walk_parse_array(<<"[", rest::binary>>, acc, %{pos: pos, arr: arr} = state) do # Logger.debug("walk_parse_array 4 #{acc} arr: #{arr + 1}") walk_parse_array(rest, '[' ++ acc, %{state | pos: pos + 1, arr: arr + 1}) end defp walk_parse_array(<<"\"", rest::binary>>, acc, %{pos: pos} = state) do # Logger.debug("walk_parse_array 5 #{acc}") case walk_parse_quote(rest, '"' ++ acc, %{state | pos: pos + 1, current: :string}) do {:ok, rest, acc, state} -> walk_parse_array(rest, acc, %{state| current: :array}) {:continue, rest, acc, state} -> {:continue, rest, acc, state} end end defp walk_parse_array(<<"#", rest::binary>>, acc, %{pos: pos} = state) do case walk_parse_comment(rest, acc, %{state | pos: pos + 1, current: :comment}) do {:ok, rest, acc, state} -> walk_parse_array(rest, acc, %{state|current: :array}) {:continue, rest, acc, state} -> {:continue, rest, acc, state} end end defp walk_parse_array(<<"//", rest::binary>>, acc, %{pos: position} = state) do case walk_parse_comment(rest, acc, %{state | pos: position + 2, current: :comment}) do {:ok, rest, acc, state} -> walk_parse_array(rest, acc, %{state|current: :array}) {:continue, rest, acc, state} -> {:continue, rest, acc, state} end end defp walk_parse_array(<<"\n", rest::binary>>, acc, %{line: line} = state) do # Logger.debug("walk_parse_array 6 #{acc}") walk_parse_array(rest, acc, %{state | pos: 0, line: line + 1}) end defp walk_parse_array(<<" ", rest::binary>>, acc, %{pos: pos} = state) do # Logger.debug "parse space #{acc}" walk_parse_array(rest, acc, %{state | pos: pos + 1}) end # everything else defp walk_parse_array(<>, acc, %{pos: pos, arr: _arr} = state) do # Logger.debug("walk_parse_array 7 (#{acc}) '#{to_string([char])}' arr:#{arr}") walk_parse_array(rest, [char] ++ acc, %{state | pos: pos + 1}) end # end of input defp walk_parse_array(binary, acc, state) when binary == "" do # Logger.debug("walk_parse_array 8 #{acc}") {:continue, binary, acc, state} end defp walk_parse_obj(<<"}", rest::binary>>, acc, %{pos: pos, obj: obj} = state) when obj == 1 do acc = '}' ++ acc # Logger.debug("walk_parse_obj 2 (#{acc}) obj:#{obj} - :ok") {:ok, rest, acc, %{state | pos: pos + 1, obj: obj - 1}} end defp walk_parse_obj(<<"}", rest::binary>>, acc, %{pos: pos, obj: obj} = state) do # Logger.debug("walk_parse_obj 3 #{acc} obj:#{obj}") walk_parse_obj(rest, '}' ++ acc, %{state | pos: pos + 1, obj: obj - 1}) end defp walk_parse_obj(<<"#", rest::binary>>, acc, %{pos: pos} = state) do case walk_parse_comment(rest, acc, %{state | pos: pos + 1, current: :comment}) do {:ok, rest, acc, state} -> walk_parse_obj(rest, acc, state) {:continue, rest, acc, state} -> {:continue, rest, acc, state} end end defp walk_parse_obj(<<"//", rest::binary>>, acc, %{pos: position} = state) do case walk_parse_comment(rest, acc, %{state | pos: position + 2, current: :comment}) do {:ok, rest, acc, state} -> walk_parse_obj(rest, acc, state) {:continue, rest, acc, state} -> {:continue, rest, acc, state} end end defp walk_parse_obj(<<"{", rest::binary>>, acc, %{pos: pos, obj: obj} = state) do # Logger.debug("walk_parse_obj 4 #{acc} obj: #{obj + 1}") walk_parse_obj(rest, '{' ++ acc, %{state | pos: pos + 1, obj: obj + 1}) end defp walk_parse_obj(<<"\"", rest::binary>>, acc, %{pos: pos} = state) do # Logger.debug("walk_parse_obj 5 (#{acc})") case walk_parse_quote(rest, '"' ++ acc, %{state | pos: pos + 1, current: :string}) do {:ok, rest, acc, state} -> walk_parse_obj(rest, acc, state) {:continue, rest, acc, state} -> {:continue, rest, acc, state} end end defp walk_parse_obj(<<"\n", rest::binary>>, acc, %{line: line} = state) do # Logger.debug("walk_parse_obj 6 #{acc}") walk_parse_obj(rest, acc, %{state | pos: 0, line: line + 1}) end defp walk_parse_obj(<<" ", rest::binary>>, acc, %{pos: pos} = state) do # Logger.debug "parse space #{acc}" walk_parse_obj(rest, acc, %{state | pos: pos + 1}) end defp walk_parse_obj(<>, acc, %{pos: pos, obj: _obj } = state) do # Logger.debug("walk_parse_obj 7 (#{acc}) '#{to_string([char])}' obj:#{obj}") walk_parse_obj(rest, [char] ++ acc, %{state | pos: pos + 1}) end defp walk_parse_obj("", acc, state) do # Logger.debug("walk_parse_obj 8 #{acc}") {:continue, "", acc, state} end defp walk_parse_value("", acc, state) do # Logger.debug("walk_parse_value :continue") {:continue, "", acc, state} end # end of input defp walk_parse_value(<>, acc, %{pos: pos} = state) when char == ?\n or char == 32 do # Logger.debug "walk_parse_value newline/space :ok" {:ok, rest, acc, %{state | pos: pos + 1}} end # space indicates end of value # defp walk_parse_value(<<" ", rest::binary>>, acc, %{pos: pos} = state) do # # Logger.debug "walk_parse_value :ok" # {:ok, rest, acc, %{state | pos: pos + 1}} # end # regular character defp walk_parse_value(<>, acc, %{pos: pos} = state) do # Logger.debug "walk_parse_value char:(#{to_string([char])})" walk_parse_value(rest, [char] ++ acc, %{state | pos: pos + 1}) end # def finish_ok(acc, rest, state) do # # IO.inspect( acc ) # {:ok, to_string(Enum.reverse(acc)), to_string(rest), state} # end # def finish_continue(acc, rest, state) do # {:continue, to_string(Enum.reverse(acc)), to_string(rest), state} # end end