%% Copyright (c) 2009-2013 %% Nick Gerakines %% Jacob Vorreuter %% %% Permission is hereby granted, free of charge, to any person %% obtaining a copy of this software and associated documentation %% files (the "Software"), to deal in the Software without %% restriction, including without limitation the rights to use, %% copy, modify, merge, publish, distribute, sublicense, and/or sell %% copies of the Software, and to permit persons to whom the %% Software is furnished to do so, subject to the following %% conditions: %% %% The above copyright notice and this permission notice shall be %% included in all copies or substantial portions of the Software. %% %% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, %% EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES %% OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND %% NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT %% HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, %% WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING %% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR %% OTHER DEALINGS IN THE SOFTWARE. -module(protobuffs_compile). -ifdef(TEST). -compile(export_all). -else. -export([scan_file/1, scan_file/2, scan_string/2, scan_string/3, generate_source/1, generate_source/2]). -endif. -record(collected,{enum=[], msg=[], extensions=[], package}). %%-------------------------------------------------------------------- %% @doc Generats a built .beam file and header file .hrl %%-------------------------------------------------------------------- -spec scan_file(ProtoFile :: string()) -> ok | {error, _}. scan_file(ProtoFile) -> scan_file(ProtoFile,[]). -spec scan_string(String :: string(), BaseName :: string()) -> ok | {error, _}. scan_string(String,BaseName) -> scan_string(String,BaseName,[]). %%-------------------------------------------------------------------- %% @doc Generats a built .beam file and header file .hrl %% Considerd option properties: output_include_dir, %% output_ebin_dir, %% imports_dir %%-------------------------------------------------------------------- -spec scan_file(ProtoFile :: string() | atom(), Options :: list()) -> ok | {error, _}. scan_file(ProtoFile,Options) when is_list(ProtoFile) -> Basename = filename:basename(ProtoFile, ".proto") ++ "_pb", {ok,String} = parse_file(ProtoFile), scan_string(String,Basename,Options); scan_file(ProtoFile,Options) when is_atom(ProtoFile) -> Basename = atom_to_list(ProtoFile) ++ "_pb", {ok,String} = parse_file(atom_to_list(ProtoFile) ++ ".proto"), scan_string(String,Basename,Options). -spec scan_string(String :: string(), Basename :: string(), Options :: list()) -> ok | {error, _}. scan_string(String,Basename,Options) -> {ok,FirstParsed} = parse_string(String), ImportPaths = ["./", "src/" | proplists:get_value(imports_dir, Options, [])], Parsed = parse_imports(FirstParsed, ImportPaths), Collected = collect_full_messages(Parsed), Messages = resolve_types(Collected#collected.msg,Collected#collected.enum), output(Basename, Messages, Collected#collected.enum, Options). %%-------------------------------------------------------------------- %% @doc Generats a source .erl file and header file .hrl %%-------------------------------------------------------------------- -spec generate_source(ProtoFile :: string() | atom() ) -> ok | {error, _}. generate_source(ProtoFile) -> generate_source(ProtoFile,[]). %%-------------------------------------------------------------------- %% @doc Generats a source .erl file and header file .hrl %% Consider option properties: output_include_dir, %% output_src_dir, %% imports_dir %%-------------------------------------------------------------------- -spec generate_source(ProtoFile :: string() | atom(), Options :: list()) -> ok | {error, _}. generate_source(ProtoFile,Options) when is_atom(ProtoFile) -> generate_source(atom_to_list(ProtoFile) ++ ".proto", Options); generate_source(ProtoFile,Options) when is_list(ProtoFile) -> Basename = filename:basename(ProtoFile, ".proto") ++ "_pb", {ok,String} = parse_file(ProtoFile), {ok,FirstParsed} = parse_string(String), ImportPaths = ["./", "src/" | proplists:get_value(imports_dir, Options, [])], Parsed = parse_imports(FirstParsed, ImportPaths), Collected = collect_full_messages(Parsed), Messages = resolve_types(Collected#collected.msg,Collected#collected.enum), output_source(Basename, Messages, Collected#collected.enum, Options). %% @hidden parse_imports(Parsed, Path) -> parse_imports(Parsed, Path, []). %% @hidden parse_imports([], _Path, Acc) -> lists:reverse(Acc); parse_imports([{import, File} = Head | Tail], Path, Acc) -> case protobuffs_file:path_open(Path, File, [read]) of {ok, F, Fullname} -> file:close(F), {ok,String} = parse_file(Fullname), {ok,FirstParsed} = parse_string(String), Parsed = lists:append(FirstParsed, [file_boundary | Tail]), parse_imports(Parsed, Path, [Head | Acc]); {error, Error} -> error_logger:error_report([ "Could not do import", {import, File}, {error, Error}, {path, Path} ]), parse_imports(Tail, Path, [Head | Acc]) end; parse_imports([Head | Tail], Path, Acc) -> parse_imports(Tail, Path, [Head | Acc]). %% @hidden output(Basename, MessagesRaw, RawEnums, Options) -> Messages = canonize_names(MessagesRaw), Enums = canonize_names(RawEnums), HeaderFile = case proplists:get_value(output_include_dir,Options) of undefined -> Basename ++ ".hrl"; HeaderPath -> filename:join(HeaderPath,Basename) ++ ".hrl" end, error_logger:info_msg("Writing header file to ~p~n",[HeaderFile]), ok = write_header_include_file(HeaderFile, Messages), {pokemon_pb, Binary, _} = code:get_object_code(pokemon_pb), {ok,{_,[{abstract_code,{_,Forms}}]}} = beam_lib:chunks(Binary, [abstract_code]), Forms1 = filter_forms(Messages, Enums, Forms, Basename, []), {ok, _, Bytes, _Warnings} = protobuffs_file:compile_forms(Forms1, proplists:get_value(compile_flags,Options,[])), BeamFile = case proplists:get_value(output_ebin_dir,Options) of undefined -> Basename ++ ".beam"; BeamPath -> filename:join(BeamPath,Basename) ++ ".beam" end, error_logger:info_msg("Writing beam file to ~p~n",[BeamFile]), protobuffs_file:write_file(BeamFile, Bytes). %% @hidden output_source(Basename, MessagesRaw, Enums, Options) -> Messages = canonize_names(MessagesRaw), HeaderFile = case proplists:get_value(output_include_dir,Options) of undefined -> Basename ++ ".hrl"; HeaderPath -> filename:join(HeaderPath,Basename) ++ ".hrl" end, error_logger:info_msg("Writing header file to ~p~n",[HeaderFile]), ok = write_header_include_file(HeaderFile, Messages), {pokemon_pb, Binary, _} = code:get_object_code(pokemon_pb), {ok,{_,[{abstract_code,{_,Forms}}]}} = beam_lib:chunks(Binary, [abstract_code]), Forms1 = filter_forms(Messages, Enums, Forms, Basename, []), SrcFile = case proplists:get_value(output_src_dir,Options) of undefined -> Basename ++ ".erl"; SrcPath -> filename:join(SrcPath,Basename) ++ ".erl" end, error_logger:info_msg("Writing src file to ~p~n",[SrcFile]), protobuffs_file:write_file(SrcFile, erl_prettypr:format(erl_syntax:form_list(Forms1))). %% @hidden parse_file(FileName) -> {ok, InFile} = protobuffs_file:open(FileName, [read]), String = parse_file(InFile,[]), file:close(InFile), {ok,String}. %% @hidden parse_file(InFile,Acc) -> case protobuffs_file:request(InFile) of {ok,Token,_EndLine} -> parse_file(InFile,Acc ++ [Token]); {error,token} -> exit(scanning_error); {eof,_} -> Acc end. parse_string(String) -> case lists:all(fun erlang:is_integer/1, String) of true -> {ok, Tokens, _Line} = protobuffs_scanner:string(String), protobuffs_parser:parse(Tokens); false -> protobuffs_parser:parse(String) end. %% @hidden filter_forms(Msgs, Enums, [{attribute,L,file,{_,_}}|Tail], Basename, Acc) -> filter_forms(Msgs, Enums, Tail, Basename, [{attribute,L,file,{"src/" ++ Basename ++ ".erl",L}}|Acc]); filter_forms(Msgs, Enums, [{attribute,L,module,pokemon_pb}|Tail], Basename, Acc) -> filter_forms(Msgs, Enums, Tail, Basename, [{attribute,L,module,list_to_atom(Basename)}|Acc]); filter_forms(Msgs, Enums, [{attribute,L,export,[{encode_pikachu,1},{decode_pikachu,1},{delimited_decode_pikachu,1}]}|Tail], Basename, Acc) -> Exports = lists:foldl( fun({Name,_,_}, Acc1) -> [{list_to_atom("encode_" ++ string:to_lower(Name)),1}, {list_to_atom("decode_" ++ string:to_lower(Name)),1}, {list_to_atom("delimited_decode_" ++ string:to_lower(Name)),1} | Acc1] end, [], Msgs), filter_forms(Msgs, Enums, Tail, Basename, [{attribute,L,export,Exports}|Acc]); filter_forms(Msgs, Enums, [{attribute,L,record,{pikachu,_}}|Tail], Basename, Acc) -> Records = [begin OutFields = [string:to_lower(A) || {_, _, _, A, _} <- lists:keysort(1, Fields)], ExtendField = case Extends of disallowed -> []; _ -> [{record_field,L,{atom,L,'$extensions'}}] end, Frm_Fields = [{record_field,L,{atom,L,list_to_atom(OutField)}}|| OutField <- OutFields] ++ ExtendField, {attribute, L, record, {atomize(Name), Frm_Fields}} end || {Name, Fields,Extends} <- Msgs], filter_forms(Msgs, Enums, Tail, Basename, Records ++ Acc); filter_forms(Msgs, Enums, [{function,L,encode_pikachu,1,[ListClause, RecordClause]}|Tail], Basename, Acc) -> Functions = [begin {function,L,list_to_atom("encode_" ++ string:to_lower(Name)),1,[replace_atom(ListClause, pikachu, atomize(Name)), replace_atom(RecordClause, pikachu, atomize(Name))]} end || {Name, _, _} <- Msgs], filter_forms(Msgs, Enums, Tail, Basename, Functions ++ Acc); filter_forms(Msgs, Enums, [{function,L,encode,2,[ListClause, RecordClause]}|Tail], Basename, Acc) -> filter_forms(Msgs, Enums, Tail, Basename, [expand_encode_function(Msgs, L, ListClause, RecordClause)|Acc]); filter_forms(Msgs, Enums, [{function,L,encode_extensions,1,[EncodeClause,Catchall]}|Tail], Basename, Acc) -> NewEncodeClauses = [replace_atom(EncodeClause, pikachu, atomize(Name)) || {Name, _Fields, Extens} <- Msgs, Extens =/= disallowed], NewClauses = NewEncodeClauses ++ [Catchall], NewFunction = {function,L,encode_extensions,1,NewClauses}, filter_forms(Msgs, Enums, Tail, Basename, [NewFunction | Acc]); filter_forms(Msgs, Enums, [{function,L,iolist,2,[Clause]}|Tail], Basename, Acc) -> filter_forms(Msgs, Enums, Tail, Basename, [expand_iolist_function(Msgs, L, Clause)|Acc]); filter_forms(Msgs, Enums, [{function,L,decode_pikachu,1,[Clause]}|Tail], Basename, Acc) -> Functions = [begin {function, L, list_to_atom("decode_" ++ string:to_lower(Name)), 1, [replace_atom(Clause, pikachu, atomize(Name))]} end || {Name, _, _} <- Msgs], filter_forms(Msgs, Enums, Tail, Basename, Functions ++ Acc); filter_forms(Msgs, Enums, [{function,L,delimited_decode_pikachu,1,[Clause]} | Tail], Basename, Acc) -> Acc2 = lists:foldl(fun({Name, _, _}, Acc1) -> Clause2 = replace_atom(Clause, pikachu, atomize(Name)), Function = {function,L,list_to_atom("delimited_decode_" ++ string:to_lower(Name)),1,[Clause2]}, [Function | Acc1] end, Acc, Msgs), filter_forms(Msgs, Enums, Tail, Basename, Acc2); filter_forms(Msgs, Enums, [{function,L,decode,2,[Clause]}|Tail], Basename, Acc) -> filter_forms(Msgs, Enums, Tail, Basename, [expand_decode_function(Msgs, L, Clause)|Acc]); filter_forms(Msgs, Enums, [{function,L,to_record,2,[Clause]}|Tail], Basename, Acc) -> filter_forms(Msgs, Enums, Tail, Basename, [expand_to_record_function(Msgs, L, Clause)|Acc]); filter_forms(Msgs, Enums, [{function,L,enum_to_int,2,[Clause]}|Tail], Basename, Acc) -> filter_forms(Msgs, Enums, Tail, Basename, [expand_enum_to_int_function(Enums, L, Clause)|Acc]); filter_forms(Msgs, Enums, [{function,L,int_to_enum,2,[Clause]}|Tail], Basename, Acc) -> filter_forms(Msgs, Enums, Tail, Basename, [expand_int_to_enum_function(Enums, L, Clause)|Acc]); filter_forms(Msgs, Enums, [{function,L,decode_extensions,1,[Clause,Catchall]}|Tail],Basename, Acc) -> NewClauses = filter_decode_extensions_clause(Msgs, Msgs, Clause, []), NewHead = {function,L,decode_extensions,1,NewClauses ++ [Catchall]}, filter_forms(Msgs, Enums, Tail,Basename,[NewHead|Acc]); filter_forms(Msgs, Enums, [{function,L,extension_size,1,[RecClause,CatchAll]}|Tail],Basename, Acc) -> NewRecClauses = filter_extension_size(Msgs, RecClause, []), NewClauses = lists:reverse([CatchAll | NewRecClauses]), NewHead = {function,L,extension_size,1,NewClauses}, filter_forms(Msgs, Enums, Tail, Basename, [NewHead|Acc]); filter_forms(Msgs, Enums, [{function,L,has_extension,2,[FilterClause,CatchallClause]}|Tail],Basename,Acc) -> NewRecClauses = filter_has_extension(Msgs, FilterClause, []), NewClauses = lists:reverse([CatchallClause | NewRecClauses]), NewHead = {function,L,has_extension,2,NewClauses}, filter_forms(Msgs, Enums, Tail, Basename, [NewHead | Acc]); filter_forms(Msgs, Enums, [{function,L,get_extension,2,[AtomClause,IntClause,Catchall]}|Tail],Basename,Acc) -> NewAtomClauses = filter_get_extension_atom(Msgs,AtomClause,[]), NewRecClauses = filter_get_extension_integer(Msgs, IntClause, NewAtomClauses), NewClauses = lists:reverse([Catchall | NewRecClauses]), NewHead = {function,L,get_extension,2,NewClauses}, filter_forms(Msgs,Enums, Tail, Basename, [NewHead | Acc]); filter_forms(Msgs, Enums, [{function,L,set_extension,3,[RecClause,Catchall]}|Tail],Basename, Acc) -> NewRecClauses = filter_set_extension(Msgs, RecClause, []), NewClauses = lists:reverse([Catchall | NewRecClauses]), NewHead = {function,L,set_extension,3,NewClauses}, filter_forms(Msgs,Enums,Tail,Basename,[NewHead|Acc]); filter_forms(Msgs, Enums, [{function,_L,with_default,2,_Args}=Func|Tail],Basename,Acc) -> Acc2 = case any_message_has_fields(Msgs) of true -> [Func|Acc]; false -> Acc end, filter_forms(Msgs,Enums,Tail,Basename,Acc2); filter_forms(Msgs, Enums, [{function,_L,pack,5,_Clauses}=Func|Tail],Basename,Acc) -> Acc2 = case any_message_has_fields(Msgs) orelse any_message_has_extentions(Msgs) of true -> [Func|Acc]; false -> Acc end, filter_forms(Msgs,Enums,Tail,Basename,Acc2); filter_forms(Msgs, Enums, [Form|Tail], Basename, Acc) -> filter_forms(Msgs, Enums, Tail, Basename, [Form|Acc]); filter_forms(_, _, [], _, Acc) -> lists:reverse(Acc). any_message_has_extentions(Msgs) -> Predicate = fun({_,_,disallowed}) -> false; (_) -> true end, lists:any(Predicate, Msgs). any_message_has_fields(Msgs) -> Predicate = fun({_,[],_}) -> false; (_) -> true end, lists:any(Predicate, Msgs). %% @hidden filter_set_extension([],_,Acc) -> Acc; filter_set_extension([{_,_,disallowed}|Tail],Clause,Acc) -> filter_set_extension(Tail,Clause,Acc); filter_set_extension([{MsgName,_,Extends}|Tail],Clause,Acc) -> {clause,L,[OldRecArg,_OldAtomArg,ValueArg],Gs,[OldSet,OldReturn]} = Clause, {match,L,{record,L,_,RecArgFields},RecVar} = OldRecArg, {match,L2,NewReturn,OldDictStore} = OldSet, {call,L2,DictStore,[_StoreKey,_StoreVal,StoreVar]} = OldDictStore, {tuple,L3,[Ok, OldReturnRec]} = OldReturn, {record,L3,ReturnRecVar,_OldName,Fields} = OldReturnRec, Folder = fun({Id, Rule, StrType, Name, Opts}, Facc) -> Type = atomize(StrType), FClause = {clause,L,[{match,L,{record,L,atomize(MsgName),RecArgFields},RecVar},{atom,L,atomize(Name)},ValueArg],Gs,[ {match,L2,NewReturn,{call,L2,DictStore,[{integer,L2,Id},{tuple,L2,[erl_parse:abstract(Rule),ValueArg,erl_parse:abstract(Type),erl_parse:abstract(Opts)]},StoreVar]}}, {tuple,L3,[Ok,{record,L3,ReturnRecVar,atomize(MsgName),Fields}]} ]}, [FClause | Facc] end, NewAcc = lists:foldl(Folder, Acc, Extends), filter_set_extension(Tail,Clause,NewAcc). %% @hidden filter_get_extension_atom([],_AtomClause,Acc) -> Acc; filter_get_extension_atom([{_,_,disallowed}|Tail],Clause,Acc) -> filter_get_extension_atom(Tail,Clause,Acc); filter_get_extension_atom([{Msg,_,Extends}|Tail],Clause,Acc) -> {clause,L,[RecArg,_OldAtom],[RecG],[OldSubcall]} = Clause, [{call,L,Guard,[Garg1,_RecName]}] = RecG, {call,L1,Subname,[RecVar,_OldInt]} = OldSubcall, NewG = [{call,L,Guard,[Garg1,{atom,L,atomize(Msg)}]}], NewClauses = [ {clause,L,[RecArg,{atom,L,atomize(FName)}],[NewG],[ {call,L1,Subname,[RecVar,{integer,L1,FId}]} ]} || {FId, _, _, FName, _} <- Extends], NewAcc = lists:append(NewClauses,Acc), filter_get_extension_atom(Tail,Clause,NewAcc). %% @hidden filter_get_extension_integer([],_,Acc) -> Acc; filter_get_extension_integer([{_,_,disallowed}|Tail],IntClause,Acc) -> filter_get_extension_integer(Tail,IntClause,Acc); filter_get_extension_integer([{Msg,_,_Extends}|Tail],IntClause,Acc) -> {clause,L,[{record,L,Pikachu,Fields},IntArg],Gs,Body} = IntClause, NewRecName = replace_atom(Pikachu, pikachu, atomize(Msg)), NewRecArg = {record,L,NewRecName,Fields}, NewClause = {clause,L,[NewRecArg,IntArg],Gs,Body}, NewAcc = [NewClause|Acc], filter_get_extension_integer(Tail,IntClause,NewAcc). %% @hidden filter_has_extension([], _, Acc) -> % non-reverseal is intentional. Acc; filter_has_extension([{_Msg,_,disallowed}|Tail], Clause, Acc) -> filter_has_extension(Tail, Clause, Acc); filter_has_extension([{MsgName,_,Extends}|Tail], Clause, Acc) -> {clause,L,[OldRecArg,_],G,[Body]} = Clause, {call, L1, {remote,L1,Dict,IsKey},[_Key,DictArg]} = Body, RecArg = replace_atom(OldRecArg,pikachu,atomize(MsgName)), Folder = fun({ID, _Rules, _Type, Name, _Other}, FoldAcc) -> AtomClause = {clause,L,[RecArg,{atom,L,atomize(Name)}],G,[ {call,L,{remote,L,Dict,IsKey},[{atom,L,atomize(Name)},DictArg]} ]}, IntClause = {clause,L,[RecArg,{integer,L,ID}],G,[ {call,L,{remote,L,Dict,IsKey},[{integer,L,ID},DictArg]} ]}, [AtomClause,IntClause|FoldAcc] end, NewClauses = lists:foldl(Folder, [], Extends), NewAcc = lists:append(Acc,NewClauses), filter_has_extension(Tail,Clause,NewAcc). %% @hidden filter_extension_size([], _RecClause, Acc) -> % the non-reversal is intentional. Acc; filter_extension_size([{_MsgName,_,disallowed}|Tail],Clause,Acc) -> filter_extension_size(Tail,Clause,Acc); filter_extension_size([{MsgName,_,_}|Tail],Clause,Acc) -> {clause,L,[OldArg],G,Body} = Clause, NewClause = {clause,L,[replace_atom(OldArg,pikachu,atomize(MsgName))],G,Body}, NewAcc = [NewClause | Acc], filter_extension_size(Tail,Clause,NewAcc). %% @hidden filter_delimited_encode_clause({MsgName, _Fields, _Extends}, {clause,L,[_PlaceholderName | Args],Guards,Content}) -> {clause,L,[{atom,L,atomize(MsgName)}|Args], Guards, Content}. %% @hidden filter_record_encode_clause({MsgName, _Fields,_Extends}, {clause,L,_Args,Guards,_Content}) -> ToIolist = {cons, L, {call,L, {atom,L,iolist}, [{atom,L,atomize(MsgName)},{var,L,'Record'}]}, {call,L, {atom,L,encode_extensions}, [{var,L,'Record'}]} }, {clause,L,[{atom,L,atomize(MsgName)},{var,L,'Record'}],Guards,[ToIolist]}. expand_iolist_function(Msgs, Line, Clause) -> {function,Line,iolist,2,[filter_iolist_clause(Msg, Clause) || Msg <- Msgs]}. filter_iolist_clause({MsgName, [], _Extends0}, {clause,L,_Args,Guards,_Content}) -> {clause,L,[{atom,L,atomize(MsgName)},{var,L,'_Record'}],Guards,[{nil,L}]}; filter_iolist_clause({MsgName, Fields0, _Extends0}, {clause,L,_Args,Guards,_Content}) -> Fields = [ case Tag of optional -> Field; _ -> {FNum,Tag,SType,SName,none} end || {FNum,Tag,SType,SName,_} = Field <- Fields0 ], Cons = lists:foldl( fun({FNum,Tag,SType,SName,Default}, Acc) -> {cons,L, {call,L,{atom,L,pack},[{integer,L,FNum}, {atom,L,Tag}, {call,L, {atom,L,with_default}, [{record_field,L, {var,L,'Record'},atomize(MsgName), {atom,L,atomize(SName)}}, erl_parse:abstract(Default)]}, {atom,L,atomize(SType)}, {nil,L}]}, Acc} end, {nil,L}, lists:reverse(lists:keysort(1, Fields))), {clause,L,[{atom,L,atomize(MsgName)},{var,L,'Record'}],Guards,[Cons]}. %% @hidden expand_decode_function(Msgs, Line, Clause) -> {function,Line,decode,2, [{clause,Line,[{atom,Line,enummsg_values},{integer,Line,1}],[],[{atom,Line,value1}]}] ++ [filter_decode_clause(Msgs, Msg, Clause) || Msg <- Msgs]}. %% @hidden filter_decode_clause(Msgs, {MsgName, Fields, Extends}, {clause,L,_Args,Guards,[_,_,C,D]}) -> Types = lists:keysort(1, [begin {FNum, list_to_atom(SName), atomize(SType), decode_opts(Msgs, Tag, SType), Def} end || {FNum,Tag,SType,SName,Def} <- Fields]), Cons = lists:foldl( fun({FNum, FName, Type, Opts, _Def}, Acc) -> {cons,L,{tuple,L,[{integer,L,FNum},{atom,L,list_to_atom(string:to_lower(atom_to_list(FName)))},{atom,L,Type},erl_parse:abstract(Opts)]},Acc} end, {nil,L}, Types), ExtendDefault = case Extends of disallowed -> {nil,L}; _ -> erl_parse:abstract([{false, '$extensions', dict:new()}]) end, Defaults = lists:foldr( fun ({_FNum, _FName, _Type, _Opts, none}, Acc) -> Acc; ({FNum, FName, _Type, _Opts, Def}, Acc) -> {cons,L,{tuple,L,[{integer,L,FNum},{atom,L,list_to_atom(string:to_lower(atom_to_list(FName)))},erl_parse:abstract(Def)]},Acc} end, ExtendDefault, Types), A = {match,L,{var,L,'Types'},Cons}, B = {match,L,{var,L,'Defaults'},Defaults}, D1 = replace_atom(D, pikachu, atomize(MsgName)), {clause,L,[{atom,L,atomize(MsgName)},{var,L,'Bytes'}],Guards,[A,B,C,D1]}. %% @hidden filter_decode_extensions_clause(_,[],_,Acc) -> lists:reverse(Acc); filter_decode_extensions_clause(Msgs,[{_,_,disallowed}|Tail],Clause,Acc) -> filter_decode_extensions_clause(Msgs,Tail,Clause,Acc); filter_decode_extensions_clause(Msgs,[{MsgName,_,Extends}|Tail],Clause,Acc) -> {clause,L,_,_,_} = Clause, Types = lists:keysort(1, [{FNum, list_to_atom(SName), atomize(SType), decode_opts(Msgs, Tag, SType), Def} || {FNum,Tag,SType,SName,Def} <- Extends]), Cons = lists:foldl( fun({FNum, FName, Type, Opts, _Def}, AccF) -> {cons,L,{tuple,L,[{integer,L,FNum},{atom,L,FName},{atom,L,Type},erl_parse:abstract(Opts)]},AccF} end, {nil,L}, Types), A = {match,L,{var,L,'Types'},Cons}, {clause,L,[Arg],Guards,[_,B,C]} = Clause, NewBody = [A,B,replace_atom(C,pikachu,atomize(MsgName))], NewClause = {clause,L,[replace_atom(Arg, pikachu, atomize(MsgName))],Guards,NewBody}, filter_decode_extensions_clause(Msgs,Tail,Clause,[NewClause|Acc]). %% @hidden expand_encode_function(Msgs, Line, ListClause, RecordClause) -> Clauses = lists:foldl(fun(Msg, Acc) -> DelimClause = filter_delimited_encode_clause(Msg, ListClause), RecClause = filter_record_encode_clause(Msg, RecordClause), Acc ++ [DelimClause, RecClause] end, [], Msgs), {function,Line,encode,2,Clauses}. %% @hidden decode_opts(Msgs, Tag, Type) -> Opts0 = if Tag == repeated -> [repeated]; Tag == repeated_packed -> [repeated_packed]; true -> [] end, case lists:keymember(Type, 1, Msgs) of true -> [is_record|Opts0]; false -> Opts0 end. %% @hidden expand_to_record_function(Msgs, Line, Clause) -> {function,Line,to_record,2,[filter_to_record_clause(Msg, Clause) || Msg <- Msgs]}. %% @hidden filter_to_record_clause({MsgName, _, Extends}, {clause,L,[_Param1,Param2],Guards,[Fold,_DecodeExtends]}) -> Fold1 = replace_atom(Fold, pikachu, atomize(MsgName)), ReturnLine = case Extends of disallowed -> {var,L,'Record1'}; _ -> {ok, Tokens, _} = erl_scan:string("decode_extensions(Record1)."), {ok, [Abstract]} = erl_parse:parse_exprs(Tokens), Abstract end, {clause,L,[{atom,L,atomize(MsgName)},Param2],Guards,[Fold1,ReturnLine]}. %% @hidden expand_enum_to_int_function([], Line, Clause) -> {function,Line,enum_to_int,2,[Clause]}; expand_enum_to_int_function(Enums, Line, Clause) -> {function,Line,enum_to_int,2,[filter_enum_to_int_clause(Enum, Clause) || Enum <- Enums]}. %% @hidden filter_enum_to_int_clause({enum,EnumTypeName,IntValue,EnumValue}, {clause,L,_Args,Guards,_}) -> {clause,L,[{atom,L,atomize(EnumTypeName)},{atom,L,EnumValue}],Guards,[{integer,L,IntValue}]}. %% @hidden expand_int_to_enum_function([], Line, Clause) -> {function,Line,int_to_enum,2,[Clause]}; expand_int_to_enum_function(Enums, Line, Clause) -> {function,Line,int_to_enum,2, [filter_int_to_enum_clause(Enum, Clause) || Enum <- Enums] ++ [Clause]}. %% @hidden filter_int_to_enum_clause({enum,EnumTypeName,IntValue,EnumValue}, {clause,L,_Args,Guards,_}) -> {clause,L,[{atom,L,atomize(EnumTypeName)},{integer,L,IntValue}],Guards,[{atom,L,EnumValue}]}. %% @hidden %% [{"Location", %% [{2,required,"string","country",none}, %% {1,required,"string","region",none}]}, %% {"Person", %% [{5,optional,"Location","location",none}, %% {4,required,"int32","age",none}, %% {3,required,"string","phone_number",none}, %% {2,required,"string","address",none}, %% {1,required,"string","name",none}]}] collect_full_messages(Data) -> collect_full_messages(Data, #collected{}). collect_full_messages([{message, Name, Fields} | Tail], Collected) -> Package = Collected#collected.package, ListName = resolve_list_name(Name, Package), FieldsOut = lists:foldl( fun ({_,_,_,_,_} = Input, TmpAcc) -> [Input | TmpAcc]; (_, TmpAcc) -> TmpAcc end, [], Fields), Enums = lists:foldl( fun ({enum,C,D}, TmpAcc) -> [{enum, [list_to_tuple([C | LN]) || LN <- ListName], D} | TmpAcc]; (_, TmpAcc) -> TmpAcc end, [], Fields), Extensions = lists:foldl( fun ({extensions, From, To}, TmpAcc) -> [{From,To}|TmpAcc]; (_, TmpAcc) -> TmpAcc end, [], Fields), SubMessages = lists:foldl( fun ({message, C, D}, TmpAcc) -> [{message, [list_to_tuple([C | LN]) || LN <- ListName], D} | TmpAcc]; (_, TmpAcc) -> TmpAcc end, [], Fields), ExtendedFields = case Extensions of [] -> disallowed; _ -> [] end, NewCollected = Collected#collected{ msg=[{[list_to_tuple(L) || L <- ListName], FieldsOut, ExtendedFields} | Collected#collected.msg], extensions=[{[list_to_tuple(L) || L <- ListName],Extensions} | Collected#collected.extensions] }, collect_full_messages(Tail ++ SubMessages ++ Enums, NewCollected); collect_full_messages([{enum, Name, Fields} | Tail], Collected) -> Package = Collected#collected.package, ListName = resolve_list_name(Name, Package), FieldsOut = lists:foldl( fun (Field, TmpAcc) -> case Field of {EnumAtom, IntValue} -> [{enum, [type_path_to_type(LN) || LN <-ListName], IntValue, EnumAtom} | TmpAcc]; _ -> TmpAcc end end, [], Fields), NewCollected = Collected#collected{enum=FieldsOut++Collected#collected.enum}, collect_full_messages(Tail, NewCollected); collect_full_messages([{syntax,_} | Tail], Collected) -> collect_full_messages(Tail, Collected); collect_full_messages([{package, PackageName} | Tail], Collected) -> collect_full_messages(Tail, Collected#collected{package = PackageName}); collect_full_messages([{option,_,_} | Tail], Collected) -> collect_full_messages(Tail, Collected); collect_full_messages([{import, _Filename} | Tail], Collected) -> collect_full_messages(Tail, Collected); collect_full_messages([{extend, Name, ExtendedFields} | Tail], Collected) -> SeekName0 = string:tokens(Name, "."), SeekName1 = [list_to_tuple(lists:reverse(SeekName0))], SeekNames0 = resolve_list_name(SeekName1, Collected#collected.package), SeekNames = [list_to_tuple(SN) || SN <- SeekNames0], #collected{msg = CollectedMsg} = Collected, BestMatch = element(1, find_message_by_path(SeekNames, CollectedMsg)), {ListName,FieldsOut,ExtendFields} = lists:keyfind(BestMatch,1,CollectedMsg), {ListName,Extensions} = lists:keyfind(BestMatch,1,Collected#collected.extensions), FunNotInReservedRange = fun(Id) -> not(19000 =< Id andalso Id =< 19999) end, FunInRange = fun(Id,From,max) -> From =< Id andalso Id =< 16#1fffffff; (Id,From,To) -> From =< Id andalso Id =< To end, ExtendedFieldsOut = lists:append(FieldsOut, lists:foldl( fun ({Id, _, _, FieldName, _} = Input, TmpAcc) -> case lists:any(fun({From,To}) -> FunNotInReservedRange(Id) andalso FunInRange(Id,From,To) end,Extensions) of true -> [Input | TmpAcc]; _ -> error_logger:error_report(["Extended field not in valid range", {message, Name}, {field_id,Id}, {field_name,FieldName}, {defined_ranges,Extensions}, {reserved_range,{19000,19999}}, {max,16#1fffffff}]), throw(out_of_range) end; (_, TmpAcc) -> TmpAcc end, [], ExtendedFields) ), NewExtends = case ExtendFields of disallowed -> disallowed; _ -> ExtendFields ++ ExtendedFieldsOut end, NewCollected = Collected#collected{msg=lists:keyreplace(ListName,1,CollectedMsg,{ListName,FieldsOut,NewExtends})}, collect_full_messages(Tail, NewCollected); collect_full_messages([file_boundary | Tail], Collected) -> collect_full_messages(Tail, Collected#collected{package = undefined}); %% Skip anything we don't understand collect_full_messages([Skip|Tail], Acc) -> error_logger:warning_report(["Unkown, skipping", {skip,Skip}]), collect_full_messages(Tail, Acc); collect_full_messages([], Collected) -> Collected. %% @hidden find_message_by_path(_TypeName, []) -> false; find_message_by_path(TypeName, [Msg | Tail]) -> Names = element(1, Msg), case [N || N <- Names, lists:member(N, TypeName)] of [] -> find_message_by_path(TypeName, Tail); _ -> Msg end. %% @hidden resolve_list_name(Name, _Package) when is_tuple(hd(Name)) -> [tuple_to_list(N) || N <- Name]; resolve_list_name(Name, undefined) when is_integer(hd(Name)) -> [[Name]]; resolve_list_name(Name, Package) when is_integer(hd(Name)) -> [[Name], [Name, Package]]. %% @hidden resolve_types(Data, Enums) -> resolve_types (Data, Data, Enums, []). resolve_types([{TypePath, Fields,Extended} | Tail], AllPaths, Enums, Acc) -> FolderFun = fun (Input, TmpAcc) -> case Input of {Index, Rules, Type, Identifier, Other} -> case is_scalar_type(Type) of true -> [Input | TmpAcc]; false -> PossiblePaths = case string:tokens(Type,".") of [Type] -> all_possible_type_paths(Type, TypePath); FullPath -> % handle types of the form Foo.Bar which are absolute, % so we just convert to a type path and check it. [lists:reverse(FullPath)] end, RealPath = case find_type(PossiblePaths, AllPaths) of false -> case is_enum_type(Type, PossiblePaths, Enums) of {true,EnumType} -> [EnumType]; false -> throw(["Unknown Type ", Type]) end; ResultType -> ResultType end, [{Index, Rules, canonical_name(RealPath), Identifier, Other} | TmpAcc] end; _ -> TmpAcc end end, FieldsOut = lists:foldl(FolderFun, [], Fields), ExtendedOut = case Extended of disallowed -> disallowed; _ -> MidExtendOut = lists:foldl(FolderFun, [], Extended), lists:reverse(MidExtendOut) end, Type = type_path_to_type(TypePath), resolve_types(Tail, AllPaths, Enums, [{Type, lists:reverse(FieldsOut), ExtendedOut } | Acc]); resolve_types([], _, _, Acc) -> Acc. %% @hidden write_header_include_file(Basename, Messages) when is_list(Basename) -> {ok, FileRef} = protobuffs_file:open(Basename, [write]), write_header_include_file(FileRef, Messages), protobuffs_file:close(FileRef); write_header_include_file(_FileRef, []) -> ok; write_header_include_file(FileRef, [{Name, Fields, Extends} | Tail]) -> OutFields = [{string:to_lower(A), Optional, Default} || {_, Optional, _, A, Default} <- lists:keysort(1, Fields)], DefName = string:to_upper(Name) ++ "_PB_H", protobuffs_file:format(FileRef, "-ifndef(~s).~n-define(~s, true).~n", [DefName, DefName]), protobuffs_file:format(FileRef, "-record(~s, {~n ", [string:to_lower(Name)]), WriteFields0 = generate_field_definitions(OutFields), WriteFields = case Extends of disallowed -> WriteFields0; _ -> ExtenStr = "'$extensions' = dict:new()", WriteFields0 ++ [ExtenStr] end, FormatString = string:join(["~s" || _ <- lists:seq(1, length(WriteFields))], ",~n "), protobuffs_file:format(FileRef, FormatString, WriteFields), protobuffs_file:format(FileRef, "~n}).~n", []), protobuffs_file:format(FileRef, "-endif.~n~n", []), write_header_include_file(FileRef, Tail). %% @hidden generate_field_definitions(Fields) -> generate_field_definitions(Fields, []). %% @hidden generate_field_definitions([], Acc) -> lists:reverse(Acc); generate_field_definitions([{Name, required, _} | Tail], Acc) -> Head = lists:flatten(io_lib:format("~s = erlang:error({required, ~s})", [Name, Name])), generate_field_definitions(Tail, [Head | Acc]); generate_field_definitions([{Name, _, none} | Tail], Acc) -> Head = lists:flatten(io_lib:format("~s", [Name])), generate_field_definitions(Tail, [Head | Acc]); generate_field_definitions([{Name, _, Default} | Tail], Acc) -> Head = lists:flatten(io_lib:format("~s = ~p", [Name, Default])), generate_field_definitions(Tail, [Head | Acc]). %% @hidden % handle ["symbol"] atomize([String]) when is_list(String) -> atomize(String); % handle ["symbol", "package_symbol"] atomize([String|[_Rest]]) when is_list(String) -> atomize(String); atomize(String) -> list_to_atom(string:to_lower(String)). %% @hidden replace_atom(Find, Find, Replace) -> Replace; replace_atom(Tuple, Find, Replace) when is_tuple(Tuple) -> list_to_tuple([replace_atom(Term, Find, Replace) || Term <- tuple_to_list(Tuple)]); replace_atom(List, Find, Replace) when is_list(List) -> [replace_atom(Term, Find, Replace) || Term <- List]; replace_atom(Other, _Find, _Replace) -> Other. %% @hidden is_scalar_type("double") -> true; is_scalar_type("float") -> true; is_scalar_type("int32") -> true; is_scalar_type("int64") -> true; is_scalar_type("uint32") -> true; is_scalar_type("uint64") -> true; is_scalar_type("sint32") -> true; is_scalar_type("sint64") -> true; is_scalar_type("fixed32") -> true; is_scalar_type("fixed64") -> true; is_scalar_type("sfixed32") -> true; is_scalar_type("sfixed64") -> true; is_scalar_type("bool") -> true; is_scalar_type("string") -> true; is_scalar_type("bytes") -> true; is_scalar_type(_) -> false. %% @hidden is_enum_type(_Type, [], _Enums) -> false; is_enum_type(Type, [TypePath|Paths], Enums) -> TypeFromPath = type_path_to_type(TypePath), case is_enum_type(TypeFromPath, Enums) of true -> {true,TypePath}; false -> is_enum_type(Type, Paths, Enums) end. is_enum_type(Type, Enums) -> lists:any(fun(Enum) -> Types = element(2, Enum), lists:member(Type, Types) end, Enums). %% @hidden sublists(List) when is_list(List) -> sublists(List,[]). sublists([],Acc) -> [ [] | Acc ]; sublists(List,Acc) -> sublists(tl(List), [ List | Acc ]). %% @hidden all_possible_type_paths(Type, TypePaths) -> all_possible_type_paths(Type, TypePaths, []). all_possible_type_paths(_Type, [], Acc) -> lists:reverse(lists:flatten(Acc)); all_possible_type_paths(Type, [TypePath | Tail], Acc) -> TypePath0 = tuple_to_list(TypePath), Head = lists:foldl(fun (TypeSuffix, AccIn) -> [list_to_tuple([Type | TypeSuffix]) | AccIn] end, [], sublists(TypePath0)), all_possible_type_paths(Type, Tail, [Head | Acc]). %% @hidden find_type([], _KnownTypes) -> false; find_type([Type | TailTypes], KnownTypes) when is_list(Type) -> find_type([list_to_tuple(Type) | TailTypes], KnownTypes); find_type([Type | TailTypes], KnownTypes) -> FilterFun = fun(KnownType) -> lists:member(Type, element(1, KnownType)) end, case lists:filter(FilterFun, KnownTypes) of [] -> find_type(TailTypes, KnownTypes); [{RealType, _, _} | _] -> RealType end. %% @hidden canonical_name([TypePath | _]) -> % yes, we're assuming the first name in the list is the most % cannonical name. type_path_to_type(TypePath). canonize_names(Messages) -> canonize_names(Messages, []). canonize_names([], Acc) -> lists:reverse(Acc); canonize_names([Enum | Tail], Acc) when element(1, Enum) == enum -> Name = canonical_name(element(2, Enum)), Enum0 = setelement(2, Enum, Name), canonize_names(Tail, [Enum0 | Acc]); canonize_names([Msg | Tail], Acc) -> Name = canonical_name(element(1, Msg)), Msg0 = setelement(1,Msg,Name), canonize_names(Tail, [Msg0 | Acc]). %% @hidden type_path_to_type([[Name|Tuple]]) when is_tuple(Tuple) -> type_path_to_type([Name | tuple_to_list(Tuple)]); type_path_to_type(TypePath) when is_tuple(TypePath) -> type_path_to_type(tuple_to_list(TypePath)); type_path_to_type(TypePath) -> case is_list(hd(TypePath)) of true -> string:join(lists:reverse(TypePath), "_"); false -> TypePath end.