%%% --------------------------------------------------------------------------------- %%% @doc %%% broen_core turns HTTP requests/responses into AMQP RPC messaging. %%% Given a HTTP Request, this module will first authenticate it using the provided %%% authentication plugin and the publish the message serialized with the serializer %%% plug over AMQP. Upon receiving a response, the module will respond back over HTTP. %%% @end %%% --------------------------------------------------------------------------------- -module(broen_core). -export([handle/4]). -export([register_metrics/0]). -define(CLIENT_REQUEST_BODY_LIMIT, 65536). -define(DEFAULT_TIMEOUT, 20). % secs -define(ACAO_HEADER, <<"access-control-allow-origin">>). -type content_type() :: unicode:unicode_binary(). %% The MIME content type -type broen_string() :: unicode:unicode_binary(). %% A binary string -type broen_nullable_string() :: unicode:unicode_binary() | null. %% A binary string that can be null -type broen_object() :: #{broen_string() => broen_string()}. %% An generic sub-object that is a map mapping a string to a string. Used for e.g. HTTP headers -type cookie_name() :: broen_string(). %% The name of a cookie -type cookie_value() :: #{ value := broen_string(), domain => broen_string(), path => broen_string(), http_only => boolean(), secure => boolean(), expires => broen_string()}. %% The cookie properties. Each cookie must define a value and may optionally define the domain it applies to and the expiration date -type broen_cookies() :: #{cookie_name() => cookie_value()}. %% The cookies object maps cookie names to the properties. -type broen_request() :: #{ appmoddata := broen_string(), protocol := http | https, cookies := broen_object(), http_headers := broen_object(), request := broen_string(), method := broen_string(), referer := broen_nullable_string(), fullpath := broen_string(), useragent := broen_nullable_string(), client_data := binary() | null, client_ip := broen_string(), routing_key := broen_string(), queryobj := broen_object(), auth_data := term(), querydata => broen_string(), postobj => broen_object(), multipartobj => term()}. % % % }. %% The format of a broen request that is sent to the serializer plugin.
%% cookies - Cookies attached to the HTTP request
%% http_headers - HTTP request headers
%% request - The HTTP method
%% method - Same as above
%% client_data - Client information
%% fullpath - Full path of the request as provided by Yaws
%% appmoddata - The URL that is turned into the routing key (i.e. what follows /call)
%% referer - The referer URL
%% useragent - User agent data
%% client_ip - IP of the client
%% routing_key - The routing key the request will be sent to
%% queryobj - The query object containing the query parameters
%% auth_data - Data returned by the authentication module
%% querydata - Same as queryobj, but in a string format
%% postobj- Data attached to a POST request
%% multipartobj - Data for the multipart request
-type broen_response() :: #{ payload := term(), status_code => integer(), media_type => content_type(), cookies => broen_cookies(), cookie_path => broen_string(), headers => broen_object()} | #{redirect := unicode:unicode_binary()}. %% The format of a broen response that should be returned by the serializer plugin
%% payload - The payload of the response
%% status_code - Status code of the response
%% media_type - The MIME content type of the payload
%% cookies - Additional cookies to be sent to user
%% cookie_path - The cookie path
%% headers - Additional headers for the HTTP response
%% Alternatively the response can also be a redirect. -export_type([content_type/0, broen_request/0, broen_response/0]). %% @doc Registers metrics with folsom -spec register_metrics() -> ok. register_metrics() -> Groups = application:get_env(broen, metric_groups, []), [begin Key = iolist_to_binary(["broen_core.query.", G]), KeyT = iolist_to_binary(["broen_core.query.", G, ".timeout"]), KeyL = iolist_to_binary(["broen_core.query.", G, ".latency"]), folsom_metrics:new_spiral(binary_to_atom(Key, utf8)), folsom_metrics:new_spiral(binary_to_atom(KeyT, utf8)), folsom_metrics:new_histogram(binary_to_atom(KeyL, utf8), slide_uniform) end || G <- Groups], [folsom_metrics:new_spiral(C) || C <- ['broen_core.success', 'broen_core.query.unknown', 'broen_core.query.unknown.timeout', 'broen_core.failure.crash', 'broen_core.failure.500', 'broen_core.failure.503', 'broen_core.failure.404']], [folsom_metrics:new_histogram(H, slide_uniform) || H <- ['broen_core.query.unknown.latency']], folsom_metrics:new_spiral('broen_auth.failure'), ok. handle(Req0, Exchange, CookiePath, Options) -> try RoutingKey = routing_key(Req0, Options), Timeout = proplists:get_value(timeout, Options, ?DEFAULT_TIMEOUT), case broen_request:check_http_origin(Req0, RoutingKey) of {_, unknown_origin} -> folsom_metrics:notify({'broen_core.failure.403', 1}), cowboy_req:reply(403, #{<<"content-type">> => <<"text/plain">>}, <<"Forbidden">>, Req0); {Origin, OriginMode} -> {AmqpRes, ExtraCookies} = amqp_call(Req0, Exchange, RoutingKey, Timeout), ReqWithCookies = lists:foldl(fun set_extra_cookie/2, Req0, ExtraCookies), case AmqpRes of {ok, PackedResponse, ContentType} -> {ok, SerializerMod} = application:get_env(broen, serializer_mod), case SerializerMod:deserialize(PackedResponse, ContentType) of {ok, Response} -> folsom_metrics:notify({'broen_core.success', 1}), build_response(ReqWithCookies, Response, CookiePath, OriginMode, Origin); {error, invalid_content_type} -> folsom_metrics:notify({'broen_core.failure.500', 1}), cowboy_req:reply(500, #{<<"content-type">> => <<"text/plain">>}, iolist_to_binary([io_lib:format("Got wrong type of Media Type in response: ~ts", [ContentType])]), ReqWithCookies) end; {error, timeout} -> {_, MetricTimeout, _} = subsystem_metric(RoutingKey), folsom_metrics:notify({MetricTimeout, 1}), cowboy_req:reply(504, #{<<"content-type">> => <<"text/plain">>}, <<"API Broen timeout">>, ReqWithCookies); {error, {reply_code, 312}} -> folsom_metrics:notify({'broen_core.failure.404', 1}), cowboy_req:reply(404, #{<<"content-type">> => <<"text/plain">>}, <<"Not found">>, ReqWithCookies); {error, no_route} -> folsom_metrics:notify({'broen_core.failure.503', 1}), cowboy_req:reply(503, #{<<"content-type">> => <<"text/plain">>}, <<"Service unavailable (no_route)">>, ReqWithCookies); {error, csrf_verification_failed} -> cowboy_req:reply(403, #{<<"content-type">> => <<"text/plain">>}, <<"Forbidden">>, ReqWithCookies); {error, Reason} -> folsom_metrics:notify({'broen_core.failure.500', 1}), cowboy_req:reply(500, #{<<"content-type">> => <<"text/plain">>}, iolist_to_binary([io_lib:format("~p~n", [Reason])]), ReqWithCookies) end end catch _: Error -> Now = erlang:timestamp(), Token = base64:encode(crypto:hash(sha256, term_to_binary(Now))), ok = lager:error("Crash: ~p Error: ~p StackTrace: ~p", [Token, Error, erlang:get_stacktrace()]), folsom_metrics:notify({'broen_core.failure.crash', 1}), cowboy_req:reply(500, #{<<"content-type">> => <<"text/plain">>}, iolist_to_binary([io_lib:format("Internal error, broen code crashed ~p~n", [Token])]), Req0) end. %% Internal functions %% --------------------------------------------------------------------------------- set_extra_cookie({CookieName, CookieValue}, R) -> Value = maps:get(value, CookieValue), MaxAge = maps:get(max_age, CookieValue), Path = maps:get(path, CookieValue, <<"/">>), Domain = maps:get(domain, CookieValue), cowboy_req:set_resp_cookie(CookieName, Value, R, #{ domain => Domain, path => Path, max_age => MaxAge }). amqp_call(Req, Exchange, RoutingKey, Timeout) -> TimeZero = os:timestamp(), {ok, AuthMod} = application:get_env(broen, auth_mod), case AuthMod:authenticate(Req) of {error, csrf_verification_failed} -> {{error, csrf_verification_failed}, []}; {error, {csrf_verification_failed, Cookies}} -> {{error, csrf_verification_failed}, Cookies}; {error, _} -> {handle_http(TimeZero, [], Req, Exchange, RoutingKey, Timeout), []}; {ok, AuthData, Cookies} -> {handle_http(TimeZero, AuthData, Req, Exchange, RoutingKey, Timeout), Cookies} end. handle_http(TimeZero, AuthData, Arg, Exch, RoutingKey, Timeout) -> Request = broen_request:build_request(Arg, RoutingKey, AuthData), {Metric, _, MetricL} = subsystem_metric(RoutingKey), folsom_metrics:notify({Metric, 1}), TimeBefore = os:timestamp(), {ok, SerializerMod} = application:get_env(broen, serializer_mod), {Packed, ContentType} = SerializerMod:serialize(Request), Reply = ad_client:call_timeout(amqp_rpc, Exch, RoutingKey, Packed, ContentType, [{timeout, timer:seconds(Timeout)}]), TimeAfter = os:timestamp(), histogram_notify(MetricL, timer:now_diff(TimeAfter, TimeZero) div 1000), lager:debug("broen_core:amqp_call ~p ~p", [timer:now_diff(TimeBefore, TimeZero), timer:now_diff(TimeAfter, TimeZero)]), case Reply of {error, timeout} -> lager:warning("broen_core:amqp_call timeout ~s ~p", [RoutingKey, Request]); _ -> ok end, Reply. routing_key(Req, Options) -> Path = cowboy_req:path_info(Req), TrailingSlash = binary:last(cowboy_req:path(Req)) == $/, case valid_route(Path) of false -> <<"route.invalid">>; true when TrailingSlash -> route(proplists:get_bool(keep_dots_in_routing_keys, Options), Path ++ [<<>>]); true -> route(proplists:get_bool(keep_dots_in_routing_keys, Options), Path) end. valid_route(Paths) -> Sum = lists:foldl(fun(El, Sum) -> Sum + byte_size(El) end, 0, Paths), Sum =< 255. %% '.' is converted to '_' iff the keep_dots_in_routing_key is false, %% otherwise it is left as a '.' route(false, Route) -> Mapped = lists:map(fun(El) -> binary:replace(El, <<".">>, <<"_">>, [global]) end, Route), route(true, Mapped); route(true, [First | Rest]) -> lists:foldl(fun(El, SoFar) -> <> end, First, Rest). %% Decoders of various responses %% --------------------------------------------------------------------------------- build_response(Req, #{redirect := URL}, _, _, _) -> cowboy_req:reply( 302, #{<<"location">> => URL}, <<>>, Req ); build_response(Req, Response, CookiePath, OriginMode, Origin) -> StatusCode = maps:get(status_code, Response, 200), Content = maps:get(payload, Response, <<>>), MediaType = maps:get(media_type, Response, <<>>), RespwithCookies = cookies(Req, Response, CookiePath), cowboy_req:reply( StatusCode, maps:from_list(headers(Response, OriginMode, Origin) ++ [{<<"content-type">>, MediaType}]), Content, RespwithCookies ). headers(Response, OriginMode, Origin) -> Headers = maps:to_list(maps:get(headers, Response, #{})), [{binary_to_list(N), binary_to_list(V)} || {N, V} <- append_cors(Headers, Origin, OriginMode)]. append_cors(Headers, _, same_origin) -> Headers; append_cors(Headers, Origin, allow_origin) -> case lists:keysearch(?ACAO_HEADER, 1, Headers) of false -> [{?ACAO_HEADER, Origin} | Headers]; _ -> Headers end. cookies(InitialReq, Response, DefaultCookiePath) -> Cookies = maps:to_list(maps:get(cookies, Response, #{})), CookiePath = maps:get(cookie_path, Response, DefaultCookiePath), DefaultExpires = iso8601:format({{2038, 1, 17}, {12, 34, 56}}), lists:foldl(fun(Cookie, Req) -> set_cookie(Cookie, CookiePath, DefaultExpires, Req) end, InitialReq, Cookies). set_cookie({CookieName, CookieValue}, DefaultCookiePath, DefaultExpires, Req) -> Expiry = parse_expiry(maps:get(expires, CookieValue, DefaultExpires)), CookiePath = maps:get(path, CookieValue, DefaultCookiePath), Domain = maps:get(domain, CookieValue, undefined), Secure = maps:get(secure, CookieValue, false), HttpOnly = maps:get(http_only, CookieValue, false), Value = maps:get(value, CookieValue), cowboy_req:set_resp_cookie(CookieName, Value, Req, #{ domain => Domain, path => CookiePath, secure => Secure, http_only => HttpOnly, max_age => Expiry }). parse_expiry(Date) -> ParsedDate = parse_date(Date), UTC = calendar:universal_time(), Secs = calendar:datetime_to_gregorian_seconds(UTC), Expires = calendar:datetime_to_gregorian_seconds(ParsedDate), if Expires - Secs > 0 -> Expires - Secs; true -> 0 end. parse_date(Date) when is_list(Date) -> parse_date(list_to_binary(Date)); parse_date(Date) -> try iso8601:parse(Date) catch _:badarg -> cow_date:parse_date(Date) end. %% Other %% --------------------------------------------------------------------------------- subsystem_metric(RK) when is_binary(RK) -> case binary:split(RK, <<".">>) of [SS | _] -> Key = iolist_to_binary(["broen_core.query.", SS]), KeyT = iolist_to_binary(["broen_core.query.", SS, ".timeout"]), KeyL = iolist_to_binary(["broen_core.query.", SS, ".latency"]), try {binary_to_existing_atom(Key, utf8), binary_to_existing_atom(KeyT, utf8), binary_to_existing_atom(KeyL, utf8)} catch error:badarg -> lager:info("subsystem_metric unkown rk ~p ~p ~p", [Key, KeyT, KeyL]), {'broen_core.query.unknown', 'broen_core.query.unknown.timeout', 'broen_core.query.unknown.latency'} end; _ -> {'broen_core.query.unknown', 'broen_core.query.unknown.timeout', 'broen_core.query.unknown.latency'} end. histogram_notify(Name, Diff) -> case folsom_metrics:notify(Name, Diff) of {error, Name, nonexistent_metric} -> folsom_metrics:new_histogram(Name, slide_uniform), folsom_metrics:notify(Name, Diff); Res -> Res end.