%%% @author zhongwen -module(observer_cli_system). -include("observer_cli.hrl"). %% net_address record from OTP lacks typed fields; keep warning local to this module. -compile(nowarn_untyped_record). -include_lib("kernel/include/net_address.hrl"). %% API -export([start/1]). -export([clean/1]). -ifdef(TEST). -export([ fill_info/2, to_list/1, info_fields/0, get_cachehit_info/2, render_sys_info/1, render_sys_info/4, render_cache_hit_rates/2, render_block_size_info/4, get_alloc/5, render_dist_node_info/1, get_address/1, render_worker/3 ]). -endif. -define(UTIL_ALLOCATORS, [ binary_alloc, driver_alloc, eheap_alloc, ets_alloc, fix_alloc, ll_alloc, sl_alloc, std_alloc, temp_alloc ]). %% @doc List System and Architecture, CPU's and Threads metrics in observer's system %% List Memory Allocators: std, ll, eheap, ets, fix, binary, driver. -spec start(ViewOpts) -> no_return() when ViewOpts :: view_opts(). start(#view_opts{sys = #system{interval = Interval}} = ViewOpts) -> Pid = spawn_link(fun() -> ?output(?CLEAR), Cmd = io_lib:format("ps -o pcpu,pmem,rss,vsz ~s", [os:getpid()]), render_worker(Cmd, Interval, ?INIT_TIME_REF) end), manager(Pid, ViewOpts). -spec clean(list()) -> ok. clean(Pids) -> observer_cli_lib:exit_processes(Pids). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% Private %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% manager(Pid, #view_opts{sys = AllocatorOpts} = ViewOpts) -> case observer_cli_lib:parse_cmd(ViewOpts, ?MODULE, [Pid]) of quit -> erlang:send(Pid, quit); {new_interval, NewInterval} -> erlang:send(Pid, {new_interval, NewInterval}), NewAllocate = AllocatorOpts#system{interval = NewInterval}, manager(Pid, ViewOpts#view_opts{sys = NewAllocate}); _ -> manager(Pid, ViewOpts) end. render_worker(Cmd, Interval, LastTimeRef) -> CacheHitInfo = recon_alloc:cache_hit_rates(), AverageBlockCurs = recon_alloc:average_block_sizes(current), AverageBlockMaxes = recon_alloc:average_block_sizes(max), SbcsToMbcsCurs = observer_cli_lib:sbcs_to_mbcs( ?UTIL_ALLOCATORS, recon_alloc:sbcs_to_mbcs(current) ), SbcsToMbcsMaxs = observer_cli_lib:sbcs_to_mbcs(?UTIL_ALLOCATORS, recon_alloc:sbcs_to_mbcs(max)), Sys = render_sys_info(Cmd), Text = "Interval: " ++ integer_to_list(Interval) ++ "ms", Menu = observer_cli_lib:render_menu(allocator, Text), BlockView = render_block_size_info( AverageBlockCurs, AverageBlockMaxes, SbcsToMbcsCurs, SbcsToMbcsMaxs ), DistNodesInfo = get_dist_nodes_info(), DistNodeView = render_dist_node_info(DistNodesInfo), HitView = render_cache_hit_rates(CacheHitInfo, erlang:length(CacheHitInfo)), LastLine = observer_cli_lib:render_last_line("q(quit)"), ?output([?CURSOR_TOP, Menu, Sys, BlockView, DistNodeView, HitView, LastLine]), NextTimeRef = observer_cli_lib:next_redraw(LastTimeRef, Interval), receive quit -> quit; {new_interval, NewInterval} -> render_worker(Cmd, NewInterval, NextTimeRef); redraw -> render_worker(Cmd, Interval, NextTimeRef) end. get_dist_nodes_info() -> case ets:info(sys_dist, size) of undefined -> []; 0 -> []; _ -> {ok, DistNodesInfo} = net_kernel:nodes_info(), DistNodesInfo end. render_dist_node_info([]) -> []; render_dist_node_info(DistNodesInfo) -> [NodeW, QueueW, PercentW, AddressW, InW, OutW, TypeW, StateW] = dist_node_widths(), Title = ?render([ ?UNDERLINE, ?GRAY_BG, ?W("Node", NodeW), ?W("Dist Node Queue Size Bytes", QueueW), ?W("Percent", PercentW), ?W("Address", AddressW), ?W("In", InW), ?W("Out", OutW), ?W("Type", TypeW), ?W("State", StateW) ]), Limit = erlang:system_info(dist_buf_busy_limit), LimitStr = integer_to_list(Limit), View = lists:map( fun({Node, Info}) -> State = proplists:get_value(state, Info), Type = proplists:get_value(type, Info), Address = get_address(Info), In = proplists:get_value(in, Info), Out = proplists:get_value(out, Info), QueueSize = get_dist_queue_size(Node), QueueSizeStr = observer_cli_lib:to_list(QueueSize), QueueSizeLimitStr = QueueSizeStr ++ "/" ++ LimitStr, Percent = case is_integer(QueueSize) of true -> Float = QueueSize / Limit, [erlang:float_to_list(Float * 100, [{decimals, 2}]), $%]; false -> "unsupported" end, ?render([ ?W(Node, NodeW), ?W(QueueSizeLimitStr, QueueW), ?W(Percent, PercentW), ?W(Address, AddressW), ?W(In, InW), ?W(Out, OutW), ?W(Type, TypeW), ?W(State, StateW) ]) end, lists:sort(DistNodesInfo) ), [Title | View]. dist_node_widths() -> observer_cli_lib:weighted_widths( [30, 20, 7, 19, 11, 11, 7, 10], [4, 1, 0, 4, 0, 0, 0, 0] ). get_address(Info) -> #net_address{address = Address} = proplists:get_value(address, Info, #net_address{}), case Address of {Addr, Port} -> case inet:ntoa(Addr) of {error, _} -> AddrStr = lists:flatten(io_lib:format("~p", [Addr])), AddrStr ++ ":" ++ integer_to_list(Port); AddrStr -> AddrStr ++ ":" ++ integer_to_list(Port) end; _ -> "unknown" end. get_dist_queue_size(Node) -> case ets:lookup(sys_dist, Node) of [] -> not_found; [Dist] -> ConnId = element(3, Dist), {ok, _, _, Size} = erlang:dist_get_stat(ConnId), Size end. render_cache_hit_rates(CacheHitInfo, Len) when Len =< 8 -> Title = ?render([ ?UNDERLINE, ?GRAY_BG, ?W("Instance", 8), ?W("Hits", 10), ?W("Calls", 11), ?W("Hit Rate", 98) ]), View = [ begin [{hit_rate, HitRate}, {hits, Hit}, {calls, Call}] = proplists:get_value( {instance, Seq}, CacheHitInfo ), HitRateStr = observer_cli_lib:to_percent(HitRate), SeqStr = lists:flatten(io_lib:format("~2..0w", [Seq])), TrueHitRate = case Hit == 0 andalso Call == 0 of true -> 0; false -> HitRate end, Process = lists:duplicate(trunc(TrueHitRate * 91), "|"), ?render([ ?W(SeqStr, 8), ?W(Hit, 10), ?W(Call, 11), ?W(Process, 89), ?W(HitRateStr, 6) ]) end || Seq <- lists:seq(0, Len - 1) ], [Title | View]; render_cache_hit_rates(CacheHitInfo, Len) -> [HitCalls1W, HitCalls2W, HitCalls3W, HitCalls4W] = cache_hit_title_widths(), Title = ?render([ ?UNDERLINE, ?GRAY_BG, "IN|", ?W(" Hits/Calls", HitCalls1W), ?W("HitRate", 6), "IN|", ?W(" Hits/Calls", HitCalls2W), ?W("HitRate", 6), "IN|", ?W(" Hits/Calls", HitCalls3W), ?W("HitRate", 6), "IN|", ?W(" Hits/Calls", HitCalls4W), ?W("HitRate", 6) ]), [HitCallsRow1W, HitCallsRow2W, HitCallsRow3W, HitCallsRow4W] = cache_hit_row_widths(), Num = Len div 4, Rows = [ begin Seq2 = Seq1 + Num, Seq3 = Seq2 + Num, Seq4 = Seq3 + Num, {SeqStr1, Hit1, Call1, HitRateStr1} = get_cachehit_info(Seq1, CacheHitInfo), {SeqStr2, Hit2, Call2, HitRateStr2} = get_cachehit_info(Seq2, CacheHitInfo), {SeqStr3, Hit3, Call3, HitRateStr3} = get_cachehit_info(Seq3, CacheHitInfo), {SeqStr4, Hit4, Call4, HitRateStr4} = get_cachehit_info(Seq4, CacheHitInfo), ?render([ SeqStr1, ?W([Hit1, "/", Call1], HitCallsRow1W), ?W(HitRateStr1, 6), SeqStr2, ?W([Hit2, "/", Call2], HitCallsRow2W), ?W(HitRateStr2, 6), SeqStr3, ?W([Hit3, "/", Call3], HitCallsRow3W), ?W(HitRateStr3, 6), SeqStr4, ?W([Hit4, "/", Call4], HitCallsRow4W), ?W(HitRateStr4, 6) ]) end || Seq1 <- lists:seq(1, Num) ], [Title | Rows]. render_block_size_info(AverageBlockCurs, AverageBlockMaxes, SbcsToMbcsCurs, SbcsToMbcsMaxs) -> [AllocatorW, CurrentMbcsW, MaxMbcsW, CurrentSbcsW, MaxSbcsW, CurrentSbcsToMbcsW, MaxSbcsToMbcsW] = block_size_widths(), Title = ?render([ ?UNDERLINE, ?GRAY_BG, ?W("Allocator Type", AllocatorW), ?W("Current Mbcs", CurrentMbcsW), ?W("Max Mbcs", MaxMbcsW), ?W("Current Sbcs", CurrentSbcsW), ?W("Max Sbcs", MaxSbcsW), ?W("Current SbcsToMbcs", CurrentSbcsToMbcsW), ?W("Max SbcsToMbcs", MaxSbcsToMbcsW) ]), View = [ begin [Type, CMC, MMC, CSC, MSBC, CSTM, MSTM] = get_alloc( AllocKey, AverageBlockCurs, AverageBlockMaxes, SbcsToMbcsCurs, SbcsToMbcsMaxs ), ?render([ ?W(Type, AllocatorW), ?W(CMC, CurrentMbcsW), ?W(MMC, MaxMbcsW), ?W(CSC, CurrentSbcsW), ?W(MSBC, MaxSbcsW), ?W(CSTM, CurrentSbcsToMbcsW), ?W(MSTM, MaxSbcsToMbcsW) ]) end || AllocKey <- ?UTIL_ALLOCATORS ], [Title | View]. cache_hit_title_widths() -> observer_cli_lib:weighted_widths([20, 20, 20, 19], [1, 1, 1, 1]). cache_hit_row_widths() -> observer_cli_lib:weighted_widths([19, 19, 19, 18], [1, 1, 1, 1]). block_size_widths() -> observer_cli_lib:weighted_widths( [16, 16, 16, 16, 16, 19, 19], [0, 1, 1, 1, 1, 2, 2] ). get_alloc(Key, Curs, Maxes, STMCurs, STMMaxes) -> CurRes = proplists:get_value(Key, Curs), MaxRes = proplists:get_value(Key, Maxes), CurMbcs = proplists:get_value(mbcs, CurRes), CurSbcs = proplists:get_value(sbcs, CurRes), MaxMbcs = proplists:get_value(mbcs, MaxRes), MaxSbcs = proplists:get_value(sbcs, MaxRes), [ atom_to_list(Key), observer_cli_lib:to_byte(CurMbcs), observer_cli_lib:to_byte(MaxMbcs), observer_cli_lib:to_byte(CurSbcs), observer_cli_lib:to_byte(MaxSbcs), proplists:get_value(Key, STMCurs), proplists:get_value(Key, STMMaxes) ]. render_sys_info(Cmd) -> SysInfo = sys_info(Cmd), {Info, Stat} = info_fields(), SystemAndCPU = fill_info(Info, SysInfo), MemAndStatistics = fill_info(Stat, SysInfo), System = proplists:get_value("System and Architecture", SystemAndCPU), CPU = proplists:get_value("CPU's and Threads", SystemAndCPU), {_, _, Memory} = lists:keyfind("Memory Usage", 1, MemAndStatistics), {_, _, Statistics} = lists:keyfind("Statistics", 1, MemAndStatistics), render_sys_info(System, CPU, Memory, Statistics). render_sys_info(System, CPU, Memory, Statistics) -> [ SystemTitleW, SystemStateTitleW, CpuTitleW, CpuStateTitleW, MemoryTitleW, MemoryStateTitleW, StatisticsTitleW, StatisticsStateTitleW ] = sys_info_title_widths(), Title = ?render([ ?GRAY_BG, ?UNDERLINE, ?W("System/Architecture", SystemTitleW), ?W("State", SystemStateTitleW), ?W("CPU's and Threads", CpuTitleW), ?W("State", CpuStateTitleW), ?W("Memory Usage", MemoryTitleW), ?W("State", MemoryStateTitleW), ?W("Statistics", StatisticsTitleW), ?W("State", StatisticsStateTitleW) ]), [ SystemW, SystemStateW, CpuW, CpuStateW, MemoryW, MemoryValueW, MemoryPercentW, StatisticsW, StatisticsStateW ] = sys_info_row_widths(), NewSystem = [ begin {Key, Value} end || {Key, Value} <- System, Key =/= "Compiled for" andalso Key =/= "smp Support" ], [{_, TotalMem} | _R] = Memory, {bytes, TotalMemInt} = TotalMem, Row = [ begin {SysKey, SysVal} = lists:nth(Pos, NewSystem), {CpuKey, CpuVal} = lists:nth(Pos, CPU), {MemKey, MemVal} = lists:nth(Pos, Memory), {StatisticsKey, StatisticsVal} = lists:nth(Pos, Statistics), {bytes, MemValInt} = MemVal, Percent = observer_cli_lib:to_percent(MemValInt / TotalMemInt), ?render([ ?W(SysKey, SystemW), ?W(to_list(SysVal), SystemStateW), ?W(CpuKey, CpuW), ?W(to_list(CpuVal), CpuStateW), ?W(MemKey, MemoryW), ?W(observer_cli_lib:to_byte(MemValInt), MemoryValueW), ?W(Percent, MemoryPercentW), ?W(StatisticsKey, StatisticsW), ?W(to_list(StatisticsVal), StatisticsStateW) ]) end || Pos <- lists:seq(1, 6) ], [CompiledForW, CompiledValueW] = compiled_for_widths(), Compile = ?render([ ?UNDERLINE, ?W("compiled for", CompiledForW), ?W(to_list(proplists:get_value("Compiled for", System)), CompiledValueW) ]), [Title | (Row ++ [Compile])]. sys_info_title_widths() -> observer_cli_lib:weighted_widths( [22, 8, 23, 7, 11, 22, 11, 11], [0, 1, 0, 1, 0, 1, 0, 1] ). sys_info_row_widths() -> observer_cli_lib:weighted_widths( [22, 8, 23, 7, 11, 13, 6, 11, 11], [0, 1, 0, 1, 0, 1, 0, 0, 1] ). compiled_for_widths() -> observer_cli_lib:weighted_widths([22, 111], [0, 1]). sys_info(Cmd) -> MemInfo = try erlang:memory() of Mem -> Mem catch _:_ -> [] end, SchedulersOnline = erlang:system_info(schedulers_online), SchedulersAvailable = case erlang:system_info(multi_scheduling) of enabled -> SchedulersOnline; _ -> 1 end, [_, CmdValue | _] = string:split(os:cmd(Cmd), "\n", all), [CpuPsV, MemPsV, RssPsV, VszPsV] = case lists:filter(fun(Y) -> Y =/= [] end, string:split(CmdValue, " ", all)) of [] -> ["--", "--", "--", "--"]; [V1, V2, V3, V4] -> [V1, V2, list_to_integer(V3) * 1024, list_to_integer(V4) * 1024] end, {{_, Input}, {_, Output}} = erlang:statistics(io), [ {ps_cpu, CpuPsV ++ "%"}, {ps_mem, MemPsV ++ "%"}, {ps_rss, RssPsV}, {ps_vsz, VszPsV}, {io_input, Input}, {io_output, Output}, {logical_processors, erlang:system_info(logical_processors)}, {logical_processors_online, erlang:system_info(logical_processors_online)}, {logical_processors_available, erlang:system_info(logical_processors_available)}, {schedulers, erlang:system_info(schedulers)}, {schedulers_online, SchedulersOnline}, {schedulers_available, SchedulersAvailable}, {otp_release, erlang:system_info(otp_release)}, {version, erlang:system_info(version)}, {system_architecture, erlang:system_info(system_architecture)}, {kernel_poll, erlang:system_info(kernel_poll)}, {smp_support, erlang:system_info(smp_support)}, {threads, erlang:system_info(threads)}, {thread_pool_size, erlang:system_info(thread_pool_size)}, {wordsize_internal, erlang:system_info({wordsize, internal})}, {wordsize_external, erlang:system_info({wordsize, external})}, {alloc_info, alloc_info()} | MemInfo ]. alloc_info() -> Alloc = erlang:system_info(alloc_util_allocators), try erlang:system_info({allocator_sizes, Alloc}) of Allocators -> Allocators catch _:_ -> [] end. fill_info([{dynamic, Key} | Rest], Data) when is_atom(Key) -> case proplists:get_value(Key, Data) of undefined -> [undefined | fill_info(Rest, Data)]; {Str, Value} -> [{Str, Value} | fill_info(Rest, Data)] end; fill_info([{Str, Key} | Rest], Data) when is_atom(Key) -> case proplists:get_value(Key, Data) of undefined -> [undefined | fill_info(Rest, Data)]; Value -> [{Str, Value} | fill_info(Rest, Data)] end; fill_info([{Str, Attrib, Key} | Rest], Data) when is_atom(Key) -> case proplists:get_value(Key, Data) of undefined -> [undefined | fill_info(Rest, Data)]; Value -> [{Str, Attrib, Value} | fill_info(Rest, Data)] end; fill_info([{Str, {Format, Key}} | Rest], Data) when is_atom(Key) -> case proplists:get_value(Key, Data) of undefined -> [undefined | fill_info(Rest, Data)]; Value -> [{Str, {Format, Value}} | fill_info(Rest, Data)] end; fill_info([{Str, Attrib, {Format, Key}} | Rest], Data) when is_atom(Key) -> case proplists:get_value(Key, Data) of undefined -> [undefined | fill_info(Rest, Data)]; Value -> [{Str, Attrib, {Format, Value}} | fill_info(Rest, Data)] end; fill_info([{Str, SubStructure} | Rest], Data) when is_list(SubStructure) -> [{Str, fill_info(SubStructure, Data)} | fill_info(Rest, Data)]; fill_info([{Str, Attrib, SubStructure} | Rest], Data) -> [{Str, Attrib, fill_info(SubStructure, Data)} | fill_info(Rest, Data)]; fill_info([], _) -> []. to_list(Val) when is_integer(Val) -> integer_to_list(Val); to_list(Val) when is_atom(Val) -> atom_to_list(Val); to_list({bytes, Val}) -> observer_cli_lib:to_byte(Val); to_list(Val) -> Val. info_fields() -> Info = [ {"System and Architecture", [ {"System Version", otp_release}, {"Erts Version", version}, {"Compiled for", system_architecture}, {"Emulator Wordsize", wordsize_external}, {"Process Wordsize", wordsize_internal}, {"Smp Support", smp_support}, {"Thread Support", threads}, {"Async thread pool size", thread_pool_size} ]}, {"CPU's and Threads", [ {"Logical CPU's", logical_processors}, {"Online Logical CPU's", logical_processors_online}, {"Available Logical CPU's", logical_processors_available}, {"Schedulers", schedulers}, {"Online schedulers", schedulers_online}, {"Available schedulers", schedulers_available} ]} ], Stat = [ {"Memory Usage", right, [ {"Total", {bytes, total}}, {"Processes", {bytes, processes}}, {"Atoms", {bytes, atom}}, {"Binaries", {bytes, binary}}, {"Code", {bytes, code}}, {"Ets", {bytes, ets}} ]}, {"Statistics", right, [ {"ps -o pcpu", ps_cpu}, {"ps -o pmem", ps_mem}, {"ps -o rss", {bytes, ps_rss}}, {"ps -o vsz", {bytes, ps_vsz}}, {"Total IOIn", {bytes, io_input}}, {"Total IOOut", {bytes, io_output}} ]} ], {Info, Stat}. get_cachehit_info(Seq, CacheHitInfo) -> [{hit_rate, HitRate}, {hits, Hit}, {calls, Call}] = proplists:get_value( {instance, Seq}, CacheHitInfo ), SeqStr = lists:flatten(io_lib:format("\e[92m~2..0w| \e[0m", [Seq])), HitRateStr = observer_cli_lib:to_percent(HitRate), {SeqStr, integer_to_list(Hit), integer_to_list(Call), HitRateStr}.