defmodule Mix.Tasks.Analytics3i.Chart do use Mix.Task require Logger @shortdoc "Chart" def run(_) do {:ok, _} = Application.ensure_all_started(:analytics_3i) Logger.add_backend {LoggerFileBackend, :debug} Logger.configure_backend {LoggerFileBackend, :debug}, path: Path.expand("../../../log/debug.log", __DIR__), format: "$time $metadata[$level] $message\n" otrasl_list = load_yaml("data/otrasl.yml") st_list = load_yaml("data/st.yml") template = load_json("data/charts.json") graph = %{ "balloonText" => "[[title]] of [[otrasl]]:[[value]]", "color" => "#ffffff", "fillAlphas" => 0.9, "fontSize" => 10, "labelText" => "[[value]]", "id" => "AmGraph-1", "title" => "st 1", "type" => "column", "valueField" => "st-1" } graphPercent = Map.merge(graph, %{ "balloonText" => "[[title]], [[category]]
[[value]] ([[percents]]%)", "labelText" => "[[value]] / [[percents]]%" }) # result = # Path.wildcard("output/*.csv") # |> Enum.reduce(%{graphs: %{}, dataProvider: %{}}, fn (file_name, acc) -> result = Path.wildcard("output/*.csv") |> Enum.reduce(%{}, fn (file_name, acc) -> {otrasl, st} = otrasl_st(file_name) store = read_csv!(file_name) |> Enum.reduce(%{}, fn raw_row, acc -> {:ok, row} = raw_row {:ok, d, _} = DateTime.from_iso8601(row["created_date"]) Map.put(acc, DateTime.to_unix(d), Map.merge(row, %{"created_date" => d})) end) store_keys = store |> Map.keys() |> Enum.sort() first = store_keys |> List.first() last = store_keys |> List.last() {:ok, first_date} = first |> DateTime.from_unix() {:ok, last_date} = last |> DateTime.from_unix() {quarts, first_quart} = quart_diff(last_date, first_date) years = (first_date.year..last_date.year) |> Enum.reduce(%{}, fn year, acc -> Map.put(acc, year, %{}) end) quart_store = store_keys |> Enum.reduce(%{quarts: years, quart: first_quart, quart_num: 1}, fn key, acc -> row = Map.get(store, key) {:ok, date} = DateTime.from_unix(key) quart = if Enum.empty?(acc[:quarts]) do acc[:quart] else detect_quart(date) end new_acc = if acc[:quart] != quart do %{quart_num: acc[:quart_num] + 1, quart: quart} else %{quart_num: acc[:quart_num], quart: acc[:quart]} end year = Map.get(acc[:quarts], date.year) unless Map.has_key?(year, quart) do year = Map.merge(year, %{quart => []}) end acc |> Map.merge(new_acc) |> Map.merge(%{quarts: Map.put(acc[:quarts], date.year, Map.merge(year, %{quart => List.insert_at(year[quart], -1, 1)}))}) end) new_acc = quart_store[:quarts] |> Enum.reduce(acc, fn {y, v}, yacc -> yacc = generate_year(yacc, y) Map.merge(yacc, %{y => generate_quarts(yacc, y, v, otrasl, st, otrasl_list, st_list)}) end) end) result |> Enum.each(fn {year, l} -> l |> Enum.each(fn {quart, data} -> graph_maker(data, year, quart, template, graph, graphPercent, st_list) end) end) end defp generate_year(acc, y) do if Map.has_key?(acc, y) do acc else Map.merge(acc, %{y => %{}}) end end defp generate_quarts(year_acc, y, v, otrasl, st, otrasl_list, st_list) do v |> Enum.reduce(year_acc[y], fn {q, l}, acc -> if Map.has_key?(acc, q) do Map.merge(acc, %{q => data_providers(acc[q], l, otrasl, st, otrasl_list, st_list)}) else Map.merge(acc, %{q => data_providers(%{dataProvider: %{}, graphs: %{}}, l, otrasl, st, otrasl_list, st_list)}) end end) end defp data_providers(acc, list, otrasl, st, otrasl_list, st_list) do count = list |> Enum.count() otrasl_data = Map.get(acc[:dataProvider], otrasl, %{name: otrasl_list[otrasl]["name"], data: %{}}) otrasl_data = Map.put(otrasl_data, :data, Map.put(otrasl_data[:data], st, count - 1)) acc = Map.put(acc, :graphs, Map.put(acc[:graphs], st, st_list[st]["name"])) Map.put(acc, :dataProvider, Map.put(acc[:dataProvider], otrasl, otrasl_data)) end defp graph_maker(result, year, quart, template, graph, graphPercent, st_list) do graphs = result[:graphs] |> Enum.map(fn {label, name} -> Map.merge(graph, %{"title" => name, "valueField" => label, "id" => "AmGraph-#{label}"}) end) graphsPercent = result[:graphs] |> Enum.map(fn {label, name} -> Map.merge(graphPercent, %{"title" => name, "valueField" => label, "id" => "AmGraph-#{label}"}) end) dataProvider = result[:dataProvider] |> Enum.map(fn {label, value} -> Map.put(value[:data], :otrasl, value[:name]) end) dataValues = result[:dataProvider] |> Enum.map(fn {_, value} -> value[:data] |> Enum.reduce(%{}, fn ({k, v}, acc) -> Map.put(acc, st_list[k]["name"], v) end) |> Map.put(:Label, value[:name]) end) # dataColumns = # st_list # |> Enum.map(fn {_, value} -> # value[:name] # end) # dataColumns = ["Label"] ++ dataColumns valueAxes = List.first(template["valueAxes"]) |> Map.merge(%{"title" => "% от числа публикаций", "stackType" => "100%"}) titles = List.first(template["titles"]) |> Map.merge(%{"text" => "СТ в СМИ по отраслям (%) #{year} Q#{quart}"}) data = Map.merge(template, %{"dataProvider" => dataProvider, "graphs" => graphsPercent, "valueAxes" => [valueAxes], "titles" => [titles]}) |> IO.inspect() write_json(data, "output/graph_percent.#{year}.#{quart}.json") valueAxes = List.first(template["valueAxes"]) |> Map.merge(%{"title" => "Число публикаций"}) titles = List.first(template["titles"]) |> Map.merge(%{"text" => "СТ в СМИ по отраслям #{year} Q#{quart}"}) data = Map.merge(template, %{"dataProvider" => dataProvider, "graphs" => graphs, "valueAxes" => [valueAxes], "titles" => [titles]}) |> IO.inspect() write_json(data, "output/graph_regular.#{year}.#{quart}.json") write_csv!(Map.keys(List.first(dataValues)), dataValues, "#{year}.#{quart}") end defp otrasl_st(file_name) do info = String.split(Path.basename(file_name), ~r{[_.]}) otrasl_name_size = Enum.count(info) - 3 otrasl_name_size = if Enum.at(info, otrasl_name_size) == "new", do: otrasl_name_size - 1, else: otrasl_name_size otrasl_name = (1..otrasl_name_size) |> Enum.map(fn x -> Enum.at(info, x) end) |> Enum.join("_") st_name_size = Enum.count(info) - otrasl_name_size - 2 st_name = (otrasl_name_size + 1..otrasl_name_size + st_name_size) |> Enum.map(fn x -> Enum.at(info, x) end) |> Enum.join("_") {otrasl_name, st_name} end defp load_yaml(path) do path = File.cwd! |> Path.join(path) {:ok, data} = YamlElixir.read_from_file path data end defp load_json(path) do path = File.cwd! |> Path.join(path) {:ok, content} = File.read(path) {:ok, data} = Poison.decode(content) data end defp detect_quart(dt) do round(Float.ceil(dt.month/3)) end defp quart_diff(dt1, dt2) do years_diff = (dt2.year - dt1.year) months_diff = if years_diff > 0 do dt2.month + (12-dt1.month) + (years_diff - 1) * 12 else dt2.month - dt1.month end {Integer.floor_div(months_diff, 3), detect_quart(dt1)} end defp write_csv!(keys, values, suffix \\ "") do File.open("output/graph_dots.#{suffix}.csv.txt", [:write, :utf8], fn(file) -> values |> CSV.encode(headers: keys, separator: ?;) |> Enum.each(&IO.write(file, &1)) end) end defp read_csv!(path) do keys = path |> File.stream!() |> Enum.take(1) |> List.first() |> String.split(",") |> Enum.map(fn k -> String.to_atom(String.trim(k)) end) path |> File.stream! |> CSV.decode(headers: true, separator: ?,) end defp write_json(data, path) do path = File.cwd! |> Path.join(path) {:ok, content} = Poison.encode(data) :ok = File.write(path, content) end # # defp load_txt(path) do # path = File.cwd! |> Path.join(path) # {:ok, content} = File.read(path) # {:ok, data} = Poison.decode(content) # # content # end end