defmodule Mix.Tasks.Arke.SeedProject do @moduledoc """ Seed a given project in the database using file data. ## Examples $ mix arke.seed_project --project my_project1 --p myproject2 $ mix arke.seed_project --all ## Command line options * `--project` - The id of the project to seed. It could be passed multiple times * `--all` - Seed all the project found in the database * `--format` -The format of the file used to import the data.One of: * `json` Default * `--persistence` - The persistence to use. One of: * `arke_postgres` - via https://github.com/elixir-ecto/postgrex (Default) """ use Mix.Task alias Arke.QueryManager alias Arke.LinkManager alias Arke.Utils.ErrorGenerator, as: Error alias Arke.Boundary.{ArkeManager} alias Arke.Core.Unit @decode_keys [:arke, :parameter, :group, :link] @supported_format ["json", "yml", "yaml"] @shortdoc "Seed a project" @persistence_repo ["arke_postgres"] @switches [ project: :string, all: :boolean, format: :string, persistence: :string, ] @aliases [ p: :project, A: :all, f: :format, ps: :persistence, ] @impl true def run(args) do case OptionParser.parse!(args, strict: @switches, aliases: @aliases) do {[], _opts}-> Mix.Tasks.Help.run(["arke.seed_project"]) {opts, []} -> persistence = parse_persistence!(opts[:persistence] || "arke_postgres") app_to_start(persistence) ++ [:arke] |> Enum.each(&Application.ensure_all_started/1) repo_module = Application.get_env(:arke, :persistence)[String.to_atom(persistence)][:repo] Mix.shell().info("--- Starting repo --- ") case start_repo(repo_module) do {:ok, pid} -> opts |> parse_file() Process.exit(pid, :normal) :ok {:error, _} -> opts |> parse_file() :ok end end end defp app_to_start("arke_postgres"), do: [:ecto_sql, :postgrex] defp parse_persistence!(ps) when ps in @persistence_repo, do: ps defp parse_persistence!(ps), do: Mix.raise("Invalid persistence: `#{ps}`\nSupported persistence are: #{Enum.join(@persistence_repo, " | ")}") defp check_file(_arke_id,project, []), do: nil defp check_file(arke_id,project, data) do unless Mix.env() == :test do {:ok, datetime} = Arke.Utils.DatetimeHandler.now(:datetime) |> Arke.Utils.DatetimeHandler.format("{ISO:Basic:Z}") dir_path = "log/arke_seed_project/#{project}" path = "#{dir_path}/#{datetime}_#{to_string(arke_id)}.log" Mix.shell().info("--- Writing errors to #{path} --- ") File.mkdir_p!(dir_path) write_log_to_file(path, data) end end defp write_log_to_file(path, data) do {:ok, body} = Jason.encode(data) {:ok, file} = File.open(path, [:append]) IO.write(file, body) File.close(file) end defp start_repo(nil), do: Mix.raise("Invalid repo module in arke configuration. Please provide a valid module accordingly to the persistence supported") # this is for arke_postgres defp start_repo(repo_module) do repo_module.start_link() end defp parse_file(opts) do Mix.shell().info("--- Parsing registry files --- ") format = opts[:format] || "json" check_format!(format) all = opts[:all] || false # get core file to decode (arke) and append all other arke_deps registry files core_registry = arke_registry("arke", format,"all") core_data = parse(core_registry,format) arke_deps_registry = get_arke_deps_registry(format,"all") arke_deps_data = parse(arke_deps_registry,format) Mix.shell().info("--- Get core data ---") core_parameter = Map.get(core_data,:parameter, []) ++ Map.get(arke_deps_data, :parameter, []) core_arke = Map.get(core_data,:arke, []) ++ Map.get(arke_deps_data, :arke, []) core_group = Map.get(core_data,:group, []) ++ Map.get(arke_deps_data, :group, []) core_link = Map.get(core_data,:link, []) ++ Map.get(arke_deps_data, :link, []) # start core manager before create everything Mix.shell().info("--- Starting core managers --- ") error_parameter_manager = Arke.handle_manager(core_parameter,:arke_system,:parameter) error_arke_manager = Arke.handle_manager(core_arke,:arke_system,:arke) error_group_manager = Arke.handle_manager(core_group,:arke_system,:group) check_file("parameter_manager","arke_system",error_parameter_manager) check_file("arke_manager","arke_system",error_arke_manager) check_file("group_manager","arke_system",error_group_manager) input_project = String.to_atom(opts[:project]) || :arke_system project_list = get_project(input_project, all) Enum.each(project_list, fn project -> if to_string(project) == "arke_system" do write_data(project,core_data,core_parameter,core_arke,core_group,core_link) else file_list = Path.wildcard("./lib/registry/*.#{format}") shared_arke_list = arke_registry("arke", format,"shared") shared_arke_deps_list = get_arke_deps_registry(format,"shared") raw_data = parse(shared_arke_list++shared_arke_deps_list++file_list,format) parameter_list = Map.get(raw_data, :parameter, []) arke_list = Map.get(raw_data, :arke, []) group_list = Map.get(raw_data, :group, []) link_list = Map.get(raw_data, :link, []) write_data(project,core_data,parameter_list,arke_list,group_list,link_list) end end) end defp write_data(input_project,core_data,parameter_list,arke_list,group_list,link_list) do project_key= to_string(input_project) # if the project is arke_system the managers have already been started so skip unless project_key =="arke_system" do Mix.shell().info("--- Parsing registry files --- ") error_parameter_manager = Arke.handle_manager(Map.get(core_data,:parameter, []),input_project,:parameter) error_arke_manager = Arke.handle_manager(Map.get(core_data,:arke, []),input_project,:arke) error_group_manager = Arke.handle_manager(Map.get(core_data,:group, []),input_project,:group) check_file("parameter_manager",project_key,error_parameter_manager) check_file("arke_manager",project_key,error_arke_manager) check_file("group_manager",project_key,error_group_manager) end error_parameter = handle_parameter(parameter_list, input_project,[]) error_arke = handle_arke(arke_list, input_project,[]) error_group = handle_group(group_list, input_project,[]) error_link = handle_link(link_list, input_project,[]) check_file("parameter",project_key,error_parameter) check_file("arke",project_key,error_arke) check_file("group",project_key,error_group) check_file("link",project_key,error_link) end defp arke_registry(package_name,format,type) do # Get arke's dependecies based on the env path. env_var = System.get_env() folder_path = get_folder(type,format) case Enum.find(env_var,fn {k,_v}-> String.contains?(String.downcase(k), "ex_dep_#{package_name}_path") end) do {_package_name, ""} -> Path.wildcard("./**/arke*/**/registry/#{folder_path}") {_package_name, local_path} -> Path.wildcard("#{local_path}/lib/registry/#{folder_path}") nil -> Path.wildcard("./**/arke*/**/registry/#{folder_path}") end end defp get_folder("shared",format),do: "shared/*.#{format}" defp get_folder("system",format),do: "system/*.#{format}" defp get_folder("all",format),do: "**/*.#{format}" defp get_project(_input_project, true) do QueryManager.filter_by(arke_id: :arke_project, project: :arke_system) |> Enum.map( fn unit -> to_string(unit.id) end) end defp get_project(input_project, _all), do: [input_project] defp check_format!(format) when format in @supported_format, do: format defp check_format!(format), do: Mix.raise("Invalid format: `#{format}`\nSupported format are: #{Enum.join(@supported_format, " | ")}") # get all the registry file for all the arke_deps except arke itself which is used alone defp get_arke_deps_registry(format,type) do Enum.reduce(Mix.Project.config() |> Keyword.get(:deps, []),[], fn tuple,acc -> name = List.first(Tuple.to_list(tuple)) if name != :arke and String.contains?(to_string(name),"arke") do arke_registry(to_string(name),format,type) ++ acc else acc end end) end defp parse(file_list,format,file_data \\ %{}) defp parse([filename | t],"json"=format, data) do try do body = File.read!(filename) json = Jason.decode!(body, keys: :atoms) new_data = Enum.reduce(@decode_keys, %{}, fn key, acc -> Map.put(acc, key, Map.get(data, key, []) ++ Map.get(json, key, [])) end) parse(t, format,new_data) rescue err in Jason.DecodeError -> %{data: data, token: token, position: position} = err Mix.raise("Json error in: #{filename}.\n Position: #{position}\n token: #{token}\n data: #{data}") err in File.Error -> %{reason: reason}= err Mix.raise("Error open file: #{filename}. \n Reason: #{reason}") end end # tutti i file parsati quindi proseguire defp parse([],format, data), do: data defp handle_parameter([%{id: id, label: nil} = current | t], project, error), do: handle_parameter([Map.put(current, :label, String.capitalize(id)) | t], project, error) defp handle_parameter( [%{id: id, type: type} = current | t], project, error ) do Mix.shell().info("--- Creating parameter #{id} --- ") with nil <- QueryManager.get_by(id: id, project: project, arke_id: type), %Unit{} = model <- ArkeManager.get(String.to_atom(type), project), {:ok, _unit} <- QueryManager.create(project, model, current) do handle_parameter(t, project, error) else nil -> handle_parameter(t, project, parse_error(create_error(:parameter, "manager does not exists for: `#{id}`") , error)) %Unit{} -> handle_parameter(t, project, parse_error(create_error(:parameter, "Record already exists in db for: `#{id}`") , error)) {:error, err} -> handle_parameter(t, project, parse_error(err, error,id)) _err -> handle_parameter(t, project, parse_error(create_error(:parameter, "Something went wrong for: `#{id}`") ,error)) end end defp handle_parameter([_current | t], project, error) do handle_parameter(t, project, parse_error(create_error(:parameter, "Missing parameter `id` or `type`") , error)) end defp handle_parameter([], _project, error), do: error defp handle_arke([%{id: id, label: nil} = current | t], project, error), do: handle_arke([Map.put(current, "label", String.capitalize(id)) | t], project, error) defp handle_arke( [%{id: id} = current | t], project, error ) do Mix.shell().info("--- Creating arke #{id} --- ") {parameter,new_data} = Map.pop(current, :parameters, []) #aggiungere try do block with nil <- QueryManager.get_by(id: id, project: project, arke_id: "arke"), %Unit{} = model <- ArkeManager.get(:arke, project), {:ok, unit} <- QueryManager.create(project, model, new_data), link_parameter_error <- link_parameter(parameter, unit, project) do if length(link_parameter_error) == 0 do handle_arke(t, project, error) else handle_arke(t, project, [%{"#{id}_parameter_association": link_parameter_error}|error]) end else nil -> handle_arke(t, project, parse_error(create_error(:arke, "manager does not exists for: `#{id}`"), error)) %Unit{}=_unit -> handle_arke(t, project, parse_error(create_error(:arke, "Record already exists in db for: `#{id}`") , error)) {:error, err} -> handle_arke(t, project, [err | error]) end end defp handle_arke([], _project, error), do: error defp handle_group( [%{id: id} = current | t], project,error ) do Mix.shell().info("--- Creating group #{id} --- ") with nil <- QueryManager.get_by(id: id, project: project, arke_id: :group), %Unit{} = model <- ArkeManager.get(:group, project), {:ok, unit} <- QueryManager.create(project, model, current), error_group <- add_arke_to_group(unit, project) do handle_group(t, project, error ++ error_group) else nil -> handle_group(t, project, parse_error(create_error(:arke, "manager does not exists for: `#{id}`"), error)) %Unit{}=_unit -> handle_group(t, project, parse_error(create_error(:arke, "Record already exists in db for: `#{id}`"),error)) {:error, err} -> handle_group(t, project, [err | error]) end end defp handle_group([], _project, error), do: error defp handle_link(_data,_project, _error \\ []) defp handle_link( [%{type: type, parent: parent, child: child} = current | t], project, error ) do Mix.shell().info("--- Creating link from #{parent} to #{child} --- ") case LinkManager.add_node( project, parent, child, type, Map.get(current, :metadata, %{}) ) do {:ok, _unit} -> handle_link(t, project, error) {:error, link_error} -> handle_link( t, project, [link_error | error] ) end end defp handle_link([current | t], project, error),do: handle_link(t, project, parse_error(create_error(:link, "invalid parameters for #{current}}"),error)) defp handle_link([], _project, error), do: error defp link_parameter(p_list, arke, project) do Mix.shell().info("--- Adding parameters to arke #{arke.id} --- ") param_link = Enum.reduce(p_list, [], fn parameter, acc -> [ %{ child: Map.get(parameter, :id), parent: to_string(arke.id), metadata: Map.get(parameter, :metadata, %{}), type: "parameter" } | acc ] end) handle_link(param_link, project, []) end defp add_arke_to_group(group, project) do Mix.shell().info("--- Adding arkes to group #{group.id} --- ") arke_list = Map.get(group, :arke_list, []) group_link = Enum.reduce(arke_list, [], fn arke, acc -> [ %{ parent: to_string(group.id), child: to_string(Map.get(arke, :id)), metadata: Map.get(arke, :metadata, %{}), type: "group" } | acc ] end) handle_link(group_link, project, []) end defp create_error(context,msg) do {:error,msg} = Error.create(context,msg) msg end defp parse_error(error_message, error_accumulator) when is_list(error_message), do: error_message ++error_accumulator defp parse_error(error_message, error_accumulator), do: [error_message | error_accumulator] defp parse_error(error_message, error_accumulator,id), do: [%{create: id, error: error_message} | error_accumulator] end