defmodule Mix.Tasks.Github.Gen do @moduledoc """ Generates the full `Noizu.Github` REST client from the OpenAPI description in `docs/github-api/api.github.com.json`. Emits, following the project's existing conventions: * one `Noizu.Github.Api.` module per spec category, with one function per operation, dispatching through `Noizu.Github.api_call/5` * one struct module `Noizu.Github.` per object schema, with a permissive `from_json/2` * typed list wrappers `Noizu.Github.Collection.` plus the generic `Noizu.Github.Collection` / `Noizu.Github.Raw` fallbacks Usage: mix github.gen All generated files carry a "do not edit" banner and live under `lib/api/`. Re-run after updating the vendored spec. """ use Mix.Task @shortdoc "Generate the Noizu.Github client from the OpenAPI spec" @spec_path "docs/github-api/api.github.com.json" @api_root "lib/api" @structs_dir "lib/api/structs" @banner "# Generated by `mix github.gen` from docs/github-api/api.github.com.json.\n# Do not edit by hand; re-run the task instead.\n" @impl true def run(_args) do spec = load_spec() schemas = spec["components"]["schemas"] || %{} params = spec["components"]["parameters"] || %{} ctx = build_context(schemas, params) clean_output() struct_count = emit_structs(ctx) emit_runtime_wrappers() {wrapper_keys, op_count, cat_count, skipped} = emit_api_modules(spec, ctx) emit_collection_wrappers(wrapper_keys, ctx) Mix.shell().info(""" github.gen complete: categories: #{cat_count} operations: #{op_count} struct schemas: #{struct_count} list wrappers: #{MapSet.size(wrapper_keys)} skipped operations:#{length(skipped)} """) unless skipped == [] do Mix.shell().info("Skipped operations:\n" <> Enum.map_join(skipped, "\n", &(" - " <> &1))) end end # ---------------------------------------------------------------------------- # Spec loading # ---------------------------------------------------------------------------- defp load_spec do path = Path.join(File.cwd!(), @spec_path) unless File.exists?(path) do Mix.raise("OpenAPI spec not found at #{@spec_path}") end path |> File.read!() |> Jason.decode!() end # ---------------------------------------------------------------------------- # Context: deterministic name maps + structability for every schema # ---------------------------------------------------------------------------- defp build_context(schemas, params) do # First pass: deterministic module name per schema key, resolving collisions. {modules, _} = schemas |> Map.keys() |> Enum.sort() |> Enum.reduce({%{}, MapSet.new()}, fn key, {acc, used} -> base = "Noizu.Github." <> camelize(key) name = dedupe(base, used) {Map.put(acc, key, name), MapSet.put(used, name)} end) structable = schemas |> Enum.filter(fn {_k, schema} -> structable?(schema, schemas) end) |> Enum.map(&elem(&1, 0)) |> MapSet.new() %{ schemas: schemas, params: params, modules: modules, structable: structable } end defp dedupe(base, used) do if MapSet.member?(used, base) do Enum.reduce_while(1..1000, nil, fn i, _ -> cand = base <> Integer.to_string(i) if MapSet.member?(used, cand), do: {:cont, nil}, else: {:halt, cand} end) else base end end # A schema is "structable" (gets its own struct) when it resolves to an object # with named properties (directly, or via allOf composition). defp structable?(schema, schemas, seen \\ MapSet.new()) defp structable?(%{"$ref" => ref}, schemas, seen) do key = ref_key(ref) if MapSet.member?(seen, key) do false else case schemas[key] do nil -> false s -> structable?(s, schemas, MapSet.put(seen, key)) end end end defp structable?(schema, schemas, seen) when is_map(schema) do cond do is_map(schema["properties"]) and map_size(schema["properties"]) > 0 -> true is_list(schema["allOf"]) -> Enum.any?(schema["allOf"], &structable?(&1, schemas, seen)) true -> false end end defp structable?(_, _, _), do: false # ---------------------------------------------------------------------------- # Output cleanup # ---------------------------------------------------------------------------- defp clean_output do root = Path.join(File.cwd!(), @api_root) if File.dir?(root) do File.rm_rf!(root) end File.mkdir_p!(Path.join(File.cwd!(), @structs_dir)) end # ---------------------------------------------------------------------------- # Struct emission # ---------------------------------------------------------------------------- defp emit_structs(ctx) do ctx.structable |> Enum.sort() |> Enum.reduce(0, fn key, count -> schema = ctx.schemas[key] module = ctx.modules[key] props = collect_properties(schema, ctx.schemas) if map_size(props) == 0 do count else code = struct_module(module, props, ctx) write_struct_file(key, code) count + 1 end end) end # Merge properties across allOf composition (deduped by name). defp collect_properties(schema, schemas, seen \\ MapSet.new()) defp collect_properties(%{"$ref" => ref}, schemas, seen) do key = ref_key(ref) if MapSet.member?(seen, key) do %{} else collect_properties(schemas[key] || %{}, schemas, MapSet.put(seen, key)) end end defp collect_properties(schema, schemas, seen) when is_map(schema) do direct = schema["properties"] || %{} composed = (schema["allOf"] || []) |> Enum.reduce(%{}, fn member, acc -> Map.merge(acc, collect_properties(member, schemas, seen)) end) Map.merge(composed, direct) end defp collect_properties(_, _, _), do: %{} defp struct_module(module, props, ctx) do field_names = props |> Map.keys() |> Enum.sort() defstruct_fields = field_names |> Enum.map(&(" " <> atom_literal(&1))) |> Enum.join(",\n") assignments = field_names |> Enum.map(fn name -> " " <> field_assignment(name, props[name], ctx) end) |> Enum.join(",\n") """ #{@banner} defmodule #{module} do @moduledoc false defstruct [ #{defstruct_fields} ] def from_json(json, headers \\\\ []) def from_json(nil, _headers), do: nil def from_json(list, headers) when is_list(list) do Enum.map(list, &from_json(&1, headers)) end def from_json(json, _headers) when is_map(json) do %__MODULE__{ #{assignments} } end def from_json(other, _headers), do: other end """ end # Emit a single `field: ` line based on the property schema. defp field_assignment(name, prop, ctx) do getter = "Map.get(json, #{atom_literal(name)})" key = key_literal(name) decoder = case decode_kind(prop, ctx) do {:ref, module} -> "#{module}.from_json(#{getter})" {:list, module} -> "Enum.map(#{getter} || [], &#{module}.from_json(&1))" :passthrough -> getter end "#{key} #{decoder}" end # Decide how a property value should be decoded. defp decode_kind(%{"$ref" => ref}, ctx) do key = ref_key(ref) if MapSet.member?(ctx.structable, key) do {:ref, ctx.modules[key]} else :passthrough end end defp decode_kind(%{"type" => "array", "items" => %{"$ref" => ref}}, ctx) do key = ref_key(ref) if MapSet.member?(ctx.structable, key) do {:list, ctx.modules[key]} else :passthrough end end # allOf wrapping a single ref (common nullable pattern). defp decode_kind(%{"allOf" => [%{"$ref" => ref}]}, ctx) do key = ref_key(ref) if MapSet.member?(ctx.structable, key) do {:ref, ctx.modules[key]} else :passthrough end end defp decode_kind(_prop, _ctx), do: :passthrough defp write_struct_file(key, code) do path = Path.join([File.cwd!(), @structs_dir, snake(key) <> ".ex"]) write!(path, code) end # ---------------------------------------------------------------------------- # Runtime wrappers: generic Collection + Raw # ---------------------------------------------------------------------------- defp emit_runtime_wrappers do raw = """ #{@banner} defmodule Noizu.Github.Raw do @moduledoc \"\"\" Generic passthrough result for endpoints without a dedicated schema (inline objects, unions, empty `204` bodies). Carries the decoded body in `:data` and any pagination links in `:links`. \"\"\" defstruct [:data, :links] def from_json(json, headers \\\\ []) def from_json(json, headers) do %__MODULE__{data: json, links: Noizu.Github.extract_links(headers)} end end """ collection = """ #{@banner} defmodule Noizu.Github.Collection do @moduledoc \"\"\" Generic list result. Wraps a bare array (or a `{total_count, items}` search envelope) of untyped items together with pagination links. \"\"\" defstruct [:items, :complete, :total, :links] def from_json(json, headers \\\\ []) def from_json(list, headers) when is_list(list) do %__MODULE__{ items: list, complete: true, total: length(list), links: Noizu.Github.extract_links(headers) } end def from_json(%{items: items} = env, headers) when is_list(items) do %__MODULE__{ items: items, complete: !Map.get(env, :incomplete_results, false), total: Map.get(env, :total_count, length(items)), links: Noizu.Github.extract_links(headers) } end def from_json(other, headers) do %__MODULE__{items: other, links: Noizu.Github.extract_links(headers)} end end """ write!(Path.join([File.cwd!(), @structs_dir, "raw.ex"]), raw) write!(Path.join([File.cwd!(), @structs_dir, "collection.ex"]), collection) end defp emit_collection_wrappers(wrapper_keys, ctx) do dir = Path.join([File.cwd!(), @structs_dir, "collection"]) File.mkdir_p!(dir) Enum.each(wrapper_keys, fn key -> item = ctx.modules[key] module = "Noizu.Github.Collection." <> camelize(key) code = """ #{@banner} defmodule #{module} do @moduledoc false defstruct [:items, :complete, :total, :links] def from_json(json, headers \\\\ []) def from_json(list, headers) when is_list(list) do %__MODULE__{ items: Enum.map(list, &#{item}.from_json(&1)), complete: true, total: length(list), links: Noizu.Github.extract_links(headers) } end def from_json(%{items: items} = env, headers) when is_list(items) do %__MODULE__{ items: Enum.map(items, &#{item}.from_json(&1)), complete: !Map.get(env, :incomplete_results, false), total: Map.get(env, :total_count, length(items)), links: Noizu.Github.extract_links(headers) } end def from_json(other, headers) do %__MODULE__{items: other, links: Noizu.Github.extract_links(headers)} end end """ write!(Path.join(dir, snake(key) <> ".ex"), code) end) end # ---------------------------------------------------------------------------- # API module emission # ---------------------------------------------------------------------------- defp emit_api_modules(spec, ctx) do operations = collect_operations(spec, ctx) by_category = Enum.group_by(operations, & &1.category) wrapper_keys = operations |> Enum.flat_map(fn op -> if op.wrapper_key, do: [op.wrapper_key], else: [] end) |> MapSet.new() skipped = operations |> Enum.filter(& &1.skipped) |> Enum.map(& &1.operation_id) Enum.each(by_category, fn {category, ops} -> emit_category(category, ops) end) op_count = Enum.count(operations, &(not &1.skipped)) {wrapper_keys, op_count, map_size(by_category), skipped} end defp collect_operations(spec, ctx) do methods = ~w(get put post delete patch) for {path, item} <- spec["paths"], {method, op} <- item, method in methods, is_map(op) do path_level_params = item["parameters"] || [] build_operation(path, method, op, path_level_params, ctx) end |> Enum.reject(&is_nil/1) end defp build_operation(path, method, op, path_level_params, ctx) do operation_id = op["operationId"] if is_nil(operation_id) do nil else {category, verb} = split_operation_id(operation_id) all_params = (path_level_params ++ (op["parameters"] || [])) |> Enum.map(&resolve_param(&1, ctx)) |> Enum.reject(&is_nil/1) path_params = Enum.filter(all_params, &(&1["in"] == "path")) query_params = Enum.filter(all_params, &(&1["in"] == "query")) {result_model, wrapper_key} = result_model(op, ctx) %{ operation_id: operation_id, category: category, function: function_name(verb), method: method, path: path, path_params: path_params, query_params: query_params, has_body: not is_nil(op["requestBody"]), result_model: result_model, wrapper_key: wrapper_key, summary: op["summary"], doc_url: get_in(op, ["externalDocs", "url"]), skipped: false } end end defp resolve_param(%{"$ref" => ref}, ctx) do key = ref_key(ref) ctx.params[key] end defp resolve_param(param, _ctx) when is_map(param), do: param defp resolve_param(_, _), do: nil # Choose the success result model + (optional) list wrapper item key. defp result_model(op, ctx) do responses = op["responses"] || %{} status = ["200", "201", "202", "203", "204"] |> Enum.find(fn s -> Map.has_key?(responses, s) end) schema = status && get_in(responses, [status, "content", "application/json", "schema"]) classify_schema(schema, ctx) end defp classify_schema(nil, _ctx), do: {"Noizu.Github.Raw", nil} defp classify_schema(%{"$ref" => ref}, ctx) do key = ref_key(ref) if MapSet.member?(ctx.structable, key) do {ctx.modules[key], nil} else {"Noizu.Github.Raw", nil} end end defp classify_schema(%{"type" => "array", "items" => %{"$ref" => ref}}, ctx) do key = ref_key(ref) if MapSet.member?(ctx.structable, key) do {"Noizu.Github.Collection." <> camelize(key), key} else {"Noizu.Github.Collection", nil} end end defp classify_schema(%{"type" => "array"}, _ctx), do: {"Noizu.Github.Collection", nil} defp classify_schema(_schema, _ctx), do: {"Noizu.Github.Raw", nil} defp emit_category(category, ops) do module = "Noizu.Github.Api." <> camelize(category) dir = Path.join([File.cwd!(), @api_root, snake(category)]) File.mkdir_p!(dir) # Guard against duplicate function/arity within a category. {functions, _used} = ops |> Enum.sort_by(& &1.operation_id) |> Enum.map_reduce(MapSet.new(), fn op, used -> {fun, name} = render_function(op, used) {fun, MapSet.put(used, name)} end) body = Enum.join(functions, "\n") code = """ #{@banner} defmodule #{module} do @moduledoc \"\"\" GitHub `#{category}` API. \"\"\" import Noizu.Github #{body} end """ write!(Path.join(dir, snake(category) <> ".ex"), code) end defp render_function(op, used) do # Positional path params (snake), excluding owner/repo which come from options. positional = op.path_params |> Enum.map(& &1["name"]) |> Enum.reject(&(&1 in ["owner", "repo"])) |> Enum.map(&snake/1) has_owner = path_has?(op.path, "owner") has_repo = path_has?(op.path, "repo") write? = op.method in ["post", "put", "patch"] or (op.method == "delete" and op.has_body) arg_list = positional ++ if(write?, do: ["body"], else: []) ++ ["options \\\\ nil"] base_name = unique_function(op.function, length(arg_list_atoms(positional, write?)), used) signature = "#{base_name}(#{Enum.join(arg_list, ", ")})" owner_repo = [ has_owner && " owner = repo_owner(options)", has_repo && " repo = repo_name(options)" ] |> Enum.filter(& &1) |> Enum.join("\n") url_expr = build_url(op.path, op.query_params) body_expr = if write?, do: "body", else: "%{}" doc = function_doc(op) lines = [ doc, " def #{signature} do", owner_repo != "" && owner_repo, " url = #{url_expr}", " body = #{body_expr}", " api_call(:#{op.method}, url, body, #{op.result_model}, options)", " end" ] |> Enum.filter(& &1) |> Enum.join("\n") {lines <> "\n", base_name} end defp arg_list_atoms(positional, write?), do: positional ++ if(write?, do: ["body"], else: []) defp unique_function(name, _arity, used) do if MapSet.member?(used, name) do Enum.reduce_while(1..1000, name, fn i, _ -> cand = "#{name}_v#{i}" if MapSet.member?(used, cand), do: {:cont, cand}, else: {:halt, cand} end) else name end end defp path_has?(path, param), do: String.contains?(path, "{#{param}}") # Build the URL expression: github_base() <> "interpolated path" <> query string. defp build_url(path, query_params) do interpolated = Regex.replace(~r/\{([^}]+)\}/, path, fn _, name -> var = if name in ["owner", "repo"], do: name, else: snake(name) "\#{#{var}}" end) base = "github_base() <> \"#{interpolated}\"" if query_params == [] do base else fields = query_params |> Enum.map(&"get_field(#{atom_literal(&1["name"])}, options, nil)") |> Enum.join(",\n ") """ ( query = [ #{fields} ] |> Enum.filter(& &1) qs = if query == [], do: "", else: "?" <> Enum.join(query, "&") #{base} <> qs )\ """ end end defp function_doc(op) do summary = op.summary || op.operation_id url = op.doc_url doc = if url do "#{summary}\n\n @see #{url}" else summary end " @doc \"\"\"\n #{doc}\n \"\"\"" end # ---------------------------------------------------------------------------- # Naming helpers # ---------------------------------------------------------------------------- defp split_operation_id(id) do case String.split(id, "/", parts: 2) do [category, verb] -> {category, verb} [single] -> {"misc", single} end end defp function_name(verb) do name = snake(verb) if Regex.match?(~r/^[a-z_]/, name), do: name, else: "op_" <> name end defp ref_key(ref), do: ref |> String.split("/") |> List.last() # Write generated source, formatting it first so re-runs stay clean. Falls # back to the raw source if formatting fails for any reason. defp write!(path, code) do formatted = try do Code.format_string!(code) |> IO.iodata_to_binary() |> Kernel.<>("\n") rescue _ -> code end File.write!(path, formatted) end defp camelize(str) do str |> String.split(~r/[^a-zA-Z0-9]+/, trim: true) |> Enum.map_join("", &capitalize_word/1) end defp capitalize_word(<>), do: String.upcase(<>) <> rest defp capitalize_word(""), do: "" defp snake(str) do str |> String.replace(~r/[^a-zA-Z0-9]+/, "_") |> String.replace(~r/([a-z0-9])([A-Z])/, "\\1_\\2") |> String.downcase() |> String.trim("_") end # Render `name:` for a key literal (struct field assignment / map key). defp key_literal(name) do if valid_identifier?(name), do: "#{name}:", else: "#{inspect(name)}:" end # Render `:name` atom literal (defstruct entry / Map.get key). defp atom_literal(name) do if valid_identifier?(name), do: ":#{name}", else: ":#{inspect(name)}" end defp valid_identifier?(name), do: Regex.match?(~r/^[a-z_][a-zA-Z0-9_]*$/, name) end