defmodule Pi.AST do @moduledoc "Structured ExAST helpers for bridge tools." alias Pi.Protocol.Tool.AST.Diff alias Pi.Protocol.Tool.AST.Match alias Pi.Protocol.Tool.AST.Replace alias Pi.Protocol.Tool.AST.Replacement alias Pi.Protocol.Tool.AST.Search alias Pi.Protocol.UI.Block alias Pi.Protocol.UI.Display @missing_ex_ast "ex_ast is not installed. Add {:ex_ast, \"~> 0.1\", only: [:dev, :test], runtime: false} to mix.exs" def search(pattern, opts \\ []) when is_binary(pattern) do with :ok <- ensure_ex_ast() do path = Keyword.get(opts, :path) paths = paths(path) matches = paths |> ast_search(pattern, search_opts(opts)) |> Enum.map(&match_payload/1) {:ok, %Search{ pattern: pattern, path: path, matches: matches, total: length(matches), display: search_display(matches) }} end end def search_many(patterns, opts \\ []) when is_map(patterns) or is_list(patterns) do with :ok <- ensure_ex_ast() do path = Keyword.get(opts, :path) paths = paths(path) named_patterns = normalize_named_patterns(patterns) matches = paths |> ast_search_many(named_patterns, search_opts(opts)) |> Enum.map(&match_payload/1) {:ok, %Search{ pattern: inspect(named_patterns, limit: 20), path: path, matches: matches, total: length(matches), display: search_display(matches) }} end end def diff(opts \\ []) do with :ok <- ensure_ex_ast() do paths = diff_paths(opts) files = paths |> Enum.map(&semantic_file_diff/1) |> Enum.reject(&(semantic_file_edit_count(&1) == 0)) total = Enum.reduce(files, 0, fn file, acc -> acc + semantic_file_edit_count(file) end) Pi.Output.tree( %{ summary: semantic_diff_summary(total, files), total: total, files: files }, opts |> Keyword.take([:depth]) |> Keyword.put_new(:depth, 6) |> Keyword.put(:preview, semantic_diff_summary(total, files)) ) end end def replace(pattern, replacement, opts \\ []) when is_binary(pattern) and is_binary(replacement) do with :ok <- ensure_ex_ast() do path = Keyword.get(opts, :path) dry_run = Keyword.get(opts, :dry_run, false) paths = paths(path) opts = Keyword.merge(search_opts(opts), dry_run: dry_run) diffs = if dry_run, do: replacement_diffs(paths, pattern, replacement, opts), else: [] replacements = paths |> ast_replace(pattern, replacement, opts) |> Enum.map(fn {file, count} -> %Replacement{file: file, count: count} end) total = Enum.reduce(replacements, 0, fn %Replacement{count: count}, acc -> acc + count end) {:ok, %Replace{ dry_run: dry_run, pattern: pattern, replacement: replacement, path: path, replacements: replacements, total: total, diffs: diffs, display: replace_display(replacements, diffs) }} end end defp normalize_named_patterns(patterns) when is_map(patterns) and map_size(patterns) <= 50 do Map.new(patterns, fn {name, pattern} -> {pattern_name(name), pattern} end) end defp normalize_named_patterns(patterns) when is_map(patterns) do raise ArgumentError, "expected at most 50 named AST patterns" end defp normalize_named_patterns(patterns) when is_list(patterns), do: patterns defp pattern_name(name) when is_atom(name), do: name defp pattern_name(name) when is_binary(name) and byte_size(name) <= 64 do if String.match?(name, ~r/^[A-Za-z_][A-Za-z0-9_]*$/u) do String.to_existing_atom(name) else raise ArgumentError, "expected named AST pattern keys to be identifier-like strings" end end defp pattern_name(_name) do raise ArgumentError, "expected named AST pattern keys to be atoms or existing atom-name strings" end defp search_opts(opts) do [] |> maybe_put(:inside, Keyword.get(opts, :inside)) |> maybe_put(:not_inside, Keyword.get(opts, :not_inside)) |> maybe_put(:allow_broad, Keyword.get(opts, :allow_broad)) |> maybe_put(:limit, Keyword.get(opts, :limit)) end defp maybe_put(opts, _key, nil), do: opts defp maybe_put(opts, key, value), do: Keyword.put(opts, key, value) defp match_payload(%{file: file, line: line, source: source} = match) do %Match{ file: file, line: line, source: source, pattern: match_pattern(match), captures: match |> Map.get(:captures, %{}) |> render_captures() } end defp match_pattern(%{pattern: pattern}), do: to_string(pattern) defp match_pattern(_match), do: nil defp search_display(matches) do %Display{ summary: "#{length(matches)} match(es)", blocks: Enum.flat_map(matches, fn %Match{} = match -> [ %Block{type: :location, path: match.file, line: match.line}, %Block{type: :source, text: match.source, language: language_from_path(match.file)} ] end) } end defp replace_display(replacements, diffs) do replacement_blocks = Enum.map(replacements, fn %Replacement{} = replacement -> %Block{ type: :text, text: "#{replacement.file}: #{replacement.count} replacement(s)", path: replacement.file } end) diff_blocks = Enum.map(diffs, fn %Diff{} = diff -> %Block{type: :diff, text: diff.diff, path: diff.file, language: diff.language} end) %Display{ summary: "#{length(replacements)} file(s)", blocks: replacement_blocks ++ diff_blocks } end defp replacement_diffs(paths, pattern, replacement, opts) do opts = Keyword.drop(opts, [:dry_run]) paths |> Enum.flat_map(&resolve_paths/1) |> Enum.flat_map(fn file -> source = File.read!(file) replaced = ex_ast_replace_all(source, pattern, replacement, opts) if source == replaced do [] else [ %Diff{ file: file, diff: unified_diff(source, replaced, file), semantic_edits: semantic_edits(source, replaced, module_name(replaced) || module_name(source)) } ] end end) end defp semantic_edits(old, new, module_name) do edits = old |> ExAST.diff(new, include_moves: false) |> Map.fetch!(:edits) edits |> high_signal_edits() |> Enum.map(&semantic_edit(&1, module_name)) end defp high_signal_edits(edits) do structural = Enum.filter(edits, &(&1.kind in [:module, :function])) if structural == [], do: edits, else: structural end defp semantic_edit(edit, module_name) do range = edit.old_range || edit.new_range function = function_info(edit) %{ op: edit.op, kind: edit.kind, summary: semantic_summary(edit, function, module_name), line: range_line(range), module: module_name, visibility: function && function.visibility, name: function && function.name, arity: function && function.arity } |> Enum.reject(fn {_key, value} -> is_nil(value) end) |> Map.new() end defp semantic_summary(%{kind: :function} = edit, function, module_name) when is_map(function) do target = function_target(function, module_name) "#{op_verb(edit.op)} #{visibility_word(function.visibility)} #{target}" end defp semantic_summary(edit, _function, _module_name), do: edit.summary defp function_info(edit) do source = get_in(edit.meta, [:new]) || get_in(edit.meta, [:old]) with source when is_binary(source) <- source, {:ok, ast} <- Code.string_to_quoted(source) do function_info_from_ast(ast) else _ -> nil end end defp function_info_from_ast({kind, _, [head | _]}) when kind in [:def, :defp, :defmacro, :defmacrop] do {name, arity} = function_head_name_arity(head) if name do %{visibility: visibility(kind), name: name, arity: arity} end end defp function_info_from_ast(_ast), do: nil defp visibility(kind) when kind in [:def, :defmacro], do: :public defp visibility(kind) when kind in [:defp, :defmacrop], do: :private defp visibility_word(:public), do: "public" defp visibility_word(:private), do: "private" defp function_target(%{name: name, arity: arity}, nil), do: "#{name}/#{arity}" defp function_target(%{name: name, arity: arity}, module_name), do: "#{module_name}.#{name}/#{arity}" defp op_verb(:insert), do: "added" defp op_verb(:delete), do: "removed" defp op_verb(:update), do: "changed" defp op_verb(:move), do: "moved" defp op_verb(op), do: to_string(op) defp function_head_name_arity({:when, _, [head | _guards]}), do: function_head_name_arity(head) defp function_head_name_arity({:\\, _, [head, _default]}), do: function_head_name_arity(head) defp function_head_name_arity({name, _, args}) when is_atom(name) and is_list(args), do: {name, length(args)} defp function_head_name_arity(_head), do: {nil, 0} defp range_line(nil), do: nil defp range_line(%{start: start}), do: start[:line] defp module_name(source) do case Code.string_to_quoted(source) do {:ok, ast} -> ast |> Macro.prewalk(nil, fn {:defmodule, _, [{:__aliases__, _, parts}, _]} = node, nil -> {node, Module.concat(parts) |> inspect()} node, acc -> {node, acc} end) |> elem(1) _ -> nil end end defp semantic_diff_summary(0, _files), do: "Elixir syntax diff: no AST changes" defp semantic_diff_summary(total, files) do "Elixir syntax diff: #{total} edit(s) in #{length(files)} file(s)" end defp semantic_file_diff(path) do git_path = git_tracked_path(path) || path old = git_show("HEAD:#{git_path}") || "" new = read_worktree_file(path) || "" module_name = module_name(new) || module_name(old) semantic_file(path, module_name, semantic_edits(old, new, module_name)) end defp semantic_file(path, module_name, edits) do public = Enum.filter(edits, &(&1[:visibility] == :public)) private = Enum.filter(edits, &(&1[:visibility] == :private)) other = Enum.reject(edits, &(&1[:visibility] in [:public, :private])) %{ file: path, module: module_name, summary: semantic_file_summary(module_name, public, private, other), public_api: public, private_helpers: semantic_edit_group(private), other_edits: semantic_edit_group(other) } |> Enum.reject(fn {_key, nil} -> true {_key, []} -> true {_key, %{count: 0}} -> true _entry -> false end) |> Map.new() end defp semantic_edit_group([]), do: nil defp semantic_edit_group(edits), do: %{count: length(edits), edits: edits} defp semantic_file_edit_count(file) do length(Map.get(file, :public_api, [])) + to_i(get_in(file, [:private_helpers, :count])) + to_i(get_in(file, [:other_edits, :count])) end defp semantic_file_summary(module_name, public, private, other) do subject = module_name || "file" [ edit_count(public, "public"), edit_count(private, "private"), edit_count(other, "other") ] |> Enum.reject(&is_nil/1) |> case do [] -> "#{subject}: no syntax changes" parts -> "#{subject}: #{Enum.join(parts, ", ")}" end end defp edit_count([], _label), do: nil defp edit_count(edits, label), do: "#{length(edits)} #{label}" defp to_i(nil), do: 0 defp to_i(value) when is_integer(value), do: value defp to_i(_value), do: 0 defp diff_paths(opts) do cond do path = Keyword.get(opts, :path) -> path |> List.wrap() |> Enum.flat_map(&resolve_paths/1) Keyword.get(opts, :changed, false) -> changed_elixir_paths() true -> [] end |> Enum.filter(&String.ends_with?(&1, [".ex", ".exs"])) |> Enum.uniq() |> Enum.sort() end defp changed_elixir_paths do diff_names = git_lines(["diff", "--name-only", "HEAD", "--", "*.ex", "*.exs"]) untracked = git_lines(["ls-files", "--others", "--exclude-standard", "--", "*.ex", "*.exs"]) diff_names ++ untracked end defp read_worktree_file(path) do cond do File.exists?(path) -> File.read!(path) root = git_root() -> root |> Path.join(path) |> maybe_read_file() true -> nil end end defp maybe_read_file(path) do if File.exists?(path), do: File.read!(path) end defp git_root do case System.cmd("git", ["rev-parse", "--show-toplevel"], stderr_to_stdout: true) do {output, 0} -> String.trim(output) _ -> nil end rescue _ in ErlangError -> nil end defp git_lines(args) do case System.cmd("git", args, stderr_to_stdout: true) do {output, 0} -> String.split(output, "\n", trim: true) _ -> [] end rescue _ in ErlangError -> [] end defp git_tracked_path(path) do case System.cmd("git", ["ls-files", "--full-name", "--", path], stderr_to_stdout: true) do {output, 0} -> output |> String.split("\n", trim: true) |> List.first() _ -> nil end rescue _ in ErlangError -> nil end defp git_show(revision) do case System.cmd("git", ["show", revision], stderr_to_stdout: true) do {output, 0} -> output _ -> nil end rescue _ in ErlangError -> nil end defp unified_diff(old, new, file) do old_lines = String.split(old, "\n") new_lines = String.split(new, "\n") if old == new do "" else ["--- ", file, "\n", "+++ ", file, "\n" | diff_lines(old_lines, new_lines)] |> IO.iodata_to_binary() end end defp diff_lines(old_lines, new_lines), do: diff_lines(old_lines, new_lines, []) defp diff_lines([], [], acc), do: acc |> Enum.reverse() |> List.flatten() defp diff_lines([old_line | old_rest], [new_line | new_rest], acc) do diff_lines(old_rest, new_rest, [diff_line(old_line, new_line) | acc]) end defp diff_lines([], [new_line | new_rest], acc) do diff_lines([], new_rest, [diff_line(nil, new_line) | acc]) end defp diff_lines([old_line | old_rest], [], acc) do diff_lines(old_rest, [], [diff_line(old_line, nil) | acc]) end defp diff_line(nil, nil), do: [] defp diff_line(line, line), do: [" ", line, "\n"] defp diff_line(nil, new_line), do: ["+", new_line, "\n"] defp diff_line(old_line, nil), do: ["-", old_line, "\n"] defp diff_line(old_line, new_line), do: ["-", old_line, "\n", "+", new_line, "\n"] defp resolve_paths(path) when is_binary(path) do cond do String.contains?(path, "*") -> Path.wildcard(path) File.dir?(path) -> Path.wildcard(Path.join(path, "**/*.ex*")) true -> [path] end end defp language_from_path(path) do case Path.extname(path || "") do ".ex" -> "elixir" ".exs" -> "elixir" ".heex" -> "heex" ext -> String.trim_leading(ext, ".") end end defp ast_search(paths, pattern, opts) do paths |> ast_files() |> search_files(pattern, opts, &search_file(&1, pattern, &2)) end defp ast_search_many(paths, patterns, opts) do paths |> ast_files() |> search_files(patterns, opts, &search_file_many(&1, patterns, &2)) end defp search_files(files, pattern, opts, search_fun) do validate_broad_search!(pattern, opts) {limit, opts} = Keyword.pop(opts, :limit) opts = Keyword.drop(opts, [:allow_broad]) files |> Enum.reduce_while({[], limit}, fn file, {acc, remaining} -> file_matches = search_fun.(file, remaining_opts(opts, remaining)) next_acc = [file_matches | acc] cond do remaining == nil -> {:cont, {next_acc, nil}} length(file_matches) >= remaining -> {:halt, {next_acc, 0}} true -> {:cont, {next_acc, remaining - length(file_matches)}} end end) |> elem(0) |> Enum.reverse() |> List.flatten() end defp remaining_opts(opts, nil), do: opts defp remaining_opts(opts, remaining), do: Keyword.put(opts, :limit, remaining) defp search_file(file, pattern, opts) do source = File.read!(file) if apply(ExAST.Prefilter, :may_match?, [source, pattern]) do matches = apply(ExAST.Patcher, :find_all, [source, pattern, Keyword.drop(opts, [:limit])]) matches = maybe_take(matches, opts[:limit]) file_matches(file, source, matches) else [] end end defp search_file_many(file, patterns, opts) do source = File.read!(file) if Enum.any?(patterns, fn {_name, pattern} -> apply(ExAST.Prefilter, :may_match?, [source, pattern]) end) do matches = apply(ExAST.Patcher, :find_many, [source, patterns, Keyword.drop(opts, [:limit])]) matches = maybe_take(matches, opts[:limit]) file_matches(file, source, matches) else [] end end defp file_matches(file, source, matches) do lines = String.split(source, "\n", trim: false) Enum.map(matches, fn match -> %{ file: file, line: match_line(match[:range]), source: source_fragment(lines, match[:range]) || node_to_string(match[:node]), captures: match[:captures] || %{}, pattern: match[:pattern] } end) end defp ast_replace(paths, pattern, replacement, opts) do dry_run = Keyword.get(opts, :dry_run, false) opts = Keyword.drop(opts, [:dry_run, :allow_broad, :limit]) paths |> ast_files() |> Enum.flat_map(&replace_file(&1, pattern, replacement, opts, dry_run)) end defp replace_file(file, pattern, replacement, opts, dry_run) do source = File.read!(file) if apply(ExAST.Prefilter, :may_match?, [source, pattern]) do replacement_for_match(file, source, pattern, replacement, opts, dry_run) else [] end end defp replacement_for_match(file, source, pattern, replacement, opts, dry_run) do matches = apply(ExAST.Patcher, :find_all, [source, pattern, opts]) replaced = ex_ast_replace_all(source, pattern, replacement, opts) cond do matches == [] -> [] source == replaced -> [] true -> write_replacement(file, replaced, matches, dry_run) end end defp write_replacement(file, replaced, matches, dry_run) do unless dry_run, do: File.write!(file, replaced) [{file, length(matches)}] end defp ex_ast_replace_all(source, pattern, replacement, opts), do: apply(ExAST.Patcher, :replace_all, [source, pattern, replacement, opts]) defp validate_broad_search!(patterns, opts) when is_list(patterns) do Enum.each(patterns, fn {_name, pattern} -> validate_broad_search!(pattern, opts) end) end defp validate_broad_search!(pattern, opts) do if pattern == "_" and is_nil(opts[:limit]) and opts[:allow_broad] != true do raise ArgumentError, "refusing broad query without a limit; pass limit or allowBroad" end end defp maybe_take(matches, nil), do: matches defp maybe_take(matches, limit), do: Enum.take(matches, limit) defp ast_files(paths) do paths |> Enum.flat_map(&resolve_paths/1) |> Enum.filter(&elixir_source?/1) |> Enum.uniq() |> Enum.sort() end defp elixir_source?(path), do: String.ends_with?(path, [".ex", ".exs"]) defp match_line(%{start: start}) when is_list(start), do: start[:line] || 1 defp match_line(_range), do: 1 defp source_fragment(lines, %{start: start, end: end_}) when is_list(start) and is_list(end_) do with start_line when is_integer(start_line) <- start[:line], start_column when is_integer(start_column) <- start[:column], end_line when is_integer(end_line) <- end_[:line], end_column when is_integer(end_column) <- end_[:column] do fragment_lines(lines, start_line, start_column, end_line, end_column) else _ -> nil end end defp source_fragment(_lines, _range), do: nil defp fragment_lines(lines, line, start_column, line, end_column) do lines |> Enum.at(line - 1, "") |> String.slice((start_column - 1)..(end_column - 2)//1) end defp fragment_lines(lines, start_line, start_column, end_line, end_column) do lines |> Enum.slice((start_line - 1)..(end_line - 1)//1) |> trim_fragment_lines(start_column, end_column) |> Enum.join("\n") end defp trim_fragment_lines([], _start_column, _end_column), do: nil defp trim_fragment_lines([first | rest], start_column, end_column) do {middle, [last]} = Enum.split(rest, max(length(rest) - 1, 0)) [String.slice(first, (start_column - 1)..-1//1)] ++ middle ++ [String.slice(last, 0..(end_column - 2)//1)] end defp node_to_string(node) do Sourceror.to_string(node, locals_without_parens: []) rescue _exception in [ArgumentError, FunctionClauseError, Protocol.UndefinedError] -> Macro.to_string(node) end defp ensure_ex_ast do if Code.ensure_loaded?(ExAST), do: :ok, else: {:error, @missing_ex_ast} end defp paths(path) when is_binary(path), do: [path] defp paths(_path), do: ["lib/"] defp render_captures(captures) when map_size(captures) == 0, do: %{} defp render_captures(captures) do Map.new(captures, fn {name, value} -> rendered = Macro.prewalk(value, fn {form, nil, args} -> {form, [], args} other -> other end) |> Macro.to_string() {to_string(name), rendered} end) end end