defmodule Mix.Tasks.Ab.Test do @moduledoc """ Mix task that compiles Elixir files, extracts modules, and runs AB.property_test on their exported functions. ## Usage mix ab.test path/to/file.ex mix ab.test path/to/directory ## Examples # Test a single file mix ab.test lib/my_module.ex # Test all .ex files in a directory recursively mix ab.test lib/ # Test with verbose output mix ab.test lib/my_module.ex --verbose # Show only failures in a structured format mix ab.test lib/my_module.ex --failures-only # Skip functions without typespecs (soft mode) mix ab.test lib/my_module.ex --soft # Test all files in a directory with soft mode mix ab.test lib/ --soft ## Options - `--verbose`, `-v`: Enable verbose output showing input/output for each test - `--failures-only`, `-f`: Show only failed tests in a structured format with: - Function name - Why it failed - Incorrect typespec - Expected typespec - File path and line number - `--soft`, `-s`: Skip functions without typespecs instead of erroring out This task will: 1. Compile the specified Elixir file(s) or all .ex files in a directory (recursively) 2. Extract the module from each compiled file 3. Find all exported functions with typespecs 4. Run AB.property_test on each function 5. Error out if functions without typespecs are found (unless --soft flag is used) When processing directories, the task will: - Recursively find all .ex files - Skip hidden directories (starting with .) - Skip common build directories (_build, deps, node_modules) """ use Mix.Task @shortdoc "Compiles Elixir files and runs AB.property_test on their exported functions" @impl Mix.Task def run(args) do # Start ExUnit ExUnit.start(autorun: false) {opts, paths} = parse_args(args) if paths == [] do Mix.shell().error( "No file or directory specified. Usage: mix ab.test or " ) System.halt(1) end # Expand paths to individual files all_files = expand_paths_to_files(paths) if all_files == [] do Mix.shell().error("No .ex files found in the specified paths") System.halt(1) end Mix.shell().info("Found #{length(all_files)} .ex files to process") Enum.each(all_files, fn file -> test_file(file, opts) end) end defp parse_args(args) do {opts, files, _} = OptionParser.parse(args, switches: [verbose: :boolean, failures_only: :boolean, soft: :boolean], aliases: [v: :verbose, f: :failures_only, s: :soft] ) {opts, files} end defp test_file(file_path, opts) do Mix.shell().info("Testing file: #{file_path}") case compile_and_extract_module(file_path) do {:ok, module} -> run_property_tests(module, opts, file_path) {:error, reason} -> Mix.shell().error("Failed to compile file: #{reason}") System.halt(1) end end defp compile_and_extract_module(file_path) do # Extract module name from the file first case extract_module_from_file(file_path) do {:ok, module} -> # Now compile the file using elixirc to preserve typespecs compile_with_elixirc(file_path, module) error -> error end end defp compile_with_elixirc(file_path, module) do # Use Code.compile_file which has access to the current Mix environment # This includes all dependencies that are already loaded case Code.compile_file(file_path) do [{^module, _}] -> {:ok, module} [{^module, _}, _] -> {:ok, module} [] -> {:error, "No module compiled"} _ -> {:error, "Unexpected compilation result"} end end defp extract_module_from_file(file_path) do case File.read(file_path) do {:ok, content} -> # Parse the AST to extract the module name case Code.string_to_quoted(content) do {:ok, ast} -> module_name = extract_module_name_from_ast(ast) if module_name do {:ok, module_name} else {:error, "Could not find module name in file"} end {:error, reason} -> {:error, "Failed to parse file: #{inspect(reason)}"} end {:error, reason} -> {:error, "Failed to read file: #{reason}"} end end defp extract_module_name_from_ast({:defmodule, _, [module_alias | _]}) do case module_alias do {:__aliases__, _, parts} -> Module.concat(parts) name when is_atom(name) -> name _ -> nil end end defp extract_module_name_from_ast({:__block__, _, statements}) do Enum.find_value(statements, fn statement -> extract_module_name_from_ast(statement) end) end defp extract_module_name_from_ast(_), do: nil defp run_property_tests(module, opts, file_path) do Mix.shell().info("Running property tests for module: #{module}") # The module should already be loaded from compilation run_tests_for_module(module, opts, file_path) end defp run_tests_for_module(module, opts, file_path) do soft_mode = Keyword.get(opts, :soft, false) # Get all exported functions with typespecs functions_with_specs = get_functions_with_specs(module) # Get all exported functions all_exported_functions = module.__info__(:functions) # Find functions without specs functions_without_specs = all_exported_functions |> Enum.reject(fn {name, arity} -> {name, arity} in functions_with_specs end) # Handle functions without specs if functions_without_specs != [] do if soft_mode do Mix.shell().info( "Warning: Found #{length(functions_without_specs)} functions without typespecs (skipping): #{format_function_list(functions_without_specs)}" ) else Mix.shell().error( "Error: Found #{length(functions_without_specs)} functions without typespecs:" ) Enum.each(functions_without_specs, fn {name, arity} -> Mix.shell().error( " - #{String.replace_leading(to_string(module), "Elixir.", "")}.#{name}/#{arity}" ) end) Mix.shell().error("Use --soft flag to skip functions without typespecs") System.halt(1) end end # Run tests for functions with specs case functions_with_specs do [] -> Mix.shell().info("No functions with typespecs found in #{module}") functions -> Mix.shell().info("Found #{length(functions)} functions with typespecs") run_tests_for_functions(module, functions, opts, file_path) end end defp get_functions_with_specs(module) do case Code.Typespec.fetch_specs(module) do {:ok, specs} -> exported_functions = module.__info__(:functions) specs |> Enum.map(fn {{function_name, arity}, _} -> {function_name, arity} end) |> Enum.filter(fn {function_name, arity} -> {function_name, arity} in exported_functions end) _ -> [] end end defp run_tests_for_functions(module, functions, opts, file_path) do verbose = Keyword.get(opts, :verbose, false) failures_only = Keyword.get(opts, :failures_only, false) if not failures_only do Mix.shell().info("\n=== Running property tests for #{length(functions)} functions ===") end results = Enum.map(functions, fn {function_name, _arity} -> if not failures_only do Mix.shell().info("Testing #{module}.#{function_name}") end run_single_property_test(module, function_name, verbose, failures_only) end) # Summary passed = Enum.count(results, &(&1 == :ok)) failed = length(results) - passed if failures_only do # Only show failures in the specified format failures = Enum.filter(results, &(&1 != :ok)) if failures != [] do Mix.shell().info("\n=== Failures ===") Enum.with_index(failures, 1) |> Enum.each(fn {failure, index} -> format_failure_output(module, failure, index, file_path) end) else Mix.shell().info("All tests passed!") end else Mix.shell().info("\n=== Summary ===") Mix.shell().info("✓ Passed: #{passed}") Mix.shell().info("✗ Failed: #{failed}") end if failed > 0 do System.halt(1) end end defp run_single_property_test(module, function_name, verbose, failures_only) do # Run the property test with details capture case AB.run_property_test_with_details(module, function_name, verbose: verbose) do {:ok, count} -> if not failures_only do Mix.shell().info( " ✓ #{module}.#{function_name} passed all property tests (#{count} runs)" ) end :ok {:error, details} -> if not failures_only do Mix.shell().error(" ✗ #{module}.#{function_name} failed property test") Mix.shell().error(" #{details.reason}") end {:error, details} end end defp format_failure_output( _module, {:error, details}, index, file_path ) do function_name = details.function_name module = details.module arity = get_arity_for_function(module, function_name) # Get typespec correction suggestion if we have the actual values suggestion = if details.inputs && details.result && details.mismatch_type do AB.suggest_typespec_correction( module, function_name, details.inputs, details.result, details.mismatch_type ) else nil end # Get line number for the function - try both approaches line_number = case get_line_number_for_function(module, function_name) do 0 -> get_line_number_from_source(file_path, function_name) line -> line end # Format the failure output - remove Elixir. prefix if present module_name = if to_string(module) |> String.starts_with?("Elixir.") do to_string(module) |> String.replace_leading("Elixir.", "") else to_string(module) end Mix.shell().info("#{index}. #{module_name}.#{function_name}/#{arity}") Mix.shell().info(" - Why it failed: #{details.reason}") # Show the actual inputs and result if available if details.inputs do Mix.shell().info(" - Generated input: #{inspect(details.inputs)}") end if details.result do Mix.shell().info(" - Actual result: #{inspect(details.result)}") end # Show typespec suggestion case suggestion do nil -> # No suggestion available :ok {:error, error_msg} -> Mix.shell().info(" - Could not analyze typespec: #{error_msg}") %{old_spec: old_spec, new_spec: new_spec, reason: _suggestion_reason} -> Mix.shell().info(" - Incorrect typespec: #{old_spec}") Mix.shell().info(" - Suggested typespec: #{new_spec}") end Mix.shell().info(" - #{file_path}:#{line_number}") Mix.shell().info("") end defp get_line_number_for_function(module, function_name) do # Try to get line number from the function's metadata case Code.Typespec.fetch_specs(module) do {:ok, specs} -> case Enum.find(specs, fn {{name, _arity}, _} -> name == function_name end) do {{_name, _arity}, {_spec, context, metadata}} -> # Try to get line number from context or metadata cond do # Check if context has line information is_list(context) and Keyword.has_key?(context, :line) -> Keyword.get(context, :line, 0) # Check if metadata has line information is_list(metadata) and Keyword.has_key?(metadata, :line) -> Keyword.get(metadata, :line, 0) # Look for line information in the metadata list is_list(metadata) -> case Enum.find(metadata, fn {key, _} -> key == :line end) do {:line, line} -> line _ -> 0 end true -> 0 end _ -> 0 end _ -> 0 end end defp get_line_number_from_source(file_path, function_name) do # Try to find the line number by parsing the source file case File.read(file_path) do {:ok, content} -> lines = String.split(content, "\n") # Look for the function definition case Enum.with_index(lines, 1) |> Enum.find({nil, 0}, fn {line, _index} -> # Look for function definition patterns String.contains?(line, "def #{function_name}") or String.contains?(line, "def #{function_name}(") or String.contains?(line, "def #{function_name} ") end) do {_line, line_number} -> line_number _ -> 0 end {:error, _} -> 0 end end defp get_arity_for_function(module, function_name) do # Get arity from the module's exported functions case module.__info__(:functions) do functions when is_list(functions) -> case Enum.find(functions, fn {name, _arity} -> name == function_name end) do {_name, arity} -> arity _ -> 0 end _ -> 0 end end defp format_function_list(functions) do functions |> Enum.map(fn {name, arity} -> "#{name}/#{arity}" end) |> Enum.join(", ") end defp expand_paths_to_files(paths) do paths |> Enum.flat_map(fn path -> if File.dir?(path) do find_ex_files_recursively(path) else if String.ends_with?(path, ".ex") do [path] else [] end end end) |> Enum.uniq() end defp find_ex_files_recursively(directory) do case File.ls(directory) do {:ok, entries} -> entries |> Enum.flat_map(fn entry -> full_path = Path.join(directory, entry) cond do File.dir?(full_path) -> # Skip hidden directories and common build directories if String.starts_with?(entry, ".") or entry in ["_build", "deps", "node_modules"] do [] else find_ex_files_recursively(full_path) end String.ends_with?(entry, ".ex") -> [full_path] true -> [] end end) {:error, _reason} -> Mix.shell().error("Could not read directory: #{directory}") [] end end end