defmodule Cmd.Indexer do @moduledoc """ This module provides the functionality for the `index` sub-command. """ defstruct [ :opts, :indexer_module, :indexer, :root, :store, :reindex, :exclude, :exclude_patterns ] @doc """ Create a new `Indexer` struct. """ def new(opts, indexer \\ Indexer) do reindex = Map.get(opts, :reindex, false) exclude = Map.get(opts, :exclude, []) expanded_exclude = expand_excludes(exclude) settings = Settings.new() project_settings = Settings.get_project(settings) |> case do {:ok, project_settings} -> project_settings {:error, :not_found} -> %{} end root = case Map.get(opts, :directory, nil) do root when is_binary(root) -> Path.absname(root) nil -> case Map.get(project_settings, "root") do nil -> raise """ Error: the project root was not found in the settings file. This can happen under the following circumstances: - the first index of a project - the first index reindexing after moving the project directory - the first index after the upgrade that made --dir optional Re-run with --dir to specify the project root directory and update the settings file. """ root -> root end end Settings.set_project( Settings.new(), Map.merge(project_settings, %{ "root" => root, "exclude" => expanded_exclude, "exclude_patterns" => exclude }) ) %__MODULE__{ opts: opts, indexer_module: indexer, indexer: indexer.new(), root: root, store: Store.new(), reindex: reindex, exclude: expanded_exclude, exclude_patterns: exclude } end @doc """ Run the indexing process using the given `Indexer` struct. If `force_reindex` is `true`, the project will be deleted and reindexed from scratch. """ def run(idx) do project = Application.get_env(:fnord, :project) UI.info("Indexing", project) UI.info("Root", idx.root) UI.info("Excluding", Enum.join(idx.exclude_patterns, " | ")) if idx.reindex do Store.delete_project(idx.store) UI.report_step("Burned all of the old data to the ground to force a full reindex!") else # Otherwise, just delete any files that no longer exist. Store.delete_missing_files(idx.store, idx.root) UI.report_step("Deleted missing and newly excluded files") end {:ok, queue} = Queue.start_link(&process_file(idx, &1)) scanner = Scanner.new(idx.root, fn file -> if file not in idx.exclude do Queue.queue(queue, file) end end) count = spin("Scanning project files", fn -> num_files = count_files_to_index(idx) total_files = Scanner.count_files(scanner) msg = "Indexing #{num_files} of #{total_files} total files" {msg, num_files} end) if count == 0 do UI.warn("No files to index in #{project}") else spin("Indexing #{project}", fn -> # count * 3 for each step in indexing a file (summary, outline, embeddings) progress_bar_start(:indexing, "tasks", count * 3) Scanner.scan(scanner) Queue.shutdown(queue) Queue.join(queue) progress_bar_end() {"All tasks complete", :ok} end) end end # ----------------------------------------------------------------------------- # Indexing process # ----------------------------------------------------------------------------- defp process_file(idx, file) do with {:ok, file_hash} <- get_file_hash(idx, file) do file_contents = File.read!(file) with {:ok, summary} <- get_summary(idx, file, file_contents), {:ok, outline} <- get_outline(idx, file, file_contents), {:ok, embeddings} <- get_embeddings(idx, file, summary, outline, file_contents) do Store.put(idx.store, file, file_hash, summary, outline, embeddings) else {:error, reason} -> UI.warn("Error processing file #{file}", reason) end end end defp count_files_to_index(idx) do idx.root |> Scanner.new(fn _ -> nil end) |> Scanner.reduce(0, nil, fn file, acc -> if file in idx.exclude do acc else case get_file_hash(idx, file) do {:ok, _} -> acc + 1 {:error, :unchanged} -> acc end end end) end defp get_file_hash(idx, file) do # Files that are missing or have missing indexes must be reindexed. cond do !Store.has_summary?(idx.store, file) -> {:ok, nil} !Store.has_outline?(idx.store, file) -> {:ok, nil} !Store.has_embeddings?(idx.store, file) -> {:ok, nil} true -> existing_hash = Store.get_hash(idx.store, file) file_hash = sha256(file) if is_nil(existing_hash) or existing_hash != file_hash do {:ok, file_hash} else {:error, :unchanged} end end end defp get_outline(idx, file, file_contents) do res = idx.indexer_module.get_outline(idx.indexer, file, file_contents) progress_bar_update(:indexing) res end defp get_summary(idx, file, file_contents) do res = idx.indexer_module.get_summary(idx.indexer, file, file_contents) progress_bar_update(:indexing) res end defp get_embeddings(idx, file, summary, outline, file_contents) do to_embed = """ # File `#{file}` ## Summary #{summary} ## Outline #{outline} ## Contents ``` #{file_contents} ``` """ result = idx.indexer_module.get_embeddings(idx.indexer, to_embed) progress_bar_update(:indexing) case result do {:error, reason} -> IO.inspect(reason) _ -> nil end result end defp sha256(file_path) do case File.read(file_path) do {:ok, content} -> :crypto.hash(:sha256, content) |> Base.encode16(case: :lower) {:error, reason} -> {:error, reason} end end # ----------------------------------------------------------------------------- # Helper functions # ----------------------------------------------------------------------------- defp expand_excludes(excludes) do excludes |> Enum.flat_map(fn exclude -> cond do # If it's a directory, expand recursively to all files and directories File.dir?(exclude) -> Path.wildcard(Path.join(exclude, "**/*"), match_dot: true) # If it's a specific file, expand to its absolute path File.exists?(exclude) -> [Path.absname(exclude)] # Assume it's a glob pattern and expand it true -> Path.wildcard(exclude, match_dot: true) end end) # Filter out directories and non-existent files |> Enum.filter(&File.regular?/1) # Convert everything to absolute paths |> Enum.map(&Path.absname/1) end # ----------------------------------------------------------------------------- # UI interaction # ----------------------------------------------------------------------------- defp quiet?(), do: Application.get_env(:fnord, :quiet) defp spin(processing, func) do if quiet?() do {_msg, result} = func.() result else Spinner.run(func, processing) end end defp progress_bar_start(name, label, total) do if !quiet?() do Owl.ProgressBar.start( id: name, label: label, total: total, timer: true, absolute_values: true ) end end defp progress_bar_end() do if !quiet?() do Owl.LiveScreen.await_render() end end defp progress_bar_update(name) do if !quiet?() do Owl.ProgressBar.inc(id: name) Owl.LiveScreen.await_render() end end end