defmodule Store.Project do defstruct [ :name, :store_path, :source_root, :exclude, :conversation_dir, :exclude_cache ] @type t :: %__MODULE__{} @type index_status :: %{ new: [Store.Project.Entry.t()], stale: [Store.Project.Entry.t()], deleted: [Store.Project.Entry.t()] } @conversation_dir "conversations" @files_dir "files" @spec new(String.t(), String.t()) :: t def new(project_name, store_path) do settings = Settings.new() project_data = Settings.get_project_data(settings, project_name) || %{} exclude = Map.get(project_data, "exclude", []) root = Settings.get_project_root_override() |> case do nil -> Map.get(project_data, "root", nil) override -> Path.expand(override) end %__MODULE__{ name: project_name, store_path: store_path, source_root: root, exclude: exclude, conversation_dir: Path.join(store_path, @conversation_dir) } end @spec save_settings(t, String.t() | nil, String.t() | nil) :: t def save_settings(project, source_root \\ nil, exclude \\ nil) do # Validate that the project name is not a global config key unless Settings.is_valid_project_name?(project.name) do raise ArgumentError, "Cannot use '#{project.name}' as project name - it conflicts with global configuration" end root = case source_root do nil -> project.source_root || raise("project root is required") _ -> Path.expand(source_root) end exclude = case exclude do nil -> project.exclude || [] [] -> project.exclude || [] exclude -> exclude |> Enum.map(&relative_to!(&1, root)) end |> Enum.filter(fn exclude -> exclude |> Path.expand(root) |> File.exists?() |> case do true -> true false -> UI.warn("Removing non-existent path from project exclude list: #{exclude}") false end end) settings = Settings.new() Settings.set_project_data(settings, project.name, %{ "root" => root, "exclude" => exclude }) new(project.name, project.store_path) end @spec create(t) :: t def create(project) do # Ensure base store directory project.store_path |> File.mkdir_p!() # Create conversations directory project.conversation_dir |> File.mkdir_p!() # Create files directory for future file entries project |> files_root() |> File.mkdir_p!() # Initialize notes Path.join(project.store_path, "notes.md") |> File.touch!() project end @spec files_root(t) :: String.t() def files_root(%__MODULE__{store_path: store_path}), do: Path.join(store_path, @files_dir) @spec delete(t) :: :ok def delete(project) do Store.Project.FilesDirMigration.ensure_files_dir_layout(project) # Delete indexed files files_root(project) |> Path.join("*/metadata.json") |> Path.wildcard() |> Enum.map(&Path.dirname/1) |> Enum.each(&File.rm_rf!/1) :ok end @spec torch(t) :: :ok def torch(project) do # Delete entire directory File.rm_rf!(project.store_path) # Remove from settings Settings.new() |> Settings.delete_project_data(project.name) :ok end @spec make_default_for_session(t) :: t def make_default_for_session(project) do Settings.set_project(project.name) project end @doc """ Resolves `path` within the project's source root. Returns `{:ok, path}` if the file exists, or `{:error, :enoent}` if it does not. """ @spec find_file(t, binary) :: {:ok, binary} | {:error, :enoent} | {:error, File.posix()} def find_file(project, path) do Util.find_file_within_root(path, project.source_root) end @doc """ Reads the project prompt from `FNORD.md` and `FNORD.local.md` in the source root. Returns: * `{:ok, prompt}` (at least one file has content) * `{:error, :not_found}` When both files have content, the resulting prompt will include instructions from both files, along with a short note indicating priority. """ @spec project_prompt(t) :: {:ok, binary} | {:error, :not_found} def project_prompt(%{source_root: nil}), do: {:error, :not_found} def project_prompt(project) do shared = project.source_root |> Path.join("FNORD.md") |> File.read() |> case do {:ok, content} when byte_size(content) > 0 -> UI.info("Loaded shared project instructions from ./FNORD.md") """ ## Shared project instructions (FNORD.md) #{content} """ _ -> nil end local = project.source_root |> Path.join("FNORD.local.md") |> File.read() |> case do {:ok, content} when byte_size(content) > 0 -> UI.info("Loaded local project instructions from ./FNORD.local.md") """ ## Local user instructions (FNORD.local.md) Note: if these local instructions conflict with earlier instructions, prefer these local instructions unless the user's explicit prompt says otherwise. #{content} """ _ -> nil end [shared, local] |> Enum.filter(&is_binary/1) |> case do [] -> {:error, :not_found} prompts -> {:ok, Enum.join(prompts, "\n\n")} end end # ---------------------------------------------------------------------------- # Entries # ---------------------------------------------------------------------------- @spec has_index?(t) :: boolean() def has_index?(project) do glob = project |> files_root() |> Path.join("**/embeddings.json") System.cmd("bash", ["-c", ~s[compgen -G "$1" > /dev/null], "--", glob]) |> case do {_, 0} -> true _ -> false end end @spec find_entry(t, String.t()) :: {:ok, Store.Project.Entry.t()} | {:error, :enoent} def find_entry(project, path) do with {:ok, resolved} <- find_file(project, path) do {:ok, Store.Project.Entry.new_from_file_path(project, resolved)} end end @spec expand_path(String.t(), t) :: String.t() def expand_path(path, %Store.Project{} = project) do Path.expand(path, project.source_root) end @spec relative_path(String.t(), t) :: String.t() def relative_path(path, project) do abs = expand_path(path, project) rel = Path.relative_to(abs, project.source_root) # `Path.relative_to/2` returns the absolute path unchanged when it can't # strip the root prefix. That happens when `abs` and `source_root` # describe the same location through different symlinks - a path that # arrived via the cwd (Path.absname, File.cwd!) is OS-canonical (macOS # /tmp -> /private/tmp) while source_root is a plain Path.expand. Left # alone, the full absolute path becomes the per-file store key # (id_for_rel_path) and the lookup misses a file that is in fact indexed. # Only on that failure do we pay to canonicalize both sides through every # symlink and retry; the common path (indexing, already-consistent paths) # keeps the cheap relativize with no extra filesystem calls. if rel == abs do Path.relative_to(Util.canonical_path(abs), Util.canonical_path(project.source_root)) else rel end end @spec find_path_in_source_root(Store.Project.t(), String.t()) :: {:ok, :dir | :file | :enoent, String.t()} def find_path_in_source_root(project, path) do path = expand_path(path, project) cond do File.dir?(path) -> {:ok, :dir, path} File.regular?(path) -> {:ok, :file, path} true -> {:ok, :enoent, path} end end @spec exists_in_store?(t) :: boolean() def exists_in_store?(project) do path = project.store_path files = Path.wildcard(Path.join(path, "*")) cond do !File.dir?(path) -> false # There was a bug at one point where store files were deleted but not the # directory itself. This check is to ensure that the directory is empty. Enum.empty?(files) -> false true -> true end end @spec stored_files(t) :: Enumerable.t() def stored_files(project) do Store.Project.FilesDirMigration.ensure_files_dir_layout(project) # Ensure legacy entries are migrated to relative-path scheme Store.Project.Entry.MigrateAbsToRelPathKeys.ensure_relative_entry_ids(project) # Scan the files directory for entry metadata project |> files_root() |> Path.join("*/metadata.json") # Expand the glob |> Path.wildcard() # Strip metadata.json from the each listing, leaving the directory path for # the individual entry. |> Stream.map(&Path.dirname/1) # Create an Entry for each directory. |> Stream.map(&Store.Project.Entry.new_from_entry_path(project, &1)) end @spec path_excluded?(t, String.t()) :: boolean() @doc """ Returns true if the given absolute path is excluded by the project's exclusion policy (user-defined patterns or gitignore). """ def path_excluded?(project, abs_path) when is_binary(abs_path) do {_project, excluded} = excluded_paths(project) Map.has_key?(excluded, abs_path) end @spec original_source_root() :: String.t() | nil @doc """ Returns the project's source root as configured in settings, ignoring any in-memory override (e.g. from a worktree session). Used by tools that need to access files outside the worktree's view, such as gitignored files that exist in source but were not propagated to the worktree. Cached in Services.Globals at session start by Cmd.Ask. Returns nil when no session has cached the value (e.g. in tests that bypass Cmd.Ask). """ def original_source_root do case Services.Globals.get_env(:fnord, :original_project_root, nil) do root when is_binary(root) -> root _ -> nil end end @doc """ Returns no source files when source_root is nil to avoid crashing. """ def source_files(%{source_root: nil} = project), do: {project, []} @spec source_files(t) :: {t, Enumerable.t()} def source_files(project) do case Store.Project.Source.mode(project) do :git -> source_files_git(project) :fs -> source_files_fs(project) end end # Git mode: enumerate the default branch's tree. Tracked files are by # definition not gitignored, so we skip that merge entirely. User # excludes (`project.exclude`) still apply. Binary-vs-text filtering # is skipped - users who track binary artifacts should list them in # `exclude`, and checking every blob would require a `git show` round # trip per file just to classify. defp source_files_git(project) do {project, user_excluded} = user_excluded_paths(project) files = project |> Store.Project.Source.list() |> Stream.filter(fn %{abs_path: abs} -> not Map.has_key?(user_excluded, abs) end) |> Stream.filter(fn %{rel_path: rel} -> not is_hidden_rel?(rel) end) |> Stream.map(fn %{abs_path: abs} -> Store.Project.Entry.new_from_file_path(project, abs) end) {project, files} end defp source_files_fs(project) do {project, excluded_paths} = excluded_paths(project) files = project |> list_all_files() |> Stream.filter(&(!is_hidden?(&1))) |> Stream.filter(&(!Map.has_key?(excluded_paths, &1))) |> Stream.filter(&is_text?(&1, project)) |> Stream.map(&Store.Project.Entry.new_from_file_path(project, &1)) {project, files} end @spec delete_missing_files(t) :: {t, Enumerable.t()} def delete_missing_files(project) do {project, excluded_paths} = excluded_paths(project) # "Missing" means two different things depending on source mode: # :git - the file isn't in the default branch's tree (even if it # still exists in the working tree - we index what's on # the default branch, so WIP-only files aren't "real" for # the index). # :fs - the file isn't in the working tree. # Excluded-by-user entries are dropped in either mode. missing? = &missing_source?(&1, project, excluded_paths) entries = project |> stored_files() |> Stream.filter(missing?) |> Stream.map(fn entry -> Store.Project.Entry.delete(entry) entry end) {project, entries} end defp missing_source?(entry, project, excluded_paths) do Map.has_key?(excluded_paths, entry.file) or not Store.Project.Source.exists?(project, entry.rel_path) end @doc """ Returns the status of the index for the given project. It classifies entries into: * `:deleted` - entries that were indexed but the source files have been removed * `:stale` - entries whose indexed metadata is stale compared to the source file * `:new` - entries for unindexed files that exist in the source """ @spec index_status(t) :: index_status def index_status(project), do: index_status(project, []) @spec index_status(t, keyword) :: index_status def index_status(project, opts) do on_progress = Keyword.get(opts, :on_progress, fn _ -> :ok end) {project, source_stream} = source_files(project) source = source_stream |> Enum.to_list() stored = project |> stored_files() |> Enum.to_list() on_progress.({:total, length(source) + length(stored)}) # Classify source entries in parallel. Per-entry cost is dominated by # File.read + JSON decode of metadata.json, which is cheap CPU-wise # but single-threaded disk I/O; fanning out across schedulers # collapses wall-clock on the scan. Results are consumed unordered - # ticks arrive in completion order and the caller's progress bar # treats them as fungible. {new, stale} = source |> Util.async_stream(&classify_source_entry/1, ordered: false) |> Enum.reduce({[], []}, fn {:ok, {:new, entry}}, {new_acc, stale_acc} -> on_progress.(:tick) {[entry | new_acc], stale_acc} {:ok, {:stale, entry}}, {new_acc, stale_acc} -> on_progress.(:tick) {new_acc, [entry | stale_acc]} {:ok, {:fresh, _entry}}, acc -> on_progress.(:tick) acc _, acc -> on_progress.(:tick) acc end) # In git mode, "deleted" means "not in the default branch's tree". # Parallel for the same reason: Source.exists? is O(1) against the # cached path map but each stored entry still costs a metadata tree # walk through stored_files/1 upstream, and running the check # concurrently keeps the deletion pass from serializing behind disk. deleted = stored |> Util.async_stream( fn entry -> {not Store.Project.Source.exists?(project, entry.rel_path), entry} end, ordered: false ) |> Enum.reduce([], fn {:ok, {true, entry}}, acc -> on_progress.(:tick) [entry | acc] {:ok, {false, _entry}}, acc -> on_progress.(:tick) acc _, acc -> on_progress.(:tick) acc end) %{ new: new, stale: stale, deleted: deleted } end # Extracted so async_stream's lambda stays a module reference rather # than capturing `project` from the enclosing scope - the entry # already carries its project and each branch only needs the entry. defp classify_source_entry(entry) do cond do not Store.Project.Entry.exists_in_store?(entry) -> {:new, entry} Store.Project.Entry.is_stale?(entry) -> {:stale, entry} true -> {:fresh, entry} end end # ----------------------------------------------------------------------------- # Conversations # ----------------------------------------------------------------------------- @spec conversations(t) :: [Store.Project.Conversation.t()] def conversations(project) do Store.Project.Conversation.list(project.store_path) end # ----------------------------------------------------------------------------- # Private functions # ----------------------------------------------------------------------------- defp list_all_files(project) do args = ["-c", "(find #{project.source_root} -type f || true) | sort"] case System.cmd("sh", args, stderr_to_stdout: true) do {output, 0} -> output |> String.split("\n", trim: true) |> Stream.filter(&(!String.ends_with?(&1, ": Permission denied"))) |> Stream.map(&Path.absname(&1, project.source_root)) {error_output, _} -> raise "find command failed: #{error_output}" end end defp is_hidden?(path) do cond do String.contains?(path, "/.github/") -> false String.starts_with?(path, ".") -> true String.contains?(path, "/.") -> true true -> false end end defp is_text?(file_path, project) do try do file_path |> Path.expand(project.source_root) |> File.stream!(1024) |> Enum.reduce_while(true, fn chunk, _acc -> if String.valid?(chunk) do {:cont, true} else {:halt, false} end end) rescue _ -> false end end # Returns ignored paths via GitCli or empty map when root is nil defp git_ignored(nil), do: %{} defp git_ignored(root), do: GitCli.ignored_files(root) # Computes excluded file paths, caching the result defp excluded_paths(%{exclude_cache: nil} = project) do {project, user_excluded} = user_excluded_paths(project) excluded = Map.merge(user_excluded, git_ignored(project.source_root)) {%{project | exclude_cache: excluded}, excluded} end defp excluded_paths(%{exclude_cache: excluded} = project) do {project, excluded} end # Just the user-level `project.exclude` patterns resolved to absolute # paths. Used by git-mode enumeration where gitignore is irrelevant # (tracked files aren't ignored, by definition). defp user_excluded_paths(project) do user_excluded = if is_nil(project.exclude) do %{} else project.exclude |> Enum.flat_map(fn exclude -> case find_path_in_source_root(project, exclude) do {:ok, :dir, path} -> Path.wildcard(Path.join(path, "**/*"), match_dot: true) {:ok, :file, path} -> [path] _ -> Path.wildcard(exclude, match_dot: true) end end) |> Enum.filter(&File.regular?/1) |> Enum.map(&Path.absname/1) |> Map.new(&{&1, true}) end {project, user_excluded} end # Hidden-path heuristic for git-mode listings. Mirrors is_hidden?/1 but # operates on relative paths: a tracked file under .github/ is fine, # but tracked dotfiles (.envrc, etc.) and anything under a dot-dir # (like a .hidden-cache/) are skipped. defp is_hidden_rel?(rel_path) do cond do String.starts_with?(rel_path, ".github/") -> false String.starts_with?(rel_path, ".") -> true String.contains?(rel_path, "/.") -> true true -> false end end defp relative_to!(path, cwd) do rel = path |> Path.expand(cwd) |> Path.relative_to(cwd) case Path.safe_relative(rel, cwd) do {:ok, clean} -> clean :error -> raise("Error: unable to calculate relative path for #{path} from #{cwd}") end end end