defmodule AI.Embeddings.Migration do @moduledoc """ Detects and performs one-time migration from OpenAI embeddings to the local embedding model. Migration is triggered when stored embedding vectors have a different dimensionality than the current model expects. The migration deletes all stored embeddings so they can be regenerated by the background indexers or a manual `fnord index` run. """ @doc """ Checks whether migration is needed and performs it if so. Returns :ok in all cases (migration is best-effort). `context` controls the user-facing messaging only: * `:ask` - rebuild happens in the background while ask runs * `:index` - full foreground reindex is about to start * `:read_only` - read-only command (search, conversations, memory); no background indexer is running, user must run `fnord index` """ @spec maybe_migrate(:ask | :index | :read_only) :: :ok def maybe_migrate(context) do case detect_stale_embeddings() do :ok -> :ok :stale -> do_migrate(context) end end # Sample one embedding file to check dimensions. If no embeddings exist # yet, there's nothing to migrate. defp detect_stale_embeddings do expected = AI.Embeddings.dimensions() with {:ok, project} <- Store.get_project(), sample when not is_nil(sample) <- sample_embedding(project) do if length(sample) == expected, do: :ok, else: :stale else # No project or no embeddings on disk - nothing to migrate _ -> :ok end end # Try to find one existing embedding vector from any index type. defp sample_embedding(project) do # File embeddings sample_file_embedding(project) || sample_commit_embedding(project) || sample_conversation_embedding(project) || sample_memory_embedding(project) end defp sample_file_embedding(project) do project |> Store.Project.files_root() |> Path.join("*/embeddings.json") |> Path.wildcard() |> List.first() |> read_embedding_file() end defp sample_commit_embedding(project) do project |> Store.Project.CommitIndex.root() |> Path.join("*/embeddings.json") |> Path.wildcard() |> List.first() |> read_embedding_file() end defp sample_conversation_embedding(project) do project |> Store.Project.ConversationIndex.root() |> Path.join("*/embeddings.json") |> Path.wildcard() |> List.first() |> read_embedding_file() end defp sample_memory_embedding(project) do project_dir = Path.join(project.store_path, "memory") global_dir = Path.join(Settings.fnord_home(), "memory") sample_memory_embedding_in_dir(project_dir) || sample_memory_embedding_in_dir(global_dir) end defp sample_memory_embedding_in_dir(dir) do dir |> Path.join("*.json") |> Path.wildcard() |> Enum.find_value(fn path -> case File.read(path) do {:ok, data} -> case SafeJson.decode(data) do {:ok, %{"embeddings" => emb}} when is_list(emb) and length(emb) > 0 -> emb _ -> nil end _ -> nil end end) end defp read_embedding_file(nil), do: nil defp read_embedding_file(path) do case File.read(path) do {:ok, data} -> case SafeJson.decode(data) do {:ok, list} when is_list(list) -> list _ -> nil end _ -> nil end end defp do_migrate(context) do UI.warn(""" [Embeddings Migration] Detected embeddings from a different model (wrong dimensions). Wiping the file, commit, and conversation indexes and clearing memory embeddings so everything can be regenerated with the local model. """) with {:ok, project} <- Store.get_project() do delete_file_index(project) delete_commit_index(project) delete_conversation_index(project) clear_memory_embeddings(project) end memory_warning() case context do :ask -> UI.info(""" File/commit/conversation indexes cleared; they'll rebuild in the background as the indexer runs. Run `fnord index` to reindex everything immediately. """) :index -> UI.info(""" Indexes cleared. Full reindexing will begin now. """) :read_only -> UI.info(""" File/commit/conversation indexes cleared. This command will return limited or empty results until you run `fnord index` to rebuild. """) end :ok end # Memories are different from the other indexes: their embeddings are # cleared in place (the memory text itself is preserved), and they are # backfilled by background services + `fnord index`. Until a memory is # re-embedded, it is effectively unreachable via semantic search - the # user needs to know that. defp memory_warning do UI.error(""" ================================================================================ MEMORIES ARE TEMPORARILY UNAVAILABLE ================================================================================ All saved memories (project + global) had their embeddings cleared as part of the model migration. The memory text is intact - only the vectors are missing. Until each memory is re-embedded with the new model, semantic recall against it will return nothing. Re-embedding happens slowly in the background via the memory indexer. To force it to finish immediately, run: fnord index ================================================================================ """) end # Nuke the entire files/ tree. Deleting only embeddings.json would leave # stale summaries on disk, generated against the previous embedding-input # contract and prompt. Reindex regenerates summary + embeddings from # scratch. # # This operation is destructive and non-transactional: if the process dies # after wiping some indexes but before others, `detect_stale_embeddings` # on the next run will still trigger correctly as long as *any* surviving # on-disk embedding (file, commit, conversation, memory) has the old # dimension. Memories have an in-place guard (Memory.do_search skips # dim-mismatched entries) and are backfilled by the memory indexer. defp delete_file_index(project) do project |> Store.Project.files_root() |> File.rm_rf() end defp delete_commit_index(project) do project |> Store.Project.CommitIndex.root() |> File.rm_rf() end defp delete_conversation_index(project) do project |> Store.Project.ConversationIndex.root() |> File.rm_rf() end defp clear_memory_embeddings(project) do # Project-scoped memories clear_memory_embeddings_in_dir(Path.join(project.store_path, "memory")) # Global memories clear_memory_embeddings_in_dir(Path.join(Settings.fnord_home(), "memory")) end # Memory files are rewritten in place with embeddings set to nil. A # non-atomic File.write/2 that gets interrupted mid-stream leaves a # truncated file on disk. Since detect_stale_embeddings samples only # one non-nil embedding, a corrupt file plus an otherwise-fully-migrated # on-disk state would skip re-migration AND leave the memory # unreadable by the Memory module. Write atomically via tmp + rename so # the on-disk state is always either the pre-wipe or post-wipe JSON. defp clear_memory_embeddings_in_dir(dir) do dir |> Path.join("*.json") |> Path.wildcard() |> Enum.each(fn path -> case File.read(path) do {:ok, data} -> case SafeJson.decode(data) do {:ok, %{"embeddings" => emb} = memory} when is_list(emb) -> updated = Map.put(memory, "embeddings", nil) case SafeJson.encode(updated) do {:ok, json} -> try do Settings.write_atomic!(path, json) rescue e -> UI.warn( "[Embeddings Migration] Failed to clear embeddings in #{path}: " <> Exception.message(e) ) :ok end _ -> :ok end _ -> :ok end _ -> :ok end end) end end