Fix RAG remove_directory wiping the entire shared collection (#1660) (#1734)

Removing one RAG directory destroyed the whole shared ChromaDB collection
(all owners + base index) instead of just that directory's chunks. Shared
root cause: PersonalDocsManager.remove_directory called rebuild_index()
(delete_collection + recreate) then re-indexed only the remaining tracked
dirs (ownerless, never personal_dir). The targeted VectorRAG.remove_directory
that should have been used was itself broken (where={"source":{"$contains":dir}}
selects nothing on scalar metadata and would over-delete siblings), and the
dead do_manage_rag path fired a second unconditional rebuild.

- VectorRAG.remove_directory: select chunks in Python by a path-boundary match
  on the stored absolute `source` (dir or dir+os.sep), abspath-normalized.
  Keys on `source` (always written), never `owner` -- no migration.
- PersonalDocsManager.remove_directory: call the targeted remove instead of
  rebuild_index() + partial reindex.
- do_manage_rag (dead code): drop the second rebuild_index() (hygiene).
- rag_server.py add path: abspath so indexed `source` matches the remove.

No schema change. Prevents future wipes (does not recover already-wiped
vectors). Adds hermetic regression tests at three layers.

Fixes #1660

Co-authored-by: Ethan <23321960+0xLeathery@users.noreply.github.com>

src/ai_interaction.py

@@ -1232,9 +1232,11 @@ async def do_manage_rag(content: str, session_id: Optional[str] = None) -> Dict:
 
         try:
             if hasattr(_personal_docs_manager, 'remove_directory'):
+                # Performs a targeted per-directory delete (#1660). The previous
+                # unconditional _rag_manager.rebuild_index() here wiped the whole
+                # collection on every remove (even for untracked dirs) and has
+                # been removed.
                 _personal_docs_manager.remove_directory(directory)
-            if _rag_manager and hasattr(_rag_manager, 'rebuild_index'):
-                _rag_manager.rebuild_index()
             return {"action": "remove_directory", "directory": directory,
                     "results": f"Directory '{directory}' removed from RAG index"}
         except Exception as e:

src/personal_docs.py

@@ -306,18 +306,17 @@ class PersonalDocsManager:
             # Refresh the index to exclude the removed directory
             self.refresh_index()
             
-            # If RAG manager is available, we should rebuild the index
-            # This is a simple approach - in production you might want more sophisticated removal
+            # Targeted delete of just this directory's chunks. This previously
+            # called rag_manager.rebuild_index(), which delete+recreates the
+            # entire shared collection (every owner + the base index) and then
+            # re-indexed only the remaining tracked dirs — ownerless and never
+            # personal_dir — a catastrophic wipe (#1660). remove_directory now
+            # removes exactly this directory's chunks and leaves the rest intact.
             if self.rag_manager:
                 try:
-                    logger.info("Rebuilding RAG index after directory removal")
-                    self.rag_manager.rebuild_index()
-                    # Re-index remaining directories
-                    for dir_path in self.indexed_directories:
-                        if os.path.exists(dir_path):
-                            self.rag_manager.index_personal_documents(dir_path)
+                    self.rag_manager.remove_directory(directory)
                 except Exception as e:
-                    logger.error(f"Failed to rebuild RAG index: {e}")
+                    logger.error(f"Failed to remove directory from RAG index: {e}")
         else:
             logger.info(f"Directory not in index: {directory}")
 

src/rag_vector.py

@@ -380,20 +380,36 @@ class VectorRAG:
             return {'success': False, 'indexed_count': indexed, 'failed_count': failed, 'message': str(e)}
 
     def remove_directory(self, directory: str) -> Dict[str, Any]:
-        """Remove all chunks from a directory. O(1) per chunk via ChromaDB."""
+        """Remove all chunks under ``directory`` (recursively), and nothing else.
+
+        Selection is a Python-side path-boundary match on each chunk's stored
+        ``source`` full path, NOT a Chroma metadata ``where`` filter. No Chroma
+        metadata operator selects a scalar string by path prefix (``$contains``
+        targets document content / list membership, not a ``source`` substring),
+        and a plain substring would over-delete siblings — removing ``/docs``
+        must not touch ``/docs2`` or ``/docs_personal``. We therefore match
+        ``source == directory`` or ``source`` startswith ``directory + os.sep``,
+        the same boundary rule add_directory uses for exclusions. ``directory``
+        is abspath-normalized so it matches the absolute ``source`` that indexing
+        always stores, regardless of how the caller passed it in.
+        """
         if not self.healthy:
             return {"success": False, "message": "Collection not initialized"}
+        directory = os.path.abspath(directory)
         try:
-            # Use ChromaDB where filter to find all docs from this directory
-            results = self._collection.get(
-                where={"source": {"$contains": directory}} if "/" in directory else {"directory": directory},
-                include=["metadatas"],
-            )
-            if not results['ids']:
+            results = self._collection.get(include=["metadatas"])
+            ids = [
+                results["ids"][i]
+                for i, m in enumerate(results["metadatas"])
+                if isinstance(m, dict)
+                and isinstance(m.get("source"), str)
+                and (m["source"] == directory or m["source"].startswith(directory + os.sep))
+            ]
+            if not ids:
                 return {"success": True, "removed_count": 0, "message": "No docs found"}
 
-            self._collection.delete(ids=results['ids'])
-            n = len(results['ids'])
+            self._collection.delete(ids=ids)
+            n = len(ids)
             logger.info(f"Removed {n} chunks from {directory}")
             return {"success": True, "removed_count": n, "message": f"Removed {n} chunks"}
         except Exception as e: