diff --git a/src/rag_vector.py b/src/rag_vector.py index fcb27c1..f1df5b8 100644 --- a/src/rag_vector.py +++ b/src/rag_vector.py @@ -7,6 +7,7 @@ configurable embedding endpoint via EMBEDDING_URL env var. """ import os +import hashlib import re import logging import numpy as np @@ -26,6 +27,10 @@ KEYWORD_WEIGHT = 0.3 COLLECTION_NAME = "odysseus_rag" +def _generate_doc_id(text: str) -> str: + return f"doc_{hashlib.sha256(text.encode('utf-8')).hexdigest()[:16]}" + + class VectorRAG: """RAG system using ChromaDB vector storage with hybrid search.""" @@ -99,7 +104,7 @@ class VectorRAG: return False try: - doc_id = f"doc_{hash(text) % 10**16}" + doc_id = _generate_doc_id(text) # Check if already exists existing = self._collection.get(ids=[doc_id]) if existing["ids"]: @@ -135,7 +140,7 @@ class VectorRAG: new_metas = [] new_ids = [] for t, m in valid: - doc_id = f"doc_{hash(t) % 10**16}" + doc_id = _generate_doc_id(t) existing = self._collection.get(ids=[doc_id]) if not existing["ids"]: new_texts.append(t) diff --git a/tests/test_rag_vector_id_stability.py b/tests/test_rag_vector_id_stability.py new file mode 100644 index 0000000..c9d2656 --- /dev/null +++ b/tests/test_rag_vector_id_stability.py @@ -0,0 +1,28 @@ +import os +import subprocess +import pytest + +def test_rag_id_stability_across_processes(): + # Run helper in subprocesses with different PYTHONHASHSEED values to ensure cross-process stability + cmd = ["./venv/bin/python", "-c", "from src.rag_vector import _generate_doc_id; print(_generate_doc_id('test_text_hash'))"] + + env0 = os.environ.copy() + env0["PYTHONHASHSEED"] = "0" + id0 = subprocess.check_output(cmd, env=env0).decode().strip() + + env1 = os.environ.copy() + env1["PYTHONHASHSEED"] = "1" + id1 = subprocess.check_output(cmd, env=env1).decode().strip() + + env_rand = os.environ.copy() + env_rand["PYTHONHASHSEED"] = "random" + id_rand = subprocess.check_output(cmd, env=env_rand).decode().strip() + + # Assert they are all equal (deterministic across seeds and processes) + assert id0 == id1 + assert id0 == id_rand + + # Assert different inputs produce different IDs + cmd_diff = ["./venv/bin/python", "-c", "from src.rag_vector import _generate_doc_id; print(_generate_doc_id('different_text_hash'))"] + id_diff = subprocess.check_output(cmd_diff, env=env0).decode().strip() + assert id0 != id_diff