feat: complete sprint 10 (Nomic 768-dim model upgrade and schema migration)

api.py

@@ -89,9 +89,13 @@ reranker_model = None
 def get_embedding_model():
     global embedding_model
     if embedding_model is None:
-        print("⏳ Loading Embedding Model on first request...")
-        embedding_model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
-        print("✅ Embedding Model loaded")
+        print("⏳ Loading Nomic Embedding Model (768-dim)...")
+        # PENDING: Run 003_upgrade_embeddings_768.sql in Supabase before this goes live
+        embedding_model = SentenceTransformer(
+            'nomic-ai/nomic-embed-text-v1', 
+            trust_remote_code=True
+        )
+        print("✅ Nomic Model loaded")
     return embedding_model
 
 def get_reranker_model():
@@ -148,7 +152,7 @@ async def health_check():
     checks = {}
     try:
         model = get_embedding_model()
-        test_vec = model.encode("test", normalize_embeddings=True)
+        test_vec = model.encode("search_query: test", normalize_embeddings=True)
         checks["embedding_model"] = {"status": "ok", "dims": len(test_vec)}
     except Exception as e:
         checks["embedding_model"] = {"status": "error", "detail": str(e)}
@@ -188,7 +192,9 @@ async def ingest_document(request: IngestRequest, admin_key: str = ""):
     doc_hash = hashlib.sha256(request.text.encode()).hexdigest()[:16]
     chunks = chunk_text(request.text)
     model = get_embedding_model()
-    embeddings = model.encode(chunks, normalize_embeddings=True, batch_size=32, show_progress_bar=False).tolist()
+    # Nomic requires search_document prefix for schemes/documents
+    prefixed_chunks = [f"search_document: {c}" for c in chunks]
+    embeddings = model.encode(prefixed_chunks, normalize_embeddings=True, batch_size=32, show_progress_bar=False).tolist()
     rows = []
     for i, (chunk, embedding) in enumerate(zip(chunks, embeddings)):
         rows.append({
@@ -221,7 +227,8 @@ async def rag_query(request: Request, query: SearchQuery):
 
         # --- STEP 2: EMBED THE ENGLISH QUERY ---
         model = get_embedding_model()
-        query_numbers = model.encode(english_query, normalize_embeddings=True).tolist()
+        # Nomic requires search_query prefix for citizens' questions
+        query_numbers = model.encode(f"search_query: {english_query}", normalize_embeddings=True).tolist()
         
         # --- STEP 3: SEMANTIC CACHE LOOKUP ---
         cache_result = supabase.rpc("check_query_cache", {

requirements.txt

@@ -1,7 +1,7 @@
 fastapi
 uvicorn[standard]
 torch>=2.2.0
-sentence-transformers
+sentence-transformers>=2.7.0
 huggingface-hub
 groq
 supabase

scripts/reembed_all.py

@@ -0,0 +1,72 @@
+"""
+GovBridge India — Full Re-embedding Script
+Run AFTER: Supabase migration 003 is executed
+Purpose: Re-embed all documents with nomic-embed-text-v1 (768-dim)
+Usage: python3 gov_backend/scripts/reembed_all.py
+"""
+import os
+import time
+from supabase import create_client
+from sentence_transformers import SentenceTransformer
+
+# Load environment variables
+SUPABASE_URL = os.environ.get('SUPABASE_URL')
+SUPABASE_KEY = os.environ.get('SUPABASE_KEY')
+BATCH_SIZE = 50
+
+def main():
+    if not SUPABASE_URL or not SUPABASE_KEY:
+        print("❌ Missing SUPABASE_URL or SUPABASE_KEY")
+        return
+
+    print("⏳ Loading nomic-embed-text-v1 model...")
+    model = SentenceTransformer(
+        'nomic-ai/nomic-embed-text-v1',
+        trust_remote_code=True
+    )
+    print("✅ Model loaded")
+
+    supabase = create_client(SUPABASE_URL, SUPABASE_KEY)
+
+    # Fetch all chunks where embedding is null
+    result = supabase.table('document_chunks') \
+        .select('id, chunk_text, scheme_title') \
+        .is_('embedding', 'null') \
+        .limit(1000) \
+        .execute()
+
+    chunks = result.data or []
+    if not chunks:
+        print("🎉 No chunks found needing re-embedding.")
+        return
+
+    print(f"Found {len(chunks)} chunks to embed. Starting migration...")
+
+    for i in range(0, len(chunks), BATCH_SIZE):
+        batch = chunks[i:i+BATCH_SIZE]
+        # Use Nomic task prefix for documents
+        texts = [
+            f"search_document: {c.get('chunk_text', '') or c.get('scheme_title', '')}"
+            for c in batch
+        ]
+        
+        print(f"🔄 Processing batch {i//BATCH_SIZE + 1}...")
+        embeddings = model.encode(
+            texts,
+            normalize_embeddings=True,
+            show_progress_bar=False
+        ).tolist()
+
+        for chunk, embedding in zip(batch, embeddings):
+            supabase.table('document_chunks') \
+                .update({'embedding': embedding}) \
+                .eq('id', chunk['id']) \
+                .execute()
+
+        print(f"✅ Batch {i//BATCH_SIZE + 1} complete.")
+        time.sleep(0.5)
+
+    print("🎉 All document chunks have been re-embedded with 768 dimensions.")
+
+if __name__ == '__main__':
+    main()