Spaces:
Sleeping
Sleeping
File size: 2,038 Bytes
fa1fead | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | # smartdocs/validate_embeddings.py
#
# WHY THIS EXISTS:
# multilingual-e5-large must place Hindi and English text about the
# same concept close together in vector space. If it doesn't, retrieval
# fails silently — Hindi queries return wrong or empty results.
# This test proves the model works correctly before we build anything else.
# Run this ONCE before Part 1. If it fails, do not write any other code.
from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity
print("Loading multilingual-e5-large model...")
print("First run: downloads ~1.1GB from HuggingFace. This takes 2-5 minutes.")
print("Subsequent runs: loads from local cache in seconds.")
print("")
model = SentenceTransformer('intfloat/multilingual-e5-large')
print("Model loaded. Running cross-language similarity test...")
print("")
# MANDATORY: multilingual-e5-large requires "passage: " prefix at ingestion
# Without this prefix, similarity scores DROP by 0.05-0.15
# This is not optional — it is how the model was trained
hindi = model.encode(["passage: भूमि अधिग्रहण मुआवजा"]) # land acquisition compensation
english = model.encode(["passage: land acquisition compensation"])
sim = cosine_similarity(hindi, english)[0][0]
print(f"Hindi text: भूमि अधिग्रहण मुआवजा")
print(f"English text: land acquisition compensation")
print(f"")
print(f"Cross-language similarity: {sim:.4f}")
print("")
if sim > 0.85:
print("✅ PASSED — Embeddings place Hindi and English in the same vector space.")
print("✅ Retrieval pipeline is validated. Safe to proceed to Part 1.")
else:
print(f"❌ FAILED — Similarity is {sim:.4f}, must be above 0.85.")
print("❌ DO NOT PROCEED. Debug model loading first.")
print(" Most likely cause: model downloaded incorrectly.")
print(" Fix: delete the .cache/huggingface folder and re-run.")
assert sim > 0.85, f"VALIDATION FAILED: {sim:.4f} — do not proceed to Part 1" |