smartdocs-api / embeddings /embedding_validator.py
Sahilalaknur's picture
Clean HF API deploy (no binary history)
fa1fead
Raw History Blame Contribute Delete
3.41 kB
# embeddings/embedding_validator.py
# WHY THIS EXISTS: Validates that multilingual-e5-large correctly places
# Hindi and English text about the same concept in the same vector space.
# If this fails, retrieval is broken at the foundation. Run before anything else.
# Target: cosine similarity > 0.85 between Hindi and English equivalents.
import sys
from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity
from config.settings import get_settings
settings = get_settings()
def validate_cross_language_similarity(
threshold: float = 0.85,
verbose: bool = True
) -> tuple[float, bool]:
"""
Validates that multilingual-e5-large embeds Hindi and English
equivalents close together in vector space.
Returns:
tuple: (similarity_score, passed)
Raises:
AssertionError: if similarity < threshold
"""
if verbose:
print(f"Loading {settings.embedding_model}...")
print(f"Device: {settings.embedding_device}")
print("")
model = SentenceTransformer(
settings.embedding_model,
device=settings.embedding_device
)
# Test pairs — same concept in Hindi and English
# MANDATORY: "passage: " prefix required by multilingual-e5-large
test_pairs = [
{
"hindi": "passage: भूमि अधिग्रहण मुआवजा",
"english": "passage: land acquisition compensation",
"concept": "land acquisition compensation"
},
{
"hindi": "passage: जीएसटी कर चालान",
"english": "passage: GST tax invoice",
"concept": "GST tax invoice"
},
{
"hindi": "passage: बीमा पॉलिसी दावा",
"english": "passage: insurance policy claim",
"concept": "insurance policy claim"
},
]
scores = []
all_passed = True
for pair in test_pairs:
hindi_emb = model.encode([pair["hindi"]])
english_emb = model.encode([pair["english"]])
sim = cosine_similarity(hindi_emb, english_emb)[0][0]
scores.append(sim)
passed = sim > threshold
if not passed:
all_passed = False
if verbose:
status = "✅" if passed else "❌"
print(f"{status} Concept: {pair['concept']}")
print(f" Similarity: {sim:.4f} (threshold: {threshold})")
print("")
avg_score = sum(scores) / len(scores)
if verbose:
print(f"Average cross-language similarity: {avg_score:.4f}")
print("")
if all_passed:
print("✅ VALIDATION PASSED — Safe to proceed.")
else:
print("❌ VALIDATION FAILED — Do not proceed.")
print(" Fix: delete ~/.cache/huggingface and re-download model.")
return avg_score, all_passed
def run_validation_gate() -> None:
"""
Hard gate — raises SystemExit if validation fails.
Used in CI pipelines and startup checks.
"""
score, passed = validate_cross_language_similarity()
assert passed, (
f"Embedding validation FAILED (score: {score:.4f}, "
f"threshold: 0.85). Do not proceed."
)
if __name__ == "__main__":
score, passed = validate_cross_language_similarity(verbose=True)
if not passed:
sys.exit(1)