File size: 3,410 Bytes
fa1fead
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
# embeddings/embedding_validator.py
# WHY THIS EXISTS: Validates that multilingual-e5-large correctly places
# Hindi and English text about the same concept in the same vector space.
# If this fails, retrieval is broken at the foundation. Run before anything else.
# Target: cosine similarity > 0.85 between Hindi and English equivalents.

import sys
from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity

from config.settings import get_settings

settings = get_settings()


def validate_cross_language_similarity(
    threshold: float = 0.85,
    verbose: bool = True
) -> tuple[float, bool]:
    """
    Validates that multilingual-e5-large embeds Hindi and English
    equivalents close together in vector space.

    Returns:
        tuple: (similarity_score, passed)

    Raises:
        AssertionError: if similarity < threshold
    """
    if verbose:
        print(f"Loading {settings.embedding_model}...")
        print(f"Device: {settings.embedding_device}")
        print("")

    model = SentenceTransformer(
        settings.embedding_model,
        device=settings.embedding_device
    )

    # Test pairs — same concept in Hindi and English
    # MANDATORY: "passage: " prefix required by multilingual-e5-large
    test_pairs = [
        {
            "hindi": "passage: भूमि अधिग्रहण मुआवजा",
            "english": "passage: land acquisition compensation",
            "concept": "land acquisition compensation"
        },
        {
            "hindi": "passage: जीएसटी कर चालान",
            "english": "passage: GST tax invoice",
            "concept": "GST tax invoice"
        },
        {
            "hindi": "passage: बीमा पॉलिसी दावा",
            "english": "passage: insurance policy claim",
            "concept": "insurance policy claim"
        },
    ]

    scores = []
    all_passed = True

    for pair in test_pairs:
        hindi_emb = model.encode([pair["hindi"]])
        english_emb = model.encode([pair["english"]])
        sim = cosine_similarity(hindi_emb, english_emb)[0][0]
        scores.append(sim)
        passed = sim > threshold

        if not passed:
            all_passed = False

        if verbose:
            status = "✅" if passed else "❌"
            print(f"{status} Concept: {pair['concept']}")
            print(f"   Similarity: {sim:.4f} (threshold: {threshold})")
            print("")

    avg_score = sum(scores) / len(scores)

    if verbose:
        print(f"Average cross-language similarity: {avg_score:.4f}")
        print("")
        if all_passed:
            print("✅ VALIDATION PASSED — Safe to proceed.")
        else:
            print("❌ VALIDATION FAILED — Do not proceed.")
            print("   Fix: delete ~/.cache/huggingface and re-download model.")

    return avg_score, all_passed


def run_validation_gate() -> None:
    """
    Hard gate — raises SystemExit if validation fails.
    Used in CI pipelines and startup checks.
    """
    score, passed = validate_cross_language_similarity()

    assert passed, (
        f"Embedding validation FAILED (score: {score:.4f}, "
        f"threshold: 0.85). Do not proceed."
    )


if __name__ == "__main__":
    score, passed = validate_cross_language_similarity(verbose=True)
    if not passed:
        sys.exit(1)