Spaces:
Running
Running
| """Similarity indicators between a quotation and a candidate source (the signals behind the verification score).""" | |
| from __future__ import annotations | |
| import re | |
| import unicodedata | |
| from typing import Dict, List | |
| from alignment import best_region, lcs_length | |
| from normalization import normalize_lenient, normalize_strict, tokenize | |
| try: # optional accelerator; identical formula (1 - distance / max_len) | |
| from rapidfuzz.distance import Levenshtein as _RapidLevenshtein | |
| except ImportError: # pragma: no cover | |
| _RapidLevenshtein = None | |
| def _edit_similarity(a: str, b: str, max_len: int = 600) -> float: | |
| """Normalised Levenshtein similarity in [0, 1] on the first ``max_len`` characters.""" | |
| a, b = a[:max_len], b[:max_len] | |
| if a == b: | |
| return 1.0 | |
| if not a or not b: | |
| return 0.0 | |
| if _RapidLevenshtein is not None: | |
| return float(_RapidLevenshtein.normalized_similarity(a, b)) | |
| prev = list(range(len(b) + 1)) | |
| for i, ca in enumerate(a): | |
| curr = [i + 1] | |
| for j, cb in enumerate(b): | |
| curr.append(min(curr[j] + 1, prev[j + 1] + 1, prev[j] + (ca != cb))) | |
| prev = curr | |
| return 1.0 - prev[-1] / max(len(a), len(b)) | |
| def _light_normalize(text: str) -> str: | |
| """Keeps diacritics (so diacritic changes lower the score) but drops punctuation and tatweel.""" | |
| text = unicodedata.normalize("NFC", text) | |
| text = re.sub(r"[،؛؟!.,:;'\"()\[\]{}<>«»\-_/\\|@#$%^&*+=~`\u0640]", " ", text) | |
| return re.sub(r"\s+", " ", text).strip() | |
| def compute_signals(claim: str, candidate: str, content_type: str = "Ayah", local: bool = True) -> Dict[str, float]: | |
| """Word-, character- and sequence-level similarity indicators between a quotation and a candidate source.""" | |
| is_quran = content_type == "Ayah" | |
| full_candidate = candidate | |
| if local: # score the snippet against the region it refers to, not against the whole verse / Hadith | |
| candidate = best_region(claim, candidate) | |
| normalize = normalize_strict if is_quran else normalize_lenient | |
| norm_claim, norm_cand = normalize(claim), normalize(candidate) | |
| claim_set, cand_set = set(tokenize(norm_claim)), set(tokenize(norm_cand)) | |
| if claim_set and cand_set: | |
| shared = claim_set & cand_set | |
| token_overlap = len(shared) / len(claim_set | cand_set) | |
| coverage = len(shared) / len(claim_set) | |
| else: | |
| token_overlap = coverage = 0.0 | |
| claim_tokens, cand_tokens = tokenize(norm_claim), tokenize(norm_cand) | |
| lcs_ratio = lcs_length(claim_tokens, cand_tokens) / len(claim_tokens) if claim_tokens else 0.0 | |
| edit_sim = _edit_similarity(norm_claim, norm_cand) | |
| claim_chars, cand_chars = set(norm_claim.replace(" ", "")), set(norm_cand.replace(" ", "")) | |
| char_overlap = len(claim_chars & cand_chars) / len(claim_chars | cand_chars) if (claim_chars or cand_chars) else 0.0 | |
| claim_flat, cand_flat = norm_claim.replace(" ", ""), norm_cand.replace(" ", "") | |
| is_substring = int(bool(claim_flat) and bool(cand_flat) and (claim_flat in cand_flat or cand_flat in claim_flat)) | |
| diacritic_sim = ( | |
| _edit_similarity(_light_normalize(claim), _light_normalize(candidate), max_len=800) if is_quran else edit_sim | |
| ) | |
| short = len(claim_tokens) < 4 | |
| if is_quran: | |
| w = ( | |
| dict(coverage=0.35, diacritic_sim=0.30, lcs_ratio=0.15, token_overlap=0.10, char_overlap=0.05, edit_sim=0.05) | |
| if short | |
| else dict(coverage=0.25, diacritic_sim=0.30, lcs_ratio=0.20, token_overlap=0.10, char_overlap=0.05, edit_sim=0.10) | |
| ) | |
| composite = ( | |
| w["coverage"] * coverage + w["diacritic_sim"] * diacritic_sim + w["lcs_ratio"] * lcs_ratio | |
| + w["token_overlap"] * token_overlap + w["char_overlap"] * char_overlap + w["edit_sim"] * edit_sim | |
| ) | |
| if is_substring and diacritic_sim >= 0.60: | |
| composite = max(composite, 0.88) | |
| elif is_substring: | |
| composite = max(composite, 0.75) | |
| else: | |
| w = ( | |
| dict(coverage=0.40, lcs_ratio=0.20, token_overlap=0.20, char_overlap=0.10, edit_sim=0.10) | |
| if short | |
| else dict(coverage=0.30, lcs_ratio=0.28, token_overlap=0.18, char_overlap=0.12, edit_sim=0.12) | |
| ) | |
| composite = ( | |
| w["coverage"] * coverage + w["lcs_ratio"] * lcs_ratio + w["token_overlap"] * token_overlap | |
| + w["char_overlap"] * char_overlap + w["edit_sim"] * edit_sim | |
| ) | |
| if is_substring: | |
| composite = max(composite, 0.82) | |
| return { | |
| "token_overlap": round(token_overlap, 4), | |
| "coverage": round(coverage, 4), | |
| "lcs_ratio": round(lcs_ratio, 4), | |
| "edit_sim": round(edit_sim, 4), | |
| "diacritic_sim": round(diacritic_sim, 4), | |
| "char_overlap": round(char_overlap, 4), | |
| "is_substring": is_substring, | |
| "source_fraction": round(min(1.0, len(claim_tokens) / max(len(tokenize(normalize(full_candidate))), 1)), 4), | |
| "composite": round(composite, 4), | |
| } | |
| def best_match_score(claim: str, candidates: List[dict], content_type: str = "Ayah", local: bool = True): | |
| """Return ``(score, candidate_with_signals)`` for the best-scoring candidate.""" | |
| best_score, best_candidate = 0.0, None | |
| for candidate in candidates: | |
| signals = compute_signals(claim, candidate.get("text", ""), content_type, local) | |
| if signals["composite"] > best_score: | |
| best_score, best_candidate = signals["composite"], {**candidate, "signals": signals} | |
| return best_score, best_candidate | |