Spaces:
Running on Zero
Running on Zero
| """Lexical scoring primitives shared by runtime gates and evaluation.""" | |
| import re | |
| import unicodedata | |
| _TOKEN_RE = re.compile(r"[a-z0-9]+") | |
| HINDI_MONTHS = { | |
| "जनवरी": 1, "फरवरी": 2, "मार्च": 3, "अप्रैल": 4, "मई": 5, "जून": 6, | |
| "जुलाई": 7, "अगस्त": 8, "सितंबर": 9, "सितम्बर": 9, "अक्टूबर": 10, | |
| "नवंबर": 11, "नवम्बर": 11, "दिसंबर": 12, "दिसम्बर": 12, | |
| } | |
| def tokens(text: str) -> list[str]: | |
| """Lowercase alphanumeric tokens (ROUGE-style basic tokenization).""" | |
| return _TOKEN_RE.findall(text.lower()) | |
| def word_base(token: str) -> str: | |
| """Crude morphological base: strips a plural 's' so "rights" and | |
| "right" match (legal prose leans heavily on pluralized nouns).""" | |
| return token[:-1] if len(token) > 3 and token.endswith("s") else token | |
| STOPWORDS = frozenset( | |
| [ | |
| "a", | |
| "an", | |
| "and", | |
| "are", | |
| "as", | |
| "at", | |
| "be", | |
| "by", | |
| "for", | |
| "from", | |
| "has", | |
| "have", | |
| "in", | |
| "is", | |
| "it", | |
| "its", | |
| "of", | |
| "on", | |
| "or", | |
| "that", | |
| "the", | |
| "to", | |
| "under", | |
| "with", | |
| "within", | |
| "without", | |
| "shall", | |
| "may", | |
| "must", | |
| ] | |
| ) | |
| _RESEARCH_STOP = STOPWORDS | frozenset( | |
| ["what", "how", "when", "why", "which", "who", "does", "do", "did", "was", "were", "can", "could", "should", "would", "find", "show", "give", "tell", "about", "explain", "summarize", "summarise", "please", "judgment", "judgement", "judgments", "judgements", "case", "cases", "court", "supreme", "india", "indian", "decision", "decisions", "verdict", "holding", "outcome", "reason", "reasons", "precedent", "precedents", "authority", "authorities", "final", "latest", "change", "changed", "happened", "order", "orders", "year", "date", "क्या", "कब", "क्यों", "कैसे", "कौन", "का", "की", "के", "को", "में", "से", "पर", "और", "है", "हैं", "था", "थी", "थे", "मुझे", "बताओ", "बताइए", "समझाओ", "फैसला", "फ़ैसला", "निर्णय", "अदालत", "न्यायालय"] | |
| ) | |
| def research_terms(text: str) -> set[str]: | |
| """Issue terms for selecting excerpts; preserve Devanagari vowel marks. | |
| This does not rewrite the request, resolve a case or assert legal support. | |
| Existing English-only evaluation tokenization remains unchanged. | |
| """ | |
| normalized = unicodedata.normalize("NFKC", text).casefold() | |
| normalized = normalized.translate(str.maketrans("०१२३४५६७८९", "0123456789")) | |
| return { | |
| word_base(term) | |
| for term in re.findall(r"[^\W_][\w\u0900-\u0903\u093a-\u094f\u0951-\u0957\u0962-\u0963]*", normalized) | |
| if (len(term) > 1 or term.isdigit()) and term not in _RESEARCH_STOP | |
| } | |