"""ADITI eval harness — THE CONTRACT. Rebuilt from the Training Handover spec (§1,§5). Enums/schemas/prompts are defined HERE, once. contract.py imports and re-exports by reference; the corpus builder never redefines them. Drift becomes an ImportError.""" import os, re, json, hashlib, unicodedata DATA = os.environ.get("ADITI_DATA", "/workspace/aditi/evaldata") DEFAULT_RUNS_DIR= os.environ.get("ADITI_RUNS", "/workspace/aditi/eval_runs") DEFAULT_MANIFEST= os.environ.get("ADITI_MANIFEST", "/workspace/aditi/eval_heldout_manifest.jsonl") # ---------------------------------------------------------------- system prompt SYS_JSON = ("You are ADITI, an OSINT content-analysis engine for Indian-language media. " "You ALWAYS reply with a single valid JSON object and nothing else: no prose, " "no markdown fences, no explanation before or after.") # ---------------------------------------------------------------- enums (handover §1) ENTITY_TYPES = ["PER","ORG","LOC","GPE","FAC","EVENT","MIL_UNIT","WEAPON","DATE","MISC"] MISINFO_LABELS = ["verified","misleading","false","unverified","satire"] HOSTILITY_LABELS = ["none","hate","defamation","offensive","fake","threat"] STANCE_LABELS = ["supportive","opposing","neutral","mixed"] SENTIMENT_LABELS = ["positive","negative","neutral"] SCRIPTS = ["Deva","Latn","Arab","Beng","Gujr","Guru","Knda","Mlym","Orya","Taml", "Telu","Mtei","Olck","Tibt","Sinh","Mixed","Other"] def _obj(props, required, **kw): d = {"type":"object","properties":props,"required":required,"additionalProperties":False} d.update(kw); return d NER_SCHEMA = _obj({"entities":{"type":"array","items":_obj( {"text":{"type":"string"},"type":{"type":"string","enum":["PER","ORG","LOC"]}}, ["text","type"])}}, ["entities"]) SENTIMENT_SCHEMA = _obj({"sentiment":{"type":"string","enum":SENTIMENT_LABELS}, "confidence":{"type":"string"}}, ["sentiment"]) CLAIM_SCHEMA = _obj({"verdict":{"type":"string","enum": ["supported","refuted","not_enough_evidence","conflicting_evidence"]}, "rationale":{"type":"string"}}, ["verdict"]) HALLUC_SCHEMA = _obj({"grounded":{"type":"boolean"}, "unsupported_spans":{"type":"array","items":{"type":"string"}}}, ["grounded"]) GROUNDED_ANSWER_SCHEMA = _obj({"answer":{"type":"string"}}, ["answer"]) MCQ_SCHEMA = _obj({"answer":{"type":"string","pattern":"^[A-J]$"}}, ["answer"]) OSINT_RECORD_KEYS = ["language","script","code_mixed","entities","claims","stance","sentiment", "misinformation","hostility","events","summary_en"] OSINT_RECORD_SCHEMA = _obj({ "language":{"type":"string"}, "script":{"type":"string","enum":SCRIPTS}, "code_mixed":{"type":"boolean"}, "entities":{"type":"array","items":_obj( {"text":{"type":"string"},"type":{"type":"string","enum":ENTITY_TYPES}},["text","type"])}, "claims":{"type":"array","items":_obj( {"claim":{"type":"string"},"verdict":{"type":"string","enum": ["supported","refuted","not_enough_evidence","conflicting_evidence"]}},["claim","verdict"])}, "stance":{"type":"string","enum":STANCE_LABELS}, "sentiment":{"type":"string","enum":SENTIMENT_LABELS}, "misinformation":{"type":"string","enum":MISINFO_LABELS}, "hostility":_obj({"label":{"type":"string","enum":HOSTILITY_LABELS}, "target":{"type":"string"}},["label"]), "events":{"type":"array","items":{"type":"string"}}, "summary_en":{"type":"string"}, }, OSINT_RECORD_KEYS) # AVeriTeC title-case -> enum (handover §5: eval_harness.py:844-849) AVERITEC_MAP = {"Supported":"supported","Refuted":"refuted", "Not Enough Evidence":"not_enough_evidence", "Conflicting Evidence/Cherrypicking":"conflicting_evidence"} # ---------------------------------------------------------------- gates (§5) GATES = {"misinfo_f1":(0.80,"ge"), "faithfulness_rate":(0.95,"ge"), "schema_valid_rate":(1.00,"ge"), "general_regression_pts":(1.00,"le")} # ---------------------------------------------------------------- held-out firewall (§5) HELDOUT_SALT="aditi4-ps18-heldout-v1"; HELDOUT_BUCKETS=100; HELDOUT_EVAL_BUCKETS=1 NLI_DEFAULT_THRESHOLD=0.70 def normalise_ws(t:str)->str: return re.sub(r"\s+"," ", unicodedata.normalize("NFC", t or "")).strip() def content_hash(t:str)->str: return hashlib.sha256(normalise_ws(t).encode("utf-8")).hexdigest() def heldout_bucket(t:str)->int: h=hashlib.sha256((HELDOUT_SALT+"|"+normalise_ws(t)).encode("utf-8")).hexdigest() return int(h[:8],16)%HELDOUT_BUCKETS def is_heldout(t:str)->bool: return heldout_bucket(t) < HELDOUT_EVAL_BUCKETS _SENT = re.compile(r"(?<=[.!?।॥])\s+|\n+") def split_sentences(t:str): return [s.strip() for s in _SENT.split(t or "") if s and s.strip()] # ---------------------------------------------------------------- validation from jsonschema import Draft7Validator _VCACHE={} def validate_schema(payload, schema): """Return None if valid, else an error string (mirrors the original's contract).""" key=id(schema); v=_VCACHE.get(key) if v is None: v=_VCACHE[key]=Draft7Validator(schema) errs=sorted(v.iter_errors(payload), key=lambda e:list(e.path)) return None if not errs else errs[0].message # ---------------------------------------------------------------- prompt templates P_NER = ("Extract named entities from the text. Types: PER, ORG, LOC.\n" "<>\nTEXT:\n{text}\n\nReply JSON: {{\"entities\":[{{\"text\":..,\"type\":..}}]}}") P_SENT = ("Classify the sentiment of the text as positive, negative or neutral.\n" "<>\nTEXT:\n{text}\n\nReply JSON: {{\"sentiment\":..}}") P_CLAIM = ("Adjudicate the CLAIM against the EVIDENCE. verdict must be one of " "supported, refuted, not_enough_evidence, conflicting_evidence.\n" "<>\nCLAIM:\n{claim}\n\nEVIDENCE:\n{evidence}\n\nReply JSON: {{\"verdict\":..}}") P_HALLUC = ("Decide whether the RESPONSE is fully grounded in the SOURCE. List any unsupported spans.\n" "<>\nSOURCE:\n{source}\n\nRESPONSE:\n{response}\n\n" "Reply JSON: {{\"grounded\":true|false,\"unsupported_spans\":[..]}}") P_GROUND_GEN = ("Answer the QUESTION using ONLY the SOURCE. If the source does not answer it, say so.\n" "<>\nSOURCE:\n{source}\n\nQUESTION:\n{question}\n\nReply JSON: {{\"answer\":..}}") P_MCQ = ("Answer the multiple-choice question with a single letter.\n" "<>\n{question}\n{choices}\n\nReply JSON: {{\"answer\":\"A\"}}") P_SCHEMA = ("Produce the full OSINT record for the text. Required keys: " + ", ".join(OSINT_RECORD_KEYS) + ".\n" "entity types: {ent}\nstance: {stance}\nsentiment: {sent}\n" "misinformation: {mis}\nhostility: {host}\n" "<>\nTEXT:\n{text}\n\nReply with the JSON record only.") def schema_prompt(iid, text): return P_SCHEMA.format(iid=iid, text=text, ent="|".join(ENTITY_TYPES), stance="|".join(STANCE_LABELS), sent="|".join(SENTIMENT_LABELS), mis="|".join(MISINFO_LABELS), host="|".join(HOSTILITY_LABELS)) if __name__=="__main__": assert len(SYS_JSON)==203, f"SYS_JSON is {len(SYS_JSON)} chars, spec says 203" assert validate_schema({"verdict":"supported","cited_evidence":[0],"confidence":"medium"}, CLAIM_SCHEMA) is not None, "v2's claim payload must be schema-invalid" assert validate_schema({"verdict":"insufficient_evidence"}, CLAIM_SCHEMA) is not None assert validate_schema({"verdict":"supported"}, CLAIM_SCHEMA) is None assert validate_schema({"entities":[{"text":"x","type":"PERSON"}]}, NER_SCHEMA) is not None print("harness: all guardrail tests pass") print(" SYS_JSON len", len(SYS_JSON)) print(" verdicts ", CLAIM_SCHEMA["properties"]["verdict"]["enum"]) print(" ner types", NER_SCHEMA["properties"]["entities"]["items"]["properties"]["type"]["enum"]) print(" misinfo ", MISINFO_LABELS) print(" hostility", HOSTILITY_LABELS) print(" stance ", STANCE_LABELS) print(" heldout%% ", HELDOUT_EVAL_BUCKETS/HELDOUT_BUCKETS*100)