| """ADITI eval harness — THE CONTRACT. Rebuilt from the Training Handover spec (§1,§5). |
| Enums/schemas/prompts are defined HERE, once. contract.py imports and re-exports by |
| reference; the corpus builder never redefines them. Drift becomes an ImportError.""" |
| import os, re, json, hashlib, unicodedata |
|
|
| DATA = os.environ.get("ADITI_DATA", "/workspace/aditi/evaldata") |
| DEFAULT_RUNS_DIR= os.environ.get("ADITI_RUNS", "/workspace/aditi/eval_runs") |
| DEFAULT_MANIFEST= os.environ.get("ADITI_MANIFEST", "/workspace/aditi/eval_heldout_manifest.jsonl") |
|
|
| |
| SYS_JSON = ("You are ADITI, an OSINT content-analysis engine for Indian-language media. " |
| "You ALWAYS reply with a single valid JSON object and nothing else: no prose, " |
| "no markdown fences, no explanation before or after.") |
|
|
| |
| ENTITY_TYPES = ["PER","ORG","LOC","GPE","FAC","EVENT","MIL_UNIT","WEAPON","DATE","MISC"] |
| MISINFO_LABELS = ["verified","misleading","false","unverified","satire"] |
| HOSTILITY_LABELS = ["none","hate","defamation","offensive","fake","threat"] |
| STANCE_LABELS = ["supportive","opposing","neutral","mixed"] |
| SENTIMENT_LABELS = ["positive","negative","neutral"] |
| SCRIPTS = ["Deva","Latn","Arab","Beng","Gujr","Guru","Knda","Mlym","Orya","Taml", |
| "Telu","Mtei","Olck","Tibt","Sinh","Mixed","Other"] |
|
|
| def _obj(props, required, **kw): |
| d = {"type":"object","properties":props,"required":required,"additionalProperties":False} |
| d.update(kw); return d |
|
|
| NER_SCHEMA = _obj({"entities":{"type":"array","items":_obj( |
| {"text":{"type":"string"},"type":{"type":"string","enum":["PER","ORG","LOC"]}}, |
| ["text","type"])}}, ["entities"]) |
| SENTIMENT_SCHEMA = _obj({"sentiment":{"type":"string","enum":SENTIMENT_LABELS}, |
| "confidence":{"type":"string"}}, ["sentiment"]) |
| CLAIM_SCHEMA = _obj({"verdict":{"type":"string","enum": |
| ["supported","refuted","not_enough_evidence","conflicting_evidence"]}, |
| "rationale":{"type":"string"}}, ["verdict"]) |
| HALLUC_SCHEMA = _obj({"grounded":{"type":"boolean"}, |
| "unsupported_spans":{"type":"array","items":{"type":"string"}}}, ["grounded"]) |
| GROUNDED_ANSWER_SCHEMA = _obj({"answer":{"type":"string"}}, ["answer"]) |
| MCQ_SCHEMA = _obj({"answer":{"type":"string","pattern":"^[A-J]$"}}, ["answer"]) |
|
|
| OSINT_RECORD_KEYS = ["language","script","code_mixed","entities","claims","stance","sentiment", |
| "misinformation","hostility","events","summary_en"] |
| OSINT_RECORD_SCHEMA = _obj({ |
| "language":{"type":"string"}, |
| "script":{"type":"string","enum":SCRIPTS}, |
| "code_mixed":{"type":"boolean"}, |
| "entities":{"type":"array","items":_obj( |
| {"text":{"type":"string"},"type":{"type":"string","enum":ENTITY_TYPES}},["text","type"])}, |
| "claims":{"type":"array","items":_obj( |
| {"claim":{"type":"string"},"verdict":{"type":"string","enum": |
| ["supported","refuted","not_enough_evidence","conflicting_evidence"]}},["claim","verdict"])}, |
| "stance":{"type":"string","enum":STANCE_LABELS}, |
| "sentiment":{"type":"string","enum":SENTIMENT_LABELS}, |
| "misinformation":{"type":"string","enum":MISINFO_LABELS}, |
| "hostility":_obj({"label":{"type":"string","enum":HOSTILITY_LABELS}, |
| "target":{"type":"string"}},["label"]), |
| "events":{"type":"array","items":{"type":"string"}}, |
| "summary_en":{"type":"string"}, |
| }, OSINT_RECORD_KEYS) |
|
|
| |
| AVERITEC_MAP = {"Supported":"supported","Refuted":"refuted", |
| "Not Enough Evidence":"not_enough_evidence", |
| "Conflicting Evidence/Cherrypicking":"conflicting_evidence"} |
|
|
| |
| GATES = {"misinfo_f1":(0.80,"ge"), "faithfulness_rate":(0.95,"ge"), |
| "schema_valid_rate":(1.00,"ge"), "general_regression_pts":(1.00,"le")} |
|
|
| |
| HELDOUT_SALT="aditi4-ps18-heldout-v1"; HELDOUT_BUCKETS=100; HELDOUT_EVAL_BUCKETS=1 |
| NLI_DEFAULT_THRESHOLD=0.70 |
|
|
| def normalise_ws(t:str)->str: |
| return re.sub(r"\s+"," ", unicodedata.normalize("NFC", t or "")).strip() |
| def content_hash(t:str)->str: |
| return hashlib.sha256(normalise_ws(t).encode("utf-8")).hexdigest() |
| def heldout_bucket(t:str)->int: |
| h=hashlib.sha256((HELDOUT_SALT+"|"+normalise_ws(t)).encode("utf-8")).hexdigest() |
| return int(h[:8],16)%HELDOUT_BUCKETS |
| def is_heldout(t:str)->bool: |
| return heldout_bucket(t) < HELDOUT_EVAL_BUCKETS |
|
|
| _SENT = re.compile(r"(?<=[.!?।॥])\s+|\n+") |
| def split_sentences(t:str): |
| return [s.strip() for s in _SENT.split(t or "") if s and s.strip()] |
|
|
| |
| from jsonschema import Draft7Validator |
| _VCACHE={} |
| def validate_schema(payload, schema): |
| """Return None if valid, else an error string (mirrors the original's contract).""" |
| key=id(schema); v=_VCACHE.get(key) |
| if v is None: v=_VCACHE[key]=Draft7Validator(schema) |
| errs=sorted(v.iter_errors(payload), key=lambda e:list(e.path)) |
| return None if not errs else errs[0].message |
|
|
| |
| P_NER = ("Extract named entities from the text. Types: PER, ORG, LOC.\n" |
| "<<ITEM:{iid}>>\nTEXT:\n{text}\n\nReply JSON: {{\"entities\":[{{\"text\":..,\"type\":..}}]}}") |
| P_SENT = ("Classify the sentiment of the text as positive, negative or neutral.\n" |
| "<<ITEM:{iid}>>\nTEXT:\n{text}\n\nReply JSON: {{\"sentiment\":..}}") |
| P_CLAIM = ("Adjudicate the CLAIM against the EVIDENCE. verdict must be one of " |
| "supported, refuted, not_enough_evidence, conflicting_evidence.\n" |
| "<<ITEM:{iid}>>\nCLAIM:\n{claim}\n\nEVIDENCE:\n{evidence}\n\nReply JSON: {{\"verdict\":..}}") |
| P_HALLUC = ("Decide whether the RESPONSE is fully grounded in the SOURCE. List any unsupported spans.\n" |
| "<<ITEM:{iid}>>\nSOURCE:\n{source}\n\nRESPONSE:\n{response}\n\n" |
| "Reply JSON: {{\"grounded\":true|false,\"unsupported_spans\":[..]}}") |
| P_GROUND_GEN = ("Answer the QUESTION using ONLY the SOURCE. If the source does not answer it, say so.\n" |
| "<<ITEM:{iid}>>\nSOURCE:\n{source}\n\nQUESTION:\n{question}\n\nReply JSON: {{\"answer\":..}}") |
| P_MCQ = ("Answer the multiple-choice question with a single letter.\n" |
| "<<ITEM:{iid}>>\n{question}\n{choices}\n\nReply JSON: {{\"answer\":\"A\"}}") |
| P_SCHEMA = ("Produce the full OSINT record for the text. Required keys: " + ", ".join(OSINT_RECORD_KEYS) + ".\n" |
| "entity types: {ent}\nstance: {stance}\nsentiment: {sent}\n" |
| "misinformation: {mis}\nhostility: {host}\n" |
| "<<ITEM:{iid}>>\nTEXT:\n{text}\n\nReply with the JSON record only.") |
|
|
| def schema_prompt(iid, text): |
| return P_SCHEMA.format(iid=iid, text=text, ent="|".join(ENTITY_TYPES), |
| stance="|".join(STANCE_LABELS), sent="|".join(SENTIMENT_LABELS), |
| mis="|".join(MISINFO_LABELS), host="|".join(HOSTILITY_LABELS)) |
|
|
| if __name__=="__main__": |
| assert len(SYS_JSON)==203, f"SYS_JSON is {len(SYS_JSON)} chars, spec says 203" |
| assert validate_schema({"verdict":"supported","cited_evidence":[0],"confidence":"medium"}, |
| CLAIM_SCHEMA) is not None, "v2's claim payload must be schema-invalid" |
| assert validate_schema({"verdict":"insufficient_evidence"}, CLAIM_SCHEMA) is not None |
| assert validate_schema({"verdict":"supported"}, CLAIM_SCHEMA) is None |
| assert validate_schema({"entities":[{"text":"x","type":"PERSON"}]}, NER_SCHEMA) is not None |
| print("harness: all guardrail tests pass") |
| print(" SYS_JSON len", len(SYS_JSON)) |
| print(" verdicts ", CLAIM_SCHEMA["properties"]["verdict"]["enum"]) |
| print(" ner types", NER_SCHEMA["properties"]["entities"]["items"]["properties"]["type"]["enum"]) |
| print(" misinfo ", MISINFO_LABELS) |
| print(" hostility", HOSTILITY_LABELS) |
| print(" stance ", STANCE_LABELS) |
| print(" heldout%% ", HELDOUT_EVAL_BUCKETS/HELDOUT_BUCKETS*100) |
|
|
