"""contract.py — redefines NOTHING. Puts the harness on sys.path, imports it, and re-exports its objects BY REFERENCE. Handover §1: "the scorer IS the contract".""" import os, sys, json, hashlib, re _HARNESS_DIR = "/workspace/aditi/harness" sys.path.insert(0, _HARNESS_DIR) os.environ.setdefault("ADITI_DATA", "/workspace/aditi/evaldata") import eval_harness as E # the scorer IS the contract SYS_JSON = E.SYS_JSON P_NER, P_SENT, P_CLAIM = E.P_NER, E.P_SENT, E.P_CLAIM P_HALLUC, P_GROUND_GEN, P_MCQ = E.P_HALLUC, E.P_GROUND_GEN, E.P_MCQ schema_prompt = E.schema_prompt NER_SCHEMA, SENTIMENT_SCHEMA = E.NER_SCHEMA, E.SENTIMENT_SCHEMA CLAIM_SCHEMA, HALLUC_SCHEMA = E.CLAIM_SCHEMA, E.HALLUC_SCHEMA GROUNDED_ANSWER_SCHEMA, MCQ_SCHEMA = E.GROUNDED_ANSWER_SCHEMA, E.MCQ_SCHEMA OSINT_RECORD_SCHEMA = E.OSINT_RECORD_SCHEMA AVERITEC_MAP = E.AVERITEC_MAP validate_schema, is_heldout = E.validate_schema, E.is_heldout content_hash, normalise_ws, split_sentences = E.content_hash, E.normalise_ws, E.split_sentences # enums DERIVED from schema objects, never retyped (handover §1) ENTITY_TYPES = E.ENTITY_TYPES NER_TYPES = NER_SCHEMA["properties"]["entities"]["items"]["properties"]["type"]["enum"] VERDICTS = CLAIM_SCHEMA["properties"]["verdict"]["enum"] SCRIPTS = OSINT_RECORD_SCHEMA["properties"]["script"]["enum"] SENTIMENT_LABELS = E.SENTIMENT_LABELS STANCE_LABELS = E.STANCE_LABELS HOSTILITY_LABELS = E.HOSTILITY_LABELS MISINFO_LABELS = E.MISINFO_LABELS # ---- label maps: source-native key -> harness enum value. None means DROP, never rename. _VERDICT = { "supported":"supported","support":"supported","supports":"supported","true":"supported", "refuted":"refuted","refutes":"refuted","false":"refuted","refute":"refuted", "not enough evidence":"not_enough_evidence","nei":"not_enough_evidence", "not enough info":"not_enough_evidence","notenoughinfo":"not_enough_evidence", "conflicting evidence/cherrypicking":"conflicting_evidence","conflicting":"conflicting_evidence", "conflicting evidence":"conflicting_evidence","mixture":"conflicting_evidence", "half true":"conflicting_evidence","partly true/misleading":"conflicting_evidence", "mostly true":"supported","mostly false":"refuted", "complicated/hard to categorise":"not_enough_evidence","other":None, "unproven":"not_enough_evidence", } _ENTITY = {"PER":"PER","PERSON":"PER","B-PER":"PER","I-PER":"PER", "ORG":"ORG","ORGANISATION":"ORG","ORGANIZATION":"ORG","B-ORG":"ORG","I-ORG":"ORG", "LOC":"LOC","GPE":"LOC","B-LOC":"LOC","I-LOC":"LOC","LOCATION":"LOC", # no slot in the NER gate enum -> DROP (this is what let PERSON reach the model in v2) "MISC":None,"O":None,"PRODUCT":None,"EVENT":None,"DATE":None,"CARDINAL":None, "WORK_OF_ART":None,"LAW":None,"NORP":None,"FAC":None,"TIME":None,"MONEY":None, "PERCENT":None,"QUANTITY":None,"ORDINAL":None,"LANGUAGE":None} _SENTIMENT = {"positive":"positive","negative":"negative","neutral":"neutral", "1":"positive","-1":"negative","0":"neutral","pos":"positive","neg":"negative", "Positive":"positive","Negative":"negative","Neutral":"neutral"} _STANCE = {"supportive":"supportive","agree":"supportive","support":"supportive","for":"supportive", "opposing":"opposing","disagree":"opposing","against":"opposing","refute":"opposing", "neutral":"neutral","discuss":"neutral","unrelated":"neutral","observing":"neutral", "mixed":"mixed","comment":"neutral"} _HOSTILITY = {"none":"none","not-abusive":"none","non-hostile":"none","normal":"none","nh":"none", "hate":"hate","hs":"hate","hateful":"hate","religious hate":"hate", "defamation":"defamation","fake":"fake","threat":"threat", "offensive":"offensive","abusive":"offensive","offn":"offensive","prfn":"offensive", "profane/untargeted":"offensive","sexism":"offensive","aggressive":"offensive"} _MISINFO = {"verified":"verified","true":"verified","real":"verified", "misleading":"misleading","partly true":"misleading","half true":"misleading", "false":"false","fake":"false","mostly false":"false", "unverified":"unverified","unproven":"unverified","satire":"satire","humor":"satire"} _LANG2SCRIPT = {"hi":"Deva","mr":"Deva","ne":"Deva","sa":"Deva","kok":"Deva","doi":"Deva","mai":"Deva", "bn":"Beng","as":"Beng","gu":"Gujr","pa":"Guru","kn":"Knda","ml":"Mlym","or":"Orya", "ta":"Taml","te":"Telu","ur":"Arab","sd":"Arab","ps":"Arab","ks":"Arab","en":"Latn", "mni":"Mtei","sat":"Olck","bo":"Tibt","si":"Sinh"} _TABLES = {"verdict":(_VERDICT,VERDICTS),"entity":(_ENTITY,NER_TYPES), "sentiment":(_SENTIMENT,SENTIMENT_LABELS),"stance":(_STANCE,STANCE_LABELS), "hostility":(_HOSTILITY,HOSTILITY_LABELS),"misinfo":(_MISINFO,MISINFO_LABELS)} for _slot,(_tbl,_enum) in _TABLES.items(): for _k,_v in _tbl.items(): if _v is not None and _v not in _enum: raise ImportError(f"contract.py: {_slot} maps {_k!r} -> {_v!r}, " f"not in harness enum {list(_enum)}") for _v in _LANG2SCRIPT.values(): if _v not in SCRIPTS: raise ImportError(f"contract.py: script {_v!r} not in harness script enum") class UnmappedLabel(ValueError): pass def canon(slot, raw, strict=True): """exact -> case-insensitive -> already-canonical passthrough -> raise.""" tbl,enum=_TABLES[slot] if raw is None: return None if raw in tbl: return tbl[raw] s=str(raw).strip() if s in tbl: return tbl[s] lo=s.lower() for k,v in tbl.items(): if str(k).lower()==lo: return v if s in enum: return s if strict: raise UnmappedLabel(f"{slot}: {raw!r} unmapped") return None def bio_entities(words, tags): """strip B-/I-/E-/S- and route every tag through canon(entity, strict=False).""" out=[];cur=None;buf=[] for w,t in zip(words,tags): t=(t or "O").strip().replace("\r","") pre,_,rest = t.partition("-") base = rest if pre in ("B","I","E","S") and rest else t typ = canon("entity", base, strict=False) if typ and (pre in ("B","S") or cur!=typ): if cur and buf: out.append({"text":" ".join(buf),"type":cur}) cur,buf=typ,[w] elif typ and cur==typ: buf.append(w) else: if cur and buf: out.append({"text":" ".join(buf),"type":cur}) cur,buf=None,[] if cur and buf: out.append({"text":" ".join(buf),"type":cur}) return out def script_for(lang, text=""): s=_LANG2SCRIPT.get((lang or "").lower()) if s: return s import unicodedata as U BLK={"DEVANAGARI":"Deva","BENGALI":"Beng","GUJARATI":"Gujr","GURMUKHI":"Guru", "KANNADA":"Knda","MALAYALAM":"Mlym","ORIYA":"Orya","TAMIL":"Taml", "TELUGU":"Telu","ARABIC":"Arab","LATIN":"Latn","SINHALA":"Sinh","TIBETAN":"Tibt"} counts={} for ch in (text or "")[:400]: if ch.isspace() or not ch.isalpha(): continue try: nm=U.name(ch).split()[0] except ValueError: continue k=BLK.get(nm) if k: counts[k]=counts.get(k,0)+1 if not counts: return "Other" top=max(counts,key=counts.get); n=counts[top] if top!="Latn" and counts.get("Latn",0) > n*0.4: return "Mixed" return top if __name__=="__main__": assert canon("verdict","Conflicting Evidence/Cherrypicking")=="conflicting_evidence" assert canon("entity","MISC") is None, "MISC has no slot in the NER gate enum" assert canon("entity","PERSON")=="PER" assert validate_schema({"verdict":"supported","cited_evidence":[0],"confidence":"medium"}, CLAIM_SCHEMA) is not None, "v2's claim payload should be schema-invalid" assert canon("hostility","abusive")=="offensive" and canon("hostility","not-abusive")=="none" assert script_for("hi")=="Deva" and script_for("ur")=="Arab" print("contract: all guardrail tests pass") print(" verdicts ",VERDICTS); print(" ner types",NER_TYPES) print(" misinfo ",MISINFO_LABELS); print(" hostility",HOSTILITY_LABELS) print(" stance ",STANCE_LABELS) # ------------------------------------------------------------------ Emitter (§1) class Emitter: """Nothing reaches the JSONL except through emit(). Refusal order: budget -> share cap -> hard cap -> contamination -> schema -> dedup.""" def __init__(self, path, budget, tokenizer_dir=None, share_cap=0.12, hard_cap=None): self.path=path; self.budget=dict(budget); self.share_cap=share_cap self.hard_cap=hard_cap or {} self.spent={k:0 for k in budget}; self.per_src={} self.seen=set(); self.invalid={}; self.dropped={} self.fh=open(path,"w",encoding="utf-8"); self.n=0 self._tok=None; self._tokdir=tokenizer_dir def tok(self): if self._tok is None: from transformers import AutoTokenizer self._tok=AutoTokenizer.from_pretrained(self._tokdir) return self._tok def ntok(self,s): return len(self.tok()(s, add_special_tokens=False)["input_ids"]) def _bump(self,d,k): d[k]=d.get(k,0)+1 def emit(self, bucket, source, user, payload, schema, heldout_text=None, system=None): system = system or SYS_JSON if self.spent.get(bucket,0) >= self.budget.get(bucket,0): self._bump(self.dropped,f"{bucket}:budget"); return False a=json.dumps(payload, ensure_ascii=False, separators=(",",":")) t=self.ntok(a) frac=min(self.share_cap, self.hard_cap.get((bucket,source), 1.0)) cap=frac*self.budget.get(bucket,0) if self.per_src.get((bucket,source),0)+t > cap: self._bump(self.dropped,f"{bucket}/{source}:sharecap"); return False if is_heldout(normalise_ws(heldout_text if heldout_text is not None else user)): self._bump(self.dropped,f"{bucket}:heldout"); return False err=validate_schema(payload, schema) if err is not None: self.invalid[f"{source}:{err[:60]}"]=self.invalid.get(f"{source}:{err[:60]}",0)+1 return False h=content_hash(user+"\x00"+a) if h in self.seen: self._bump(self.dropped,f"{bucket}:dup"); return False self.seen.add(h) self.fh.write(json.dumps({"bucket":bucket,"source":source,"messages":[ {"role":"system","content":system},{"role":"user","content":user}, {"role":"assistant","content":a}]}, ensure_ascii=False)+"\n") self.spent[bucket]=self.spent.get(bucket,0)+t self.per_src[(bucket,source)]=self.per_src.get((bucket,source),0)+t self.n+=1; return True def close(self): self.fh.close() def report(self, out=None): L=[] L.append(f"records={self.n} buckets={len(self.budget)}") L.append(f"{'bucket':<16}{'spent':>10}{'budget':>10}{'fill%':>8}") for b in sorted(self.budget): s,bd=self.spent.get(b,0),self.budget[b] L.append(f"{b:<16}{s:>10}{bd:>10}{100*s/max(bd,1):>7.1f}%") if self.invalid: L.append("\nSCHEMA-INVALID (dropped, would have failed the gate):") for k,v in sorted(self.invalid.items(),key=lambda x:-x[1])[:20]: L.append(f" {v:>6} {k}") else: L.append("\nSCHEMA-INVALID: none") txt="\n".join(L) if out: open(out,"w").write(txt) return txt