| """contract.py — redefines NOTHING. Puts the harness on sys.path, imports it, and |
| re-exports its objects BY REFERENCE. Handover §1: "the scorer IS the contract".""" |
| import os, sys, json, hashlib, re |
| _HARNESS_DIR = "/workspace/aditi/harness" |
| sys.path.insert(0, _HARNESS_DIR) |
| os.environ.setdefault("ADITI_DATA", "/workspace/aditi/evaldata") |
| import eval_harness as E |
|
|
| SYS_JSON = E.SYS_JSON |
| P_NER, P_SENT, P_CLAIM = E.P_NER, E.P_SENT, E.P_CLAIM |
| P_HALLUC, P_GROUND_GEN, P_MCQ = E.P_HALLUC, E.P_GROUND_GEN, E.P_MCQ |
| schema_prompt = E.schema_prompt |
| NER_SCHEMA, SENTIMENT_SCHEMA = E.NER_SCHEMA, E.SENTIMENT_SCHEMA |
| CLAIM_SCHEMA, HALLUC_SCHEMA = E.CLAIM_SCHEMA, E.HALLUC_SCHEMA |
| GROUNDED_ANSWER_SCHEMA, MCQ_SCHEMA = E.GROUNDED_ANSWER_SCHEMA, E.MCQ_SCHEMA |
| OSINT_RECORD_SCHEMA = E.OSINT_RECORD_SCHEMA |
| AVERITEC_MAP = E.AVERITEC_MAP |
| validate_schema, is_heldout = E.validate_schema, E.is_heldout |
| content_hash, normalise_ws, split_sentences = E.content_hash, E.normalise_ws, E.split_sentences |
|
|
| |
| ENTITY_TYPES = E.ENTITY_TYPES |
| NER_TYPES = NER_SCHEMA["properties"]["entities"]["items"]["properties"]["type"]["enum"] |
| VERDICTS = CLAIM_SCHEMA["properties"]["verdict"]["enum"] |
| SCRIPTS = OSINT_RECORD_SCHEMA["properties"]["script"]["enum"] |
| SENTIMENT_LABELS = E.SENTIMENT_LABELS |
| STANCE_LABELS = E.STANCE_LABELS |
| HOSTILITY_LABELS = E.HOSTILITY_LABELS |
| MISINFO_LABELS = E.MISINFO_LABELS |
|
|
| |
| _VERDICT = { |
| "supported":"supported","support":"supported","supports":"supported","true":"supported", |
| "refuted":"refuted","refutes":"refuted","false":"refuted","refute":"refuted", |
| "not enough evidence":"not_enough_evidence","nei":"not_enough_evidence", |
| "not enough info":"not_enough_evidence","notenoughinfo":"not_enough_evidence", |
| "conflicting evidence/cherrypicking":"conflicting_evidence","conflicting":"conflicting_evidence", |
| "conflicting evidence":"conflicting_evidence","mixture":"conflicting_evidence", |
| "half true":"conflicting_evidence","partly true/misleading":"conflicting_evidence", |
| "mostly true":"supported","mostly false":"refuted", |
| "complicated/hard to categorise":"not_enough_evidence","other":None, "unproven":"not_enough_evidence", |
| } |
| _ENTITY = {"PER":"PER","PERSON":"PER","B-PER":"PER","I-PER":"PER", |
| "ORG":"ORG","ORGANISATION":"ORG","ORGANIZATION":"ORG","B-ORG":"ORG","I-ORG":"ORG", |
| "LOC":"LOC","GPE":"LOC","B-LOC":"LOC","I-LOC":"LOC","LOCATION":"LOC", |
| |
| "MISC":None,"O":None,"PRODUCT":None,"EVENT":None,"DATE":None,"CARDINAL":None, |
| "WORK_OF_ART":None,"LAW":None,"NORP":None,"FAC":None,"TIME":None,"MONEY":None, |
| "PERCENT":None,"QUANTITY":None,"ORDINAL":None,"LANGUAGE":None} |
| _SENTIMENT = {"positive":"positive","negative":"negative","neutral":"neutral", |
| "1":"positive","-1":"negative","0":"neutral","pos":"positive","neg":"negative", |
| "Positive":"positive","Negative":"negative","Neutral":"neutral"} |
| _STANCE = {"supportive":"supportive","agree":"supportive","support":"supportive","for":"supportive", |
| "opposing":"opposing","disagree":"opposing","against":"opposing","refute":"opposing", |
| "neutral":"neutral","discuss":"neutral","unrelated":"neutral","observing":"neutral", |
| "mixed":"mixed","comment":"neutral"} |
| _HOSTILITY = {"none":"none","not-abusive":"none","non-hostile":"none","normal":"none","nh":"none", |
| "hate":"hate","hs":"hate","hateful":"hate","religious hate":"hate", |
| "defamation":"defamation","fake":"fake","threat":"threat", |
| "offensive":"offensive","abusive":"offensive","offn":"offensive","prfn":"offensive", |
| "profane/untargeted":"offensive","sexism":"offensive","aggressive":"offensive"} |
| _MISINFO = {"verified":"verified","true":"verified","real":"verified", |
| "misleading":"misleading","partly true":"misleading","half true":"misleading", |
| "false":"false","fake":"false","mostly false":"false", |
| "unverified":"unverified","unproven":"unverified","satire":"satire","humor":"satire"} |
| _LANG2SCRIPT = {"hi":"Deva","mr":"Deva","ne":"Deva","sa":"Deva","kok":"Deva","doi":"Deva","mai":"Deva", |
| "bn":"Beng","as":"Beng","gu":"Gujr","pa":"Guru","kn":"Knda","ml":"Mlym","or":"Orya", |
| "ta":"Taml","te":"Telu","ur":"Arab","sd":"Arab","ps":"Arab","ks":"Arab","en":"Latn", |
| "mni":"Mtei","sat":"Olck","bo":"Tibt","si":"Sinh"} |
|
|
| _TABLES = {"verdict":(_VERDICT,VERDICTS),"entity":(_ENTITY,NER_TYPES), |
| "sentiment":(_SENTIMENT,SENTIMENT_LABELS),"stance":(_STANCE,STANCE_LABELS), |
| "hostility":(_HOSTILITY,HOSTILITY_LABELS),"misinfo":(_MISINFO,MISINFO_LABELS)} |
| for _slot,(_tbl,_enum) in _TABLES.items(): |
| for _k,_v in _tbl.items(): |
| if _v is not None and _v not in _enum: |
| raise ImportError(f"contract.py: {_slot} maps {_k!r} -> {_v!r}, " |
| f"not in harness enum {list(_enum)}") |
| for _v in _LANG2SCRIPT.values(): |
| if _v not in SCRIPTS: |
| raise ImportError(f"contract.py: script {_v!r} not in harness script enum") |
|
|
| class UnmappedLabel(ValueError): pass |
| def canon(slot, raw, strict=True): |
| """exact -> case-insensitive -> already-canonical passthrough -> raise.""" |
| tbl,enum=_TABLES[slot] |
| if raw is None: return None |
| if raw in tbl: return tbl[raw] |
| s=str(raw).strip() |
| if s in tbl: return tbl[s] |
| lo=s.lower() |
| for k,v in tbl.items(): |
| if str(k).lower()==lo: return v |
| if s in enum: return s |
| if strict: raise UnmappedLabel(f"{slot}: {raw!r} unmapped") |
| return None |
|
|
| def bio_entities(words, tags): |
| """strip B-/I-/E-/S- and route every tag through canon(entity, strict=False).""" |
| out=[];cur=None;buf=[] |
| for w,t in zip(words,tags): |
| t=(t or "O").strip().replace("\r","") |
| pre,_,rest = t.partition("-") |
| base = rest if pre in ("B","I","E","S") and rest else t |
| typ = canon("entity", base, strict=False) |
| if typ and (pre in ("B","S") or cur!=typ): |
| if cur and buf: out.append({"text":" ".join(buf),"type":cur}) |
| cur,buf=typ,[w] |
| elif typ and cur==typ: buf.append(w) |
| else: |
| if cur and buf: out.append({"text":" ".join(buf),"type":cur}) |
| cur,buf=None,[] |
| if cur and buf: out.append({"text":" ".join(buf),"type":cur}) |
| return out |
|
|
| def script_for(lang, text=""): |
| s=_LANG2SCRIPT.get((lang or "").lower()) |
| if s: return s |
| import unicodedata as U |
| BLK={"DEVANAGARI":"Deva","BENGALI":"Beng","GUJARATI":"Gujr","GURMUKHI":"Guru", |
| "KANNADA":"Knda","MALAYALAM":"Mlym","ORIYA":"Orya","TAMIL":"Taml", |
| "TELUGU":"Telu","ARABIC":"Arab","LATIN":"Latn","SINHALA":"Sinh","TIBETAN":"Tibt"} |
| counts={} |
| for ch in (text or "")[:400]: |
| if ch.isspace() or not ch.isalpha(): continue |
| try: nm=U.name(ch).split()[0] |
| except ValueError: continue |
| k=BLK.get(nm) |
| if k: counts[k]=counts.get(k,0)+1 |
| if not counts: return "Other" |
| top=max(counts,key=counts.get); n=counts[top] |
| if top!="Latn" and counts.get("Latn",0) > n*0.4: return "Mixed" |
| return top |
|
|
| if __name__=="__main__": |
| assert canon("verdict","Conflicting Evidence/Cherrypicking")=="conflicting_evidence" |
| assert canon("entity","MISC") is None, "MISC has no slot in the NER gate enum" |
| assert canon("entity","PERSON")=="PER" |
| assert validate_schema({"verdict":"supported","cited_evidence":[0],"confidence":"medium"}, |
| CLAIM_SCHEMA) is not None, "v2's claim payload should be schema-invalid" |
| assert canon("hostility","abusive")=="offensive" and canon("hostility","not-abusive")=="none" |
| assert script_for("hi")=="Deva" and script_for("ur")=="Arab" |
| print("contract: all guardrail tests pass") |
| print(" verdicts ",VERDICTS); print(" ner types",NER_TYPES) |
| print(" misinfo ",MISINFO_LABELS); print(" hostility",HOSTILITY_LABELS) |
| print(" stance ",STANCE_LABELS) |
|
|
| |
| class Emitter: |
| """Nothing reaches the JSONL except through emit(). Refusal order: |
| budget -> share cap -> hard cap -> contamination -> schema -> dedup.""" |
| def __init__(self, path, budget, tokenizer_dir=None, share_cap=0.12, hard_cap=None): |
| self.path=path; self.budget=dict(budget); self.share_cap=share_cap |
| self.hard_cap=hard_cap or {} |
| self.spent={k:0 for k in budget}; self.per_src={} |
| self.seen=set(); self.invalid={}; self.dropped={} |
| self.fh=open(path,"w",encoding="utf-8"); self.n=0 |
| self._tok=None; self._tokdir=tokenizer_dir |
| def tok(self): |
| if self._tok is None: |
| from transformers import AutoTokenizer |
| self._tok=AutoTokenizer.from_pretrained(self._tokdir) |
| return self._tok |
| def ntok(self,s): |
| return len(self.tok()(s, add_special_tokens=False)["input_ids"]) |
| def _bump(self,d,k): d[k]=d.get(k,0)+1 |
| def emit(self, bucket, source, user, payload, schema, heldout_text=None, system=None): |
| system = system or SYS_JSON |
| if self.spent.get(bucket,0) >= self.budget.get(bucket,0): |
| self._bump(self.dropped,f"{bucket}:budget"); return False |
| a=json.dumps(payload, ensure_ascii=False, separators=(",",":")) |
| t=self.ntok(a) |
| frac=min(self.share_cap, self.hard_cap.get((bucket,source), 1.0)) |
| cap=frac*self.budget.get(bucket,0) |
| if self.per_src.get((bucket,source),0)+t > cap: |
| self._bump(self.dropped,f"{bucket}/{source}:sharecap"); return False |
| if is_heldout(normalise_ws(heldout_text if heldout_text is not None else user)): |
| self._bump(self.dropped,f"{bucket}:heldout"); return False |
| err=validate_schema(payload, schema) |
| if err is not None: |
| self.invalid[f"{source}:{err[:60]}"]=self.invalid.get(f"{source}:{err[:60]}",0)+1 |
| return False |
| h=content_hash(user+"\x00"+a) |
| if h in self.seen: self._bump(self.dropped,f"{bucket}:dup"); return False |
| self.seen.add(h) |
| self.fh.write(json.dumps({"bucket":bucket,"source":source,"messages":[ |
| {"role":"system","content":system},{"role":"user","content":user}, |
| {"role":"assistant","content":a}]}, ensure_ascii=False)+"\n") |
| self.spent[bucket]=self.spent.get(bucket,0)+t |
| self.per_src[(bucket,source)]=self.per_src.get((bucket,source),0)+t |
| self.n+=1; return True |
| def close(self): self.fh.close() |
| def report(self, out=None): |
| L=[] |
| L.append(f"records={self.n} buckets={len(self.budget)}") |
| L.append(f"{'bucket':<16}{'spent':>10}{'budget':>10}{'fill%':>8}") |
| for b in sorted(self.budget): |
| s,bd=self.spent.get(b,0),self.budget[b] |
| L.append(f"{b:<16}{s:>10}{bd:>10}{100*s/max(bd,1):>7.1f}%") |
| if self.invalid: |
| L.append("\nSCHEMA-INVALID (dropped, would have failed the gate):") |
| for k,v in sorted(self.invalid.items(),key=lambda x:-x[1])[:20]: L.append(f" {v:>6} {k}") |
| else: |
| L.append("\nSCHEMA-INVALID: none") |
| txt="\n".join(L) |
| if out: open(out,"w").write(txt) |
| return txt |
|
|
