| |
| """The KEYSTONE measurement (2026-10-25, all four voices + Tetsu agreed): the LIVE gate's real harm-catch rate on |
| an EXTERNAL, published benchmark -- NOT a simulation we wrote. Tien: "a simulated 28/28 is a test of the |
| simulation, not the gate." |
| |
| Set: HateCheck (Rottger et al. 2021), a published hate-speech functional test. Hateful cases are genuinely |
| harmful content the gate SHOULD convict (harm-catch / recall); non-hateful contrasts it SHOULD clear |
| (false-conviction / the over-conviction surface). We measure BOTH -- recall without the false-positive rate is |
| the trap, because a gate that convicts everything has 100% recall and no value. |
| |
| Builds the SAME gate the node uses (build_semantic_quorum + ReasoningSentinel + DIVINE_PRINCIPLES) and judges |
| each case as a shared-text transaction, exactly as the agent door does. Read-only; nothing is sent or acted on; |
| the harmful items get CONVICTED, which is the safe direction. Reports aggregate rates only. |
| |
| Usage: python tools/gate_harmcatch.py [N_PER_CLASS] (needs the node's venv for the judge providers).""" |
| import csv, os, random, sys |
| HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| sys.path.insert(0, HERE) |
| os.environ.setdefault("COVENANT_FORCE_NO_SANDBOX", "1") |
| import covenant_unified_v8 as cov |
|
|
| N = int(sys.argv[1]) if len(sys.argv) > 1 else 50 |
| HATECHECK = os.path.join(HERE, "eval", "outside", "raw", "hatecheck", "test.csv") |
|
|
|
|
| def verdict(sentinel, text): |
| tx = cov.Transaction(sender_pubkey="eval", receiver="collective", |
| data={"origin": "human", "kind": "shared", "message": str(text)[:4000]}, |
| amount=0.0, benefit_score=0.5) |
| ok, message, _benefit, result = sentinel.evaluate_transaction(tx) |
| alleges_nothing = bool(result is not None and not ok and ( |
| getattr(result, "not_understood", False) or getattr(result, "uncertain", False))) |
| if ok: |
| return "cleared" |
| return "alleges_nothing" if alleges_nothing else "convicted" |
|
|
|
|
| def run(): |
| rng = random.Random(7) |
| rows = list(csv.DictReader(open(HATECHECK, encoding="utf-8", errors="replace"))) |
| hateful = [r["test_case"] for r in rows if r.get("label_gold") == "hateful"] |
| benign = [r["test_case"] for r in rows if r.get("label_gold") == "non-hateful"] |
| rng.shuffle(hateful); rng.shuffle(benign) |
| hateful, benign = hateful[:N], benign[:N] |
| print("building the live gate (build_semantic_quorum + ReasoningSentinel)...", flush=True) |
| judge = cov.build_semantic_quorum() |
| sentinel = cov.ReasoningSentinel(judge, cov.DIVINE_PRINCIPLES) |
| from collections import Counter |
| res = {"hateful": Counter(), "non-hateful": Counter()} |
| for label, items in (("hateful", hateful), ("non-hateful", benign)): |
| for i, t in enumerate(items): |
| res[label][verdict(sentinel, t)] += 1 |
| if (i + 1) % 10 == 0: |
| print(" %s %d/%d" % (label, i + 1, len(items)), flush=True) |
| nh, nb = sum(res["hateful"].values()), sum(res["non-hateful"].values()) |
| caught = res["hateful"]["convicted"] + res["hateful"]["alleges_nothing"] |
| fconv = res["non-hateful"]["convicted"] + res["non-hateful"]["alleges_nothing"] |
| print("\n=== LIVE GATE on HateCheck (external, published) -- real numbers ===") |
| print("HARM-CATCH (hateful not cleared): %d/%d = %.1f%% breakdown %s" % (caught, nh, 100.0 * caught / max(1, nh), dict(res["hateful"]))) |
| print("FALSE-CONVICTION (non-hateful not cleared): %d/%d = %.1f%% breakdown %s" % (fconv, nb, 100.0 * fconv / max(1, nb), dict(res["non-hateful"]))) |
| print("\nReading: high harm-catch is good; high false-conviction is the over-conviction (the trap). The pair is") |
| print("the keystone number -- it is NOT the 28/28 sim; it is the live gate on someone else's labelled set.") |
|
|
|
|
| if __name__ == "__main__": |
| run() |
|
|
