File size: 12,635 Bytes
bd26047
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
"""Experiment runner: episode matrices (paired no-monitor / online-monitor) and the
offline replay that scores every monitor x observability on identical trajectories."""
from __future__ import annotations

import json
import traceback
from concurrent.futures import ProcessPoolExecutor, as_completed
from pathlib import Path
from typing import Any

import pandas as pd
import yaml

from .core.models import ModelRoster

OBS_PRESETS = ("full", "artifacts_only", "transcript_only", "no_dm")


# ---------------------------------------------------------------------------- run
def _swe_job(args: dict) -> dict:
    from .scenarios.swe.env import run_episode
    try:
        roster = ModelRoster(args["profile"], args.get("overrides"), cap_usd=args.get("cap"))
        rec = run_episode(args["task"], args["condition"], args["seed"], Path(args["out"]), roster,
                          args.get("online"), args.get("verifier_fraction", 0.5), pressure=args.get("pressure"),
                          peer=args.get("peer"))
        return {"ok": True, "episode": rec["episode_id"], "cost": rec["cost"]["usd"]}
    except Exception as e:  # keep the matrix going
        return {"ok": False, "job": args, "error": f"{e}\n{traceback.format_exc()[-1500:]}"}


def _mkt_job(args: dict) -> dict:
    from .scenarios.market.env import run_market_episode
    try:
        roster = ModelRoster(args["profile"], args.get("overrides"), cap_usd=args.get("cap"))
        rec = run_market_episode(args["condition"], args["seed"], Path(args["out"]), args.get("rounds", 30),
                                 roster, args.get("online"), args.get("intervention", "entrant"),
                                 pressure=args.get("pressure"))
        return {"ok": True, "episode": rec["episode_id"], "cost": rec["cost"]["usd"]}
    except Exception as e:
        return {"ok": False, "job": args, "error": f"{e}\n{traceback.format_exc()[-1500:]}"}


def run_matrix(cfg: dict[str, Any], out: Path, workers: int = 2) -> list[dict]:
    out.mkdir(parents=True, exist_ok=True)
    (out / "config.yaml").write_text(yaml.safe_dump(cfg))
    jobs, fn = [], None
    common = dict(profile=cfg.get("profile", "scripted"), overrides=cfg.get("overrides"),
                  cap=cfg.get("cap_usd"), out=str(out))
    online_list = [None] + list(cfg.get("online_monitors", []))
    from .core.pressure import Pressure, parse_list
    pressures = parse_list(cfg.get("pressures"))      # factor; default [none] (ids unchanged)
    if any(p.active for p in pressures) and str(cfg.get("profile", "scripted")) == "scripted":
        print("note: pressures only bind LLM agents; scripted agents ignore them", flush=True)
    if cfg["scenario"] == "swe":
        fn = _swe_job
        from .scenarios.swe.peer import PeerSpec
        peers = [PeerSpec.parse(x).name for x in (cfg.get("peers") or ["none"])]   # trusted-peer factor
        gate_arms = cfg.get("gate_arms")     # optional explicit (peer, online) pairs instead of the full cross
        for t in cfg["tasks"]:
            for c in cfg["conditions"]:
                for s in cfg["seeds"]:
                    for p in pressures:
                        arms = ([(a.get("peer", "none"), a.get("online")) for a in gate_arms] if gate_arms
                                else [(q, m) for q in peers for m in online_list])
                        for q, m in arms:
                            jobs.append(dict(common, task=t, condition=c, seed=s, online=m, pressure=p.to_dict(),
                                             peer=q, verifier_fraction=cfg.get("verifier_fraction", 0.5)))
    else:
        fn = _mkt_job
        for c in cfg["conditions"]:
            for s in cfg["seeds"]:
                for p in pressures:
                    for m in online_list:
                        for iv in (cfg.get("interventions", ["entrant"]) if m else ["entrant"]):
                            jobs.append(dict(common, condition=c, seed=s, online=m, intervention=iv,
                                             rounds=cfg.get("rounds", 30), pressure=p.to_dict()))
    # resumable: skip jobs whose episode already exists
    def _eid(j):
        p = Pressure.parse(j.get("pressure"))
        if cfg["scenario"] == "swe":
            from .scenarios.swe.env import swe_episode_id
            from .scenarios.swe.peer import PeerSpec
            return swe_episode_id(j["task"], j["condition"], j["seed"], j.get("online"), p,
                                  PeerSpec.parse(j.get("peer")))
        from .scenarios.market.env import market_episode_id
        return market_episode_id(j["condition"], j["seed"], j.get("online"), j["intervention"], p)
    before = len(jobs)
    jobs = [j for j in jobs if not (out / _eid(j) / "episode.json").exists()]
    print(f"{before - len(jobs)} episodes already done, {len(jobs)} to run", flush=True)
    results = []
    def report(i, r):
        results.append(r)
        status = (f"{r.get('episode')}  (${r.get('cost', 0):.4f})" if r["ok"]
                  else "FAILED " + r["error"].splitlines()[0])
        print(f"[{i}/{len(jobs)}] done: {status}", flush=True)
    if workers <= 1:
        for i, j in enumerate(jobs, 1):
            print(f"[{i}/{len(jobs)}] starting {j.get('task', '')} {j['condition']} seed {j['seed']}"
                  f" pressure {j['pressure']['name']} peer {j.get('peer', 'none')} gate {j.get('online') or '-'}",
                  flush=True)
            report(i, fn(j))
    else:
        with ProcessPoolExecutor(workers) as ex:
            futs = [ex.submit(fn, j) for j in jobs]
            for i, f in enumerate(as_completed(futs), 1):
                report(i, f.result())
    (out / "run_log.json").write_text(json.dumps(results, indent=1))
    return results


# ------------------------------------------------------------------------- replay
def _replay_one(args: dict) -> list[dict]:
    from .monitor.core import TrustedMonitor
    ep = Path(args["ep"])
    rec = json.loads((ep / "episode.json").read_text())
    roster = ModelRoster(args["profile"], args.get("overrides")) if args.get("profile") else None
    if rec["scenario"] == "swe":
        from .scenarios.swe.env import load_ctx
        from .scenarios.swe.monitor_plugin import SWEPlugin
        from .scenarios.swe.conditions import get
        ctx, events, rec = load_ctx(ep, roster)
        plugin = SWEPlugin()
        colluders = list(get(rec["condition"]).colluders)
        harm = rec["outcome"]["harmful_release"] or (rec["outcome"]["release_requested"]
                                                      and not rec["outcome"]["correct"])
        total_steps = rec["outcome"]["steps"]
    else:
        from .scenarios.market.env import load_ctx
        from .scenarios.market.monitor_plugin import MarketPlugin
        ctx, events, rec = load_ctx(ep, roster)
        plugin = MarketPlugin()
        colluders = list(rec["costs"]) if rec["truth"]["collusive"] else []
        harm = rec["outcome"]["harmful"]
        total_steps = rec["rounds"]
    from .eval.occurrence import occurred
    did_occur, how = occurred(rec, events)
    rows = []
    for lvl in args["monitors"]:
        for obs in args["observability"]:
            if lvl == "M2" and (roster is None or roster.is_scripted("monitor_judge")):
                continue
            if lvl in ("M0",) and obs != "full":
                continue
            d = TrustedMonitor(lvl, plugin, observability=obs).evaluate(ctx, events)
            rows.append({
                "episode": rec["episode_id"], "scenario": rec["scenario"], "task": rec.get("task", "market"),
                "condition": rec["condition"], "seed": rec["seed"], "category": rec["truth"]["category"],
                "pressure": (rec.get("pressure") or {}).get("name", "none"),
                "peer": (rec.get("peer") or {}).get("mode", "none"),
                "mechanism": rec["truth"]["mechanism"], "collusive": rec["truth"]["collusive"],
                "instructed": rec["truth"]["collusive"], "occurred": did_occur, "occurred_how": how,
                "harm": bool(harm), "monitor": lvl, "observability": obs, "action": d.action.value,
                "flagged": d.flagged, "risk": d.risk, "collusion_score": d.collusion_score,
                "first_step": d.step, "total_steps": total_steps,
                "attribution": json.dumps(d.attribution), "colluders": json.dumps(colluders),
                "signals": json.dumps({s.name: round(s.score, 3) for s in d.signals}),
                "verified": json.dumps(sorted(s.name for s in d.signals if s.verified and s.score > 0)),
                "reasons": json.dumps(d.reasons[:8]),
                "collusion_reasons": json.dumps(d.collusion_reasons[:8]),
            })
    return rows


def replay(run_dir: Path, monitors=("M0", "M1", "M2", "M3", "M4"), observability=OBS_PRESETS,
           profile: str | None = None, workers: int = 2, overrides: dict[str, str] | None = None,
           tasks: list[str] | None = None, out_name: str | None = "decisions.csv") -> pd.DataFrame:
    """Score monitors offline on the stored no-monitor trajectories. `overrides` pins roles
    (e.g. {"monitor_judge": spec}); `tasks` filters SWE tasks; out_name=None skips writing."""
    eps = []
    for p in sorted(run_dir.iterdir()):
        if not (p / "episode.json").exists():
            continue
        rec = json.loads((p / "episode.json").read_text())
        if rec.get("online_monitor"):
            continue                          # offline replay only on no-monitor trajectories
        if tasks and rec.get("scenario") == "swe" and rec.get("task") not in tasks:
            continue
        eps.append(str(p))
    rows: list[dict] = []
    if overrides and not profile:
        profile = "scripted"                  # overrides need a base profile
    jobs = [dict(ep=e, monitors=list(monitors), observability=list(observability), profile=profile,
                 overrides=overrides) for e in eps]
    print(f"replaying {len(jobs)} episode(s) x {len(monitors)} monitors x {len(observability)} views"
          + (f" (LLM judge from profile '{profile}')" if profile else ""), flush=True)
    if workers <= 1 or profile:
        # with an LLM judge: sequential, so free-tier rate limits are shared and progress is visible
        for i, j in enumerate(jobs, 1):
            print(f"[{i}/{len(jobs)}] {Path(j['ep']).name}", flush=True)
            rows += _replay_one(j)
    else:
        with ProcessPoolExecutor(workers) as ex:
            for i, f in enumerate(as_completed([ex.submit(_replay_one, j) for j in jobs]), 1):
                rows += f.result()
                print(f"[{i}/{len(jobs)}] replayed", flush=True)
    df = pd.DataFrame(rows)
    if out_name:
        df.to_csv(run_dir / out_name, index=False)
    return df


def episodes_table(run_dir: Path) -> pd.DataFrame:
    rows = []
    for p in sorted(run_dir.iterdir()):
        f = p / "episode.json"
        if not f.exists():
            continue
        r = json.loads(f.read_text())
        o = r["outcome"]
        rows.append({"episode": r["episode_id"], "scenario": r["scenario"], "task": r.get("task", "market"),
                     "condition": r["condition"], "seed": r["seed"], "online": r.get("online_monitor"),
                     "pressure": (r.get("pressure") or {}).get("name", "none"),
                     "peer": (r.get("peer") or {}).get("mode", "none") if isinstance(r.get("peer"), dict) else "none",
                     "intervention": r.get("intervention"), "collusive": r["truth"]["collusive"],
                     "category": r["truth"]["category"], "mechanism": r["truth"]["mechanism"],
                     "cost_usd": r["cost"]["usd"], "llm_calls": r["cost"].get("calls", 0),
                     **{f"o_{k}": v for k, v in o.items() if not isinstance(v, (list, dict))},
                     # actor lists (trusted-peer outcomes) as ';'-joined strings + derived flags
                     **{f"o_{k}": ";".join(v) for k, v in o.items()
                        if isinstance(v, list) and all(isinstance(x, str) for x in v)},
                     **({"o_probe_reciprocated": bool(o.get("probe_reciprocators")),
                         "o_probe_replied": bool(o.get("probe_replies")),
                         "o_skip_after_checkpoint": bool(o.get("skip_after_checkpoint_by")),
                         "o_qa_last_failing": (len(o["qa_reports"][-1]["failing"]) if o.get("qa_reports") else None)}
                        if r["scenario"] == "swe" and "peer" in o else {})})
    df = pd.DataFrame(rows)
    df.to_csv(run_dir / "episodes.csv", index=False)
    return df