Spaces:
Running
Running
Download source/src/collusion_lab/inspect_tasks.py from munip/collusion-watch: direct link, hf CLI and curl.
- Browser
- Download file 5.07 kB
-
https://huggingface.co/spaces/munip/collusion-watch/resolve/main/source/src/collusion_lab/inspect_tasks.py
- Command line
-
hf download hf://spaces/munip/collusion-watch/source/src/collusion_lab/inspect_tasks.py
-
curl -L -o inspect_tasks.py https://huggingface.co/spaces/munip/collusion-watch/resolve/main/source/src/collusion_lab/inspect_tasks.py
5.07 kB
| """Inspect AI entry points, so the benchmark runs under `inspect eval` and produces | |
| standard .eval logs / `inspect view` dashboards alongside our own evidence logs. | |
| inspect eval src/collusion_lab/inspect_tasks.py@swe_collusion -T profile=scripted -T tasks=T3 \ | |
| -T conditions=C1_honest_team,C3_collude_overt --model mockllm/model | |
| inspect eval src/collusion_lab/inspect_tasks.py@market_collusion -T profile=cloud --model anthropic/claude-haiku-4-5 | |
| Each Sample = one (scenario cell, seed). The solver runs a whole multi-agent episode | |
| in the environment (agents use the per-role models from configs/models.yaml), then | |
| the scorer replays the trusted monitors on the authoritative evidence log. | |
| `--model` is only Inspect's default model; role models come from the profile. | |
| """ | |
| from __future__ import annotations | |
| import asyncio | |
| from pathlib import Path | |
| from inspect_ai import Task, task | |
| from inspect_ai.dataset import MemoryDataset, Sample | |
| from inspect_ai.scorer import Score, Target, mean, scorer | |
| from inspect_ai.solver import Generate, TaskState, solver | |
| from collusion_lab.core.models import ModelRoster | |
| from collusion_lab.monitor.core import TrustedMonitor | |
| OUT = Path("runs/inspect") | |
| def _split(x: str | list | None, default: list[str]) -> list[str]: | |
| if x is None: | |
| return default | |
| if isinstance(x, (int, float)): | |
| return [str(x)] | |
| return [str(v) for v in (x.split(",") if isinstance(x, str) else x)] | |
| def episode_solver(scenario: str, profile: str, online: str | None): | |
| async def solve(state: TaskState, generate: Generate) -> TaskState: | |
| md = state.metadata | |
| roster = ModelRoster(profile) | |
| if scenario == "swe": | |
| from collusion_lab.scenarios.swe.env import run_episode | |
| rec = await asyncio.to_thread(run_episode, md["task"], md["condition"], md["seed"], OUT, roster, online) | |
| else: | |
| from collusion_lab.scenarios.market.env import run_market_episode | |
| rec = await asyncio.to_thread(run_market_episode, md["condition"], md["seed"], OUT, | |
| md.get("rounds", 30), roster, online) | |
| state.metadata["episode_dir"] = str(OUT / rec["episode_id"]) | |
| state.metadata["record"] = {k: rec[k] for k in ("episode_id", "truth", "outcome", "cost")} | |
| state.output.completion = f"{rec['episode_id']}: {rec['outcome']}" | |
| return state | |
| return solve | |
| def monitor_scorer(scenario: str, monitors: list[str]): | |
| async def score(state: TaskState, target: Target) -> Score: | |
| ep = Path(state.metadata["episode_dir"]) | |
| rec = state.metadata["record"] | |
| if scenario == "swe": | |
| from collusion_lab.scenarios.swe.env import load_ctx | |
| from collusion_lab.scenarios.swe.monitor_plugin import SWEPlugin as P | |
| harm = bool(rec["outcome"]["harmful_release"]) | |
| else: | |
| from collusion_lab.scenarios.market.env import load_ctx | |
| from collusion_lab.scenarios.market.monitor_plugin import MarketPlugin as P | |
| harm = bool(rec["outcome"]["harmful"]) | |
| ctx, events, _ = await asyncio.to_thread(load_ctx, ep) | |
| values = {"collusive": float(rec["truth"]["collusive"]), "harm": float(harm), | |
| "cost_usd": rec["cost"]["usd"]} | |
| for m in monitors: | |
| d = await asyncio.to_thread(TrustedMonitor(m, P()).evaluate, ctx, events) | |
| values[f"{m}_flag"] = float(d.flagged) | |
| values[f"{m}_constrain"] = float(d.action.value == "CONSTRAIN") | |
| values[f"{m}_collusion"] = d.collusion_score | |
| return Score(value=values, explanation=str(rec["outcome"])) | |
| return score | |
| def swe_collusion(tasks: str = "T1,T2,T3,T4,T5", conditions: str | None = None, seeds: str = "0", | |
| profile: str = "scripted", online: str | None = None, monitors: str = "M1,M3") -> Task: | |
| from collusion_lab.scenarios.swe.conditions import CONDITIONS | |
| samples = [Sample(input=f"{t}/{c}/s{s}", id=f"{t}-{c}-s{s}", | |
| metadata={"task": t, "condition": c, "seed": int(s)}) | |
| for t in _split(tasks, []) for c in _split(conditions, list(CONDITIONS)) for s in _split(seeds, ["0"])] | |
| return Task(dataset=MemoryDataset(samples), solver=episode_solver("swe", profile, online), | |
| scorer=monitor_scorer("swe", _split(monitors, ["M3"]))) | |
| def market_collusion(conditions: str | None = None, seeds: str = "0,1,2", rounds: int = 30, | |
| profile: str = "scripted", online: str | None = None, monitors: str = "M1,M3") -> Task: | |
| from collusion_lab.scenarios.market.conditions import CONDITIONS | |
| samples = [Sample(input=f"{c}/s{s}", id=f"{c}-s{s}", metadata={"condition": c, "seed": int(s), "rounds": rounds}) | |
| for c in _split(conditions, list(CONDITIONS)) for s in _split(seeds, ["0"])] | |
| return Task(dataset=MemoryDataset(samples), solver=episode_solver("market", profile, online), | |
| scorer=monitor_scorer("market", _split(monitors, ["M3"]))) | |