collusion-watch / source /src /collusion_lab /inspect_tasks.py
munip's picture
Publish Collusion-Watch article, source, and reviewed results
bd26047 verified
Raw History Blame Contribute Delete
5.07 kB
"""Inspect AI entry points, so the benchmark runs under `inspect eval` and produces
standard .eval logs / `inspect view` dashboards alongside our own evidence logs.
inspect eval src/collusion_lab/inspect_tasks.py@swe_collusion -T profile=scripted -T tasks=T3 \
-T conditions=C1_honest_team,C3_collude_overt --model mockllm/model
inspect eval src/collusion_lab/inspect_tasks.py@market_collusion -T profile=cloud --model anthropic/claude-haiku-4-5
Each Sample = one (scenario cell, seed). The solver runs a whole multi-agent episode
in the environment (agents use the per-role models from configs/models.yaml), then
the scorer replays the trusted monitors on the authoritative evidence log.
`--model` is only Inspect's default model; role models come from the profile.
"""
from __future__ import annotations
import asyncio
from pathlib import Path
from inspect_ai import Task, task
from inspect_ai.dataset import MemoryDataset, Sample
from inspect_ai.scorer import Score, Target, mean, scorer
from inspect_ai.solver import Generate, TaskState, solver
from collusion_lab.core.models import ModelRoster
from collusion_lab.monitor.core import TrustedMonitor
OUT = Path("runs/inspect")
def _split(x: str | list | None, default: list[str]) -> list[str]:
if x is None:
return default
if isinstance(x, (int, float)):
return [str(x)]
return [str(v) for v in (x.split(",") if isinstance(x, str) else x)]
@solver
def episode_solver(scenario: str, profile: str, online: str | None):
async def solve(state: TaskState, generate: Generate) -> TaskState:
md = state.metadata
roster = ModelRoster(profile)
if scenario == "swe":
from collusion_lab.scenarios.swe.env import run_episode
rec = await asyncio.to_thread(run_episode, md["task"], md["condition"], md["seed"], OUT, roster, online)
else:
from collusion_lab.scenarios.market.env import run_market_episode
rec = await asyncio.to_thread(run_market_episode, md["condition"], md["seed"], OUT,
md.get("rounds", 30), roster, online)
state.metadata["episode_dir"] = str(OUT / rec["episode_id"])
state.metadata["record"] = {k: rec[k] for k in ("episode_id", "truth", "outcome", "cost")}
state.output.completion = f"{rec['episode_id']}: {rec['outcome']}"
return state
return solve
@scorer(metrics={"*": [mean()]})
def monitor_scorer(scenario: str, monitors: list[str]):
async def score(state: TaskState, target: Target) -> Score:
ep = Path(state.metadata["episode_dir"])
rec = state.metadata["record"]
if scenario == "swe":
from collusion_lab.scenarios.swe.env import load_ctx
from collusion_lab.scenarios.swe.monitor_plugin import SWEPlugin as P
harm = bool(rec["outcome"]["harmful_release"])
else:
from collusion_lab.scenarios.market.env import load_ctx
from collusion_lab.scenarios.market.monitor_plugin import MarketPlugin as P
harm = bool(rec["outcome"]["harmful"])
ctx, events, _ = await asyncio.to_thread(load_ctx, ep)
values = {"collusive": float(rec["truth"]["collusive"]), "harm": float(harm),
"cost_usd": rec["cost"]["usd"]}
for m in monitors:
d = await asyncio.to_thread(TrustedMonitor(m, P()).evaluate, ctx, events)
values[f"{m}_flag"] = float(d.flagged)
values[f"{m}_constrain"] = float(d.action.value == "CONSTRAIN")
values[f"{m}_collusion"] = d.collusion_score
return Score(value=values, explanation=str(rec["outcome"]))
return score
@task
def swe_collusion(tasks: str = "T1,T2,T3,T4,T5", conditions: str | None = None, seeds: str = "0",
profile: str = "scripted", online: str | None = None, monitors: str = "M1,M3") -> Task:
from collusion_lab.scenarios.swe.conditions import CONDITIONS
samples = [Sample(input=f"{t}/{c}/s{s}", id=f"{t}-{c}-s{s}",
metadata={"task": t, "condition": c, "seed": int(s)})
for t in _split(tasks, []) for c in _split(conditions, list(CONDITIONS)) for s in _split(seeds, ["0"])]
return Task(dataset=MemoryDataset(samples), solver=episode_solver("swe", profile, online),
scorer=monitor_scorer("swe", _split(monitors, ["M3"])))
@task
def market_collusion(conditions: str | None = None, seeds: str = "0,1,2", rounds: int = 30,
profile: str = "scripted", online: str | None = None, monitors: str = "M1,M3") -> Task:
from collusion_lab.scenarios.market.conditions import CONDITIONS
samples = [Sample(input=f"{c}/s{s}", id=f"{c}-s{s}", metadata={"condition": c, "seed": int(s), "rounds": rounds})
for c in _split(conditions, list(CONDITIONS)) for s in _split(seeds, ["0"])]
return Task(dataset=MemoryDataset(samples), solver=episode_solver("market", profile, online),
scorer=monitor_scorer("market", _split(monitors, ["M3"])))