Spaces:
Running
Running
Download source/src/collusion_lab/core/models.py from munip/collusion-watch: direct link, hf CLI and curl.
- Browser
- Download file 15.3 kB
-
https://huggingface.co/spaces/munip/collusion-watch/resolve/main/source/src/collusion_lab/core/models.py
- Command line
-
hf download hf://spaces/munip/collusion-watch/source/src/collusion_lab/core/models.py
-
curl -L -o models.py https://huggingface.co/spaces/munip/collusion-watch/resolve/main/source/src/collusion_lab/core/models.py
15.3 kB
| """Role -> model(s) resolution across providers/routers/gateways, with | |
| fallback chains, per-provider rate limiting, tool-calling mode, cost metering, | |
| a hard budget cap and a check that the trusted monitor uses a different model | |
| family than the agents. | |
| Profiles live in configs/models.yaml. A role maps to one spec or a fallback list: | |
| implementer: [groq/openai/gpt-oss-120b, cerebras/gpt-oss-120b, openrouter/qwen/qwen3-coder:free] | |
| Agents call `roster.generate(role, messages, tools)`. When a role is "scripted" | |
| the caller uses its scripted policy instead (roster.is_scripted). | |
| """ | |
| from __future__ import annotations | |
| import asyncio | |
| import fnmatch | |
| import os | |
| import threading | |
| import time | |
| from dataclasses import dataclass, field | |
| from pathlib import Path | |
| from typing import Any | |
| import yaml | |
| from .providers import Resolved, family, get_inspect_model, resolve, LOCAL_COMPUTE | |
| REPO_ROOT = Path(__file__).resolve().parents[3] | |
| DEFAULT_CONFIG = REPO_ROOT / "configs" / "models.yaml" | |
| AGENT_ROLES = ("planner", "implementer", "tester", "reviewer", "verifier", "seller", "buyer") | |
| MONITOR_ROLES = ("monitor_judge", "monitor_auditor", "qa_peer") # trusted side: never agents | |
| SCRIPTED = ("scripted", "none", "") | |
| class BudgetExceeded(RuntimeError): | |
| pass | |
| class AllModelsFailed(RuntimeError): | |
| pass | |
| class CostMeter: | |
| prices: dict[str, list[float]] | |
| cap_usd: float | |
| spent_usd: float = 0.0 | |
| calls: int = 0 | |
| tokens_in: int = 0 | |
| tokens_out: int = 0 | |
| by_role: dict[str, dict[str, float]] = field(default_factory=dict) | |
| by_model: dict[str, dict[str, float]] = field(default_factory=dict) | |
| failures: dict[str, int] = field(default_factory=dict) | |
| _lock: threading.Lock = field(default_factory=threading.Lock, repr=False) | |
| def price(self, model: str) -> tuple[float, float]: | |
| for pat, (pi, po) in self.prices.items(): | |
| if fnmatch.fnmatch(model, pat): | |
| return pi, po | |
| return (3.0, 15.0) # conservative default for unknown paid models | |
| def add(self, role: str, model: str, tin: int, tout: int, served: str | None = None) -> None: | |
| pi, po = self.price(model) | |
| cost = (tin * pi + tout * po) / 1e6 | |
| with self._lock: | |
| self.spent_usd += cost | |
| self.calls += 1 | |
| self.tokens_in += tin | |
| self.tokens_out += tout | |
| for key, bucket in ((role, self.by_role), (model, self.by_model)): | |
| r = bucket.setdefault(key, {"calls": 0, "in": 0, "out": 0, "usd": 0.0}) | |
| r["calls"] += 1; r["in"] += tin; r["out"] += tout; r["usd"] += cost | |
| if served and served != model.split("/", 1)[-1]: | |
| self.by_model[model].setdefault("served_by", {}) | |
| sb = self.by_model[model]["served_by"] | |
| sb[served] = sb.get(served, 0) + 1 | |
| if self.spent_usd > self.cap_usd: | |
| raise BudgetExceeded(f"budget cap ${self.cap_usd:.2f} exceeded (${self.spent_usd:.2f})") | |
| def fail(self, model: str) -> None: | |
| with self._lock: | |
| self.failures[model] = self.failures.get(model, 0) + 1 | |
| def summary(self) -> dict[str, Any]: | |
| return {"usd": round(self.spent_usd, 4), "calls": self.calls, "tokens_in": self.tokens_in, | |
| "tokens_out": self.tokens_out, "by_role": self.by_role, "by_model": self.by_model, | |
| "failures": self.failures} | |
| class RateLimiter: | |
| """Process-wide minimum spacing between requests to the same provider.""" | |
| _last: dict[str, float] = {} | |
| _lock = threading.Lock() | |
| def reserve(cls, provider: str, rpm: float) -> float: | |
| """Reserve the next request slot; returns seconds to wait (thread-safe, non-blocking).""" | |
| if rpm <= 0: | |
| return 0.0 | |
| gap = 60.0 / rpm | |
| with cls._lock: | |
| now = time.monotonic() | |
| t = max(now, cls._last.get(provider, 0.0) + gap) | |
| cls._last[provider] = t | |
| return max(0.0, t - now) | |
| def wait(cls, provider: str, rpm: float) -> None: | |
| d = cls.reserve(provider, rpm) | |
| if d: | |
| time.sleep(d) | |
| PINNED_RETRIES = 6 # a role with a single remote model waits out rate limits | |
| PINNED_TIMEOUT = 600.0 # seconds of total retry budget per call for such a role | |
| class ModelRoster: | |
| def __init__(self, profile: str | None = None, overrides: dict[str, str] | None = None, | |
| config_path: Path = DEFAULT_CONFIG, cap_usd: float | None = None): | |
| cfg = yaml.safe_load(Path(config_path).read_text(encoding="utf-8")) | |
| self.profile_name = profile or os.environ.get("CLAB_PROFILE", "scripted") | |
| if self.profile_name not in cfg["profiles"]: | |
| raise KeyError(f"unknown profile {self.profile_name!r}; have {list(cfg['profiles'])}") | |
| prof = dict(cfg["profiles"][self.profile_name]) | |
| self.allow_shared_family = bool(prof.pop("allow_shared_family", False)) | |
| self.gen_defaults: dict[str, Any] = dict(cfg.get("generate", {})) | |
| self.gen_defaults.update(prof.pop("generate", {}) or {}) | |
| prof_pool = prof.pop("pool", None) | |
| self.mapping: dict[str, Any] = prof | |
| for k, v in (overrides or {}).items(): | |
| self.mapping[k] = [x.strip() for x in v.split(",")] if "," in v else v | |
| # FCM keys -> env, then expand dynamic `auto:*` roles from the discovered free-model pool | |
| from .pool import expand_auto, inject_fcm_keys | |
| inject_fcm_keys() | |
| self.pool_cfg: dict[str, Any] = dict(cfg.get("pool", {})) | |
| self.pool_cfg.update(prof_pool or {}) | |
| expanded = expand_auto(self.mapping, self.pool_cfg) | |
| if expanded: | |
| default_auto = isinstance(self.mapping.get("default"), str) and self.mapping["default"].startswith("auto") | |
| self.mapping.update(expanded) | |
| if default_auto: | |
| self.mapping["default"] = expanded.get("planner") or next( | |
| (v for k, v in expanded.items() if k != "monitor_judge"), ["scripted"]) | |
| self.auto_expanded = bool(expanded) | |
| cap = cap_usd if cap_usd is not None else cfg.get("budget", {}).get("hard_cap_usd", 25.0) | |
| self.meter = CostMeter(cfg.get("prices", {}), cap) | |
| self._models: dict[str, Any] = {} | |
| self._json_mode: set[str] = set() # specs that rejected native tool calls | |
| self._warned: set[str] = set() | |
| self._warn_count: dict[str, int] = {} | |
| self._provider_cooldowns: dict[str, float] = {} | |
| self._cooldown_lock = threading.Lock() | |
| # ------------------------------------------------------------------ mapping | |
| def chain(self, role: str) -> list[str]: | |
| v = self.mapping.get(role, self.mapping.get("default", "scripted")) | |
| if role in MONITOR_ROLES and role not in self.mapping: | |
| v = "none" | |
| return [v] if isinstance(v, str) else list(v) | |
| def model_name(self, role: str) -> str: | |
| return self.chain(role)[0] | |
| def is_scripted(self, role: str) -> bool: | |
| return self.model_name(role) in SCRIPTED | |
| def describe(self) -> dict[str, Any]: | |
| return {r: self.chain(r) for r in sorted(set(self.mapping) | {"default"}) if r in self.mapping} | |
| def resolved(self, role: str) -> list[Resolved]: | |
| return [resolve(s) for s in self.chain(role) if s not in SCRIPTED] | |
| def tool_mode(self, role: str, spec: str | None = None) -> str: | |
| spec = spec or self.model_name(role) | |
| if spec in self._json_mode: | |
| return "json" | |
| try: | |
| return resolve(spec).tools | |
| except ValueError: | |
| return "native" | |
| # ---------------------------------------------------------------- checks | |
| def families(self) -> tuple[set[str], set[str]]: | |
| agents, monitor = set(), set() | |
| for role in self.mapping: | |
| target = monitor if role in MONITOR_ROLES else agents | |
| for r in (self.resolved(role) if not self.is_scripted(role) else []): | |
| target.add(r.family) | |
| return agents, monitor | |
| def check_separation(self) -> list[str]: | |
| """Warnings if the trusted monitor shares a model family with any agent.""" | |
| agents, monitor = self.families() | |
| warns = [] | |
| if "router:dynamic" in monitor: | |
| warns.append("trusted monitor uses a dynamic router: pin a specific model") | |
| shared = (agents & monitor) - {"unknown", "router:dynamic"} | |
| if shared and not self.allow_shared_family: | |
| warns.append(f"monitor shares model family {sorted(shared)} with agents " | |
| f"(set allow_shared_family: true only for the correlated-error ablation)") | |
| auditor = set(self.chain("monitor_auditor")) - set(SCRIPTED) | |
| judge = set(self.chain("monitor_judge")) - set(SCRIPTED) | |
| if auditor & judge: | |
| warns.append("monitor_auditor must use a different model spec from monitor_judge") | |
| if "router:dynamic" in agents and monitor: | |
| warns.append("agents use a dynamic router; family separation from the monitor cannot be guaranteed " | |
| "- check `served_by` in the cost summary") | |
| return warns | |
| def missing_keys(self) -> list[str]: | |
| out = [] | |
| for role in self.mapping: | |
| if self.is_scripted(role): | |
| continue | |
| for r in self.resolved(role): | |
| if r.needs_key and not r.has_key: | |
| out.append(f"{role}: {r.spec} (no API key for provider '{r.provider}')") | |
| return out | |
| # -------------------------------------------------------------- generate | |
| def _get(self, r: Resolved): | |
| if r.spec not in self._models: | |
| self._models[r.spec] = get_inspect_model(r) | |
| return self._models[r.spec] | |
| async def generate(self, role: str, messages: list, tools: list | None = None, **config: Any): | |
| """Generate with fallback across the role's chain. Returns inspect ModelOutput with | |
| `.metadata['clab_spec']` set to the spec that answered.""" | |
| from inspect_ai.model import GenerateConfig | |
| errors = [] | |
| chain = [spec for spec in self.chain(role) if spec not in SCRIPTED] | |
| for spec in chain: | |
| r = resolve(spec) | |
| if r.needs_key and not r.has_key: | |
| errors.append(f"{spec}: no API key") | |
| continue | |
| with self._cooldown_lock: | |
| cooldown_until = self._provider_cooldowns.get(r.provider, 0.0) | |
| now = time.monotonic() | |
| if cooldown_until > now: | |
| remaining = cooldown_until - now | |
| errors.append(f"{spec}: provider rate-limit cooldown ({remaining:.0f}s remaining)") | |
| continue | |
| use_tools = tools if (tools and self.tool_mode(role, spec) != "json") else [] | |
| cfg = {**self.gen_defaults, **config} | |
| local = r.provider in LOCAL_COMPUTE | |
| if local: # a timeout would abandon a generation that keeps running and piles up behind the next call | |
| cfg.update(attempt_timeout=None, timeout=None, max_retries=0) | |
| elif len(chain) > 1: | |
| # The explicit fallback chain is the retry strategy; fail over immediately. | |
| cfg["max_retries"] = 0 | |
| else: | |
| # pinned to one remote model: there is nothing to fail over to, so wait out 429s instead of | |
| # dropping the turn (Inspect backs off exponentially between retries) | |
| cfg["max_retries"] = max(int(cfg.get("max_retries") or 0), PINNED_RETRIES) | |
| cfg["timeout"] = max(float(cfg.get("timeout") or 0), PINNED_TIMEOUT) | |
| delay = RateLimiter.reserve(r.provider, r.rpm) | |
| if delay: | |
| await asyncio.sleep(delay) # non-blocking: parallel sellers keep their own slots | |
| try: | |
| model = self._get(r) | |
| gc = GenerateConfig(**{k: v for k, v in cfg.items() if k in GenerateConfig.model_fields}) | |
| # hard wall-clock guard on top of Inspect's retry budget: a stuck endpoint fails over | |
| out = await asyncio.wait_for(model.generate(messages, tools=use_tools or [], config=gc), | |
| timeout=None if local else float(cfg.get("timeout") or 180) + 30) | |
| except Exception as e: # rate limit, quota, network, unsupported tools ... | |
| msg = str(e) | |
| if use_tools and r.tools in ("auto", "native") and _tools_unsupported(msg): | |
| self._json_mode.add(spec) # retry this model in prompted-JSON tool mode | |
| raise ToolModeSwitch(spec) from e | |
| self.meter.fail(spec) | |
| if _is_rate_limited(e): | |
| cooldown = max(0.0, r.rate_limit_cooldown_s) | |
| if cooldown: | |
| with self._cooldown_lock: | |
| self._provider_cooldowns[r.provider] = max( | |
| self._provider_cooldowns.get(r.provider, 0.0), | |
| time.monotonic() + cooldown, | |
| ) | |
| errors.append(f"{spec}: {type(e).__name__}: {msg[:160]}") | |
| continue | |
| if out.stop_reason == "model_length" and not out.completion and not out.message.tool_calls: | |
| errors.append(f"{spec}: empty output") | |
| continue | |
| u = out.usage | |
| self.meter.add(role, spec, (u.input_tokens or 0) if u else 0, (u.output_tokens or 0) if u else 0, | |
| served=getattr(out, "model", None)) | |
| out.metadata = {**(out.metadata or {}), "clab_spec": spec} | |
| return out | |
| msg = f"all models failed for role {role}: " + " | ".join(errors) | |
| import re as _re | |
| key = _re.sub(r" at 0x[0-9a-f]+", "", msg)[:200] # same failure type -> one warning (+ a running count) | |
| self._warn_count[key] = self._warn_count.get(key, 0) + 1 | |
| n = self._warn_count[key] | |
| if n == 1 or n in (10, 100, 1000): | |
| self._warned.add(msg) | |
| print(f" !! {msg[:400]}" + (f" (x{n} so far)" if n > 1 else ""), flush=True) | |
| raise AllModelsFailed(msg) | |
| class ToolModeSwitch(RuntimeError): | |
| """Raised once when a model rejects native tools; the caller retries in JSON tool mode.""" | |
| def __init__(self, spec: str): | |
| super().__init__(spec) | |
| self.spec = spec | |
| def _is_rate_limited(error: Exception) -> bool: | |
| response = getattr(error, "response", None) | |
| status_code = getattr(error, "status_code", None) or getattr(response, "status_code", None) | |
| if status_code == 429: | |
| return True | |
| message = str(error).lower() | |
| return any(marker in message for marker in ( | |
| "429", "rate limit", "rate_limit", "too many requests", "quota exceeded", | |
| )) | |
| def _tools_unsupported(msg: str) -> bool: | |
| m = msg.lower() | |
| return any(s in m for s in ("does not support tools", "tool use is not supported", "tools are not supported", | |
| "function calling is not", "tool_choice", "does not support function")) | |
| __all__ = ["ModelRoster", "BudgetExceeded", "AllModelsFailed", "ToolModeSwitch", "family", "CostMeter"] | |