Spaces:
Running
Running
File size: 19,286 Bytes
bd26047 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 | """Dynamic model pool: use *whatever free model is available* for the API keys you
configured in free-coding-models (FCM), then fall back to the FCM router, local
Ollama and any other keyed provider.
Startup pipeline (`clab pool discover`, or automatically for profiles using `auto:` specs):
1. keys - read FCM's config (~/.free-coding-models.json or a copy next to the repo):
`apiKeys: {provider: key}`; export them as the env vars our registry uses
(no need to duplicate keys in .env).
2. discover - candidates from (a) FCM router sets in that config, (b) each keyed provider's
live `/models` catalogue (OpenRouter: `:free` / zero-price only),
(c) `free-coding-models --json` if the CLI is on PATH, (d) local Ollama tags.
3. probe - one tiny text call + one tool-call probe per candidate (concurrently, capped);
results cached in .cache/model_pool.json (TTL).
4. assign - per role: a ranked fallback chain; agent roles get different families where
possible; the trusted monitor gets a model from a family that NO agent chain
uses; tails: FCM router (fcm/fcm) and local Ollama models.
Agents call models *directly at the provider* (with the FCM keys) so the served model is known
and pinned; the FCM router is only a last resort (it injects its own pre-prompt and picks models
dynamically, which matters for an experiment).
"""
from __future__ import annotations
import asyncio
import json
import math
import os
import re
import shutil
import subprocess
import time
import urllib.request
from dataclasses import asdict, dataclass, field
from pathlib import Path
from typing import Any
from .providers import REPO_ROOT, family, registry, resolve
CACHE_FILE = Path(os.environ.get("CLAB_CACHE", REPO_ROOT / ".cache")) / "model_pool.json"
# FCM provider keys -> our registry names (identity when equal)
FCM_TO_REGISTRY = {"googleai": "google", "google-ai-studio": "google", "gemini": "google", "nim": "nvidia",
"github-models": "github", "hf": "huggingface", "ovhcloud": "ovh", "alibaba": "dashscope",
"ollama-cloud": "ollama_cloud", "cloudflare-workers-ai": "cloudflare", "mistral-codestral": "codestral"}
NON_CHAT = re.compile(r"embed|whisper|tts|speech|audio|transcri|guard|moderation|rerank|safety|clip|"
r"flux|stable-diffusion|sdxl|image-gen|dall-e|ocr|reward|retriever|parse|nv-embed|bge|e5-", re.I)
CODER = re.compile(r"coder|code|devstral|codestral|swe|qwen3|gpt-oss|kimi|deepseek|glm|nemotron", re.I)
# --------------------------------------------------------------------------- FCM config
def fcm_config_path() -> Path | None:
cands = [os.environ.get("FCM_CONFIG", ""), str(Path.home() / ".free-coding-models.json"),
str(REPO_ROOT.parent / ".free-coding-models.json"), str(REPO_ROOT / ".free-coding-models.json"),
str(Path.home() / ".free-coding-models" / "config.json")]
for c in cands:
if c and Path(c).exists():
return Path(c)
return None
def load_fcm_config() -> dict[str, Any]:
p = fcm_config_path()
return json.loads(p.read_text(encoding="utf-8")) if p else {}
def fcm_keys(cfg: dict[str, Any] | None = None) -> dict[str, str]:
"""{registry_provider: key} from FCM `apiKeys` (strings or {key|apiKey|token} objects)."""
cfg = cfg if cfg is not None else load_fcm_config()
out = {}
for prov, v in (cfg.get("apiKeys") or {}).items():
key = v if isinstance(v, str) else next((v.get(k) for k in ("key", "apiKey", "api_key", "token")
if isinstance(v, dict) and v.get(k)), None)
if key:
out[FCM_TO_REGISTRY.get(prov.lower(), prov.lower())] = key
return out
def inject_fcm_keys(verbose: bool = False) -> dict[str, str]:
"""Export FCM keys as the env vars our registry reads (never overrides existing env)."""
reg = registry()["providers"]
applied = {}
for prov, key in fcm_keys().items():
envs = reg.get(prov, {}).get("api_key_env", [])
if envs and not any(os.environ.get(e) for e in envs):
os.environ[envs[0]] = key
applied[prov] = envs[0]
if verbose and applied:
print("FCM keys applied:", ", ".join(f"{p}->{e}" for p, e in applied.items()))
return applied
def fcm_router_sets(cfg: dict[str, Any] | None = None) -> dict[str, list[dict]]:
cfg = cfg if cfg is not None else load_fcm_config()
return {n: s.get("models", []) for n, s in ((cfg.get("router") or {}).get("sets") or {}).items()}
# --------------------------------------------------------------------------- discovery
@dataclass
class Candidate:
spec: str
provider: str
model: str
family: str
source: str
fcm_priority: int | None = None
ok: bool | None = None
tools: str = "?"
latency_s: float | None = None
error: str = ""
score: float = 0.0
size_b: float | None = None
def _http_json(url: str, key: str | None, timeout: int = 15) -> Any:
req = urllib.request.Request(url, headers={"Authorization": f"Bearer {key}"} if key else {})
with urllib.request.urlopen(req, timeout=timeout) as r:
return json.loads(r.read().decode("utf-8"))
def _size(model: str) -> float | None:
m = re.findall(r"(\d+(?:\.\d+)?)\s*b(?![a-z])", model.lower())
return max(float(x) for x in m) if m else None
def provider_catalog(provider: str) -> list[str]:
r = resolve(f"{provider}/x")
if not r.base_url or not r.has_key:
return []
try:
data = _http_json(r.base_url.rstrip("/") + "/models", r.api_key)
except Exception:
return []
items = data.get("data", data) if isinstance(data, dict) else data
ids = []
for it in items if isinstance(items, list) else []:
mid = it.get("id") if isinstance(it, dict) else (it if isinstance(it, str) else None)
if not mid or NON_CHAT.search(mid):
continue
if provider == "openrouter": # free only
pr = (it.get("pricing") or {}) if isinstance(it, dict) else {}
if not (mid.endswith(":free") or (str(pr.get("prompt")) in ("0", "0.0") and str(pr.get("completion")) in ("0", "0.0"))):
continue
if isinstance(it, dict) and it.get("supported_parameters") and "tools" not in it["supported_parameters"]:
continue
ids.append(mid)
return ids
def fcm_cli_models() -> list[tuple[str, str, str]]:
"""(provider, model, tier) from `free-coding-models --json` (best effort, schema not documented)."""
exe = shutil.which("free-coding-models")
if not exe:
return []
try:
out = subprocess.run([exe, "--json"], capture_output=True, text=True, timeout=120,
encoding="utf-8", errors="replace").stdout
data = json.loads(out[out.index("["):]) if "[" in out else []
except Exception:
return []
res = []
for d in data if isinstance(data, list) else []:
if not isinstance(d, dict):
continue
prov = next((d[k] for k in ("providerKey", "provider", "origin", "source") if isinstance(d.get(k), str)), None)
mid = next((d[k] for k in ("modelId", "model", "id") if isinstance(d.get(k), str)), None)
status = str(d.get("status", d.get("up", "up"))).lower()
if prov and mid and status not in ("down", "false", "error", "auth"):
res.append((FCM_TO_REGISTRY.get(prov.lower(), prov.lower()), mid, str(d.get("tier", ""))))
return res
def ollama_models() -> list[str]:
base = resolve("ollama/x").base_url or "http://localhost:11434/v1"
root = re.sub(r"/v1/?$", "", base)
try:
return [m["name"] for m in _http_json(root + "/api/tags", None, timeout=4).get("models", [])]
except Exception:
return []
def fcm_router_up() -> bool:
base = resolve("fcm/fcm").base_url or "http://localhost:19280/v1"
try:
return _http_json(re.sub(r"/v1/?$", "", base) + "/health", None, timeout=3) is not None
except Exception:
return False
def gather_candidates(sources: list[str]) -> list[Candidate]:
reg = registry()["providers"]
# only providers for which a real key is configured (FCM config or env), not built-in placeholders
keyed = [p for p, cfg in reg.items() if p not in ("ollama", "fcm", "litellm", "custom", "mockllm")
and any(os.environ.get(e) for e in cfg.get("api_key_env", []))]
seen: dict[str, Candidate] = {}
def add(provider: str, model: str, source: str, prio: int | None = None):
if provider not in reg or (provider not in ("ollama",) and provider not in keyed):
return
spec = f"{provider}/{model}"
if spec not in seen:
seen[spec] = Candidate(spec, provider, model, family(model), source, prio, size_b=_size(model))
elif prio is not None and seen[spec].fcm_priority is None:
seen[spec].fcm_priority = prio
if "fcm_sets" in sources:
for sname, models in fcm_router_sets().items():
for m in models:
add(FCM_TO_REGISTRY.get(m["provider"].lower(), m["provider"].lower()), m["model"],
f"fcm-set:{sname}", m.get("priority"))
if "fcm_cli" in sources:
for p, m, _tier in fcm_cli_models():
add(p, m, "fcm-cli")
if "provider_catalogs" in sources:
for p in keyed:
for m in provider_catalog(p):
add(p, m, "catalog")
if "ollama" in sources:
for m in ollama_models():
add("ollama", m, "ollama")
return list(seen.values())
# --------------------------------------------------------------------------- probe + rank
def prerank(c: Candidate) -> float:
s = 0.0
if c.fcm_priority is not None:
s += 3 - 0.2 * c.fcm_priority
if c.size_b:
s += min(3.0, math.log10(max(c.size_b, 1)) * 1.2)
if CODER.search(c.model):
s += 1.0
if c.provider == "ollama":
s -= 2.0 # local small models are a fallback tier
return s
async def _probe_all(cands: list[Candidate], concurrency: int, timeout: int) -> None:
from .probe import probe_guarded
sem = asyncio.Semaphore(concurrency)
done = 0
async def one(c: Candidate):
nonlocal done
async with sem:
r = await probe_guarded(c.spec, timeout)
c.ok = r["text"] == "ok" or str(r["text"]).startswith("odd")
c.tools, c.latency_s, c.error = r["tools"], r["latency_s"], r["error"]
done += 1
print(f" [{done}/{len(cands)}] {c.spec}: {'OK' if c.ok else 'FAIL'} tools={c.tools} "
f"{'' if c.latency_s is None else str(c.latency_s) + 's'} {c.error[:70]}", flush=True)
await asyncio.gather(*[one(c) for c in cands])
def score(c: Candidate) -> float:
if not c.ok:
return -1e9
s = prerank(c)
s += 2.0 if c.tools == "native" else 0.0
if c.latency_s is not None:
s -= min(2.0, c.latency_s / 10)
return round(s, 3)
def discover(sources: list[str] | None = None, max_probe: int = 24, concurrency: int = 6,
timeout: int = 30, ttl_hours: float = 6, refresh: bool = False, verbose: bool = True) -> list[Candidate]:
inject_fcm_keys(verbose)
if not refresh and CACHE_FILE.exists():
d = json.loads(CACHE_FILE.read_text(encoding="utf-8"))
if time.time() - d.get("created", 0) < ttl_hours * 3600 and any(c.get("ok") for c in d["candidates"]):
return [Candidate(**c) for c in d["candidates"]]
sources = sources or ["fcm_sets", "fcm_cli", "provider_catalogs", "ollama"]
cands = gather_candidates(sources)
if verbose:
by = {}
for c in cands:
by[c.provider] = by.get(c.provider, 0) + 1
print(f"discovered {len(cands)} candidate models: {by}")
cands.sort(key=prerank, reverse=True)
# keep FCM-set models and local models always; cap the rest per provider for probe budget
picked, per_prov = [], {}
for c in cands:
cap = max(4, max_probe // max(1, len({x.provider for x in cands})))
if c.source.startswith("fcm-set") or c.provider == "ollama" or per_prov.get(c.provider, 0) < cap:
picked.append(c)
per_prov[c.provider] = per_prov.get(c.provider, 0) + 1
if len(picked) >= max_probe + 8:
break
if verbose:
print(f"probing {len(picked)} models (text + tool call)...")
from .agent_loop import run_sync
run_sync(_probe_all(picked, concurrency, timeout))
for c in picked:
c.score = score(c)
picked.sort(key=lambda c: c.score, reverse=True)
CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
CACHE_FILE.write_text(json.dumps({"created": time.time(), "candidates": [asdict(c) for c in picked]}, indent=1),
encoding="utf-8")
return picked
# --------------------------------------------------------------------------- assign
AGENT_ORDER = ["implementer", "tester", "reviewer", "verifier", "planner", "seller"]
def assign(roles: dict[str, str], pool: list[Candidate], chain_length: int = 4, diverse: bool = True,
last_resort: list[str] | None = None, monitor_family: str | None = None) -> dict[str, list[str]]:
"""roles: {role: 'auto:<kind>'} (kind: coder | agent | chat | judge). Returns explicit chains."""
good = [c for c in pool if c.ok]
if not good and not last_resort:
raise RuntimeError("model pool is empty: no free model answered (check keys / `clab fcm status`)")
last_resort = last_resort or []
agents = [r for r in roles if r != "monitor_judge"]
out: dict[str, list[str]] = {}
# monitor first: best text-capable model whose family is not the best agent family;
# that family is then excluded from every agent chain. (If the monitor is pinned explicitly,
# `monitor_family` is passed in and excluded the same way.)
if "monitor_judge" in roles:
top_agent_fam = next((c.family for c in good if c.provider != "ollama"), None)
fams = []
for c in good:
if c.family not in fams and c.family not in ("unknown", "router:dynamic"):
fams.append(c.family)
choice = next((f for f in fams if f != top_agent_fam), None) or next(iter(fams), None)
monitor_family = choice
mchain = [c.spec for c in good if c.family == choice][:chain_length]
out["monitor_judge"] = mchain or ["none"]
agent_pool = [c for c in good if c.family != monitor_family]
used_fams: set[str] = set()
for role in sorted(agents, key=lambda r: AGENT_ORDER.index(r) if r in AGENT_ORDER else 99):
kind = roles[role].split(":", 1)[1] if ":" in roles[role] else "agent"
need_tools = role != "seller" and kind != "chat"
# sellers act every round (many calls): favour fast models; coders favour capability
speed_w = 0.6 if kind == "chat" else 0.0
ranked = sorted(agent_pool, key=lambda c: c.score + (1.0 if kind == "coder" and CODER.search(c.model) else 0)
- (3.0 if need_tools and c.tools != "native" else 0)
- speed_w * (c.latency_s or 10.0), reverse=True)
primary = next((c for c in ranked if not diverse or c.family not in used_fams), ranked[0] if ranked else None)
chain = ([primary.spec] if primary else []) + [c.spec for c in ranked if primary is None or c.spec != primary.spec]
chain = chain[:chain_length]
if primary:
used_fams.add(primary.family)
tail = [s for s in last_resort if s not in chain and resolve(s).family != monitor_family]
out[role] = chain + tail
return out
def expand_auto(mapping: dict[str, Any], pool_cfg: dict[str, Any], verbose: bool = False) -> dict[str, list[str]]:
"""Replace `auto:*` role entries with discovered chains (uses the cached pool when fresh)."""
auto_roles = {r: (v if isinstance(v, str) else v[0]) for r, v in mapping.items()
if (isinstance(v, str) and v.startswith("auto")) or (isinstance(v, list) and v and str(v[0]).startswith("auto"))}
if not auto_roles:
return {}
roles = dict(auto_roles)
if "default" in roles:
d = roles.pop("default")
for r in ("planner", "implementer", "tester", "reviewer", "verifier", "seller"):
if r not in mapping:
roles[r] = d
pool = discover(pool_cfg.get("sources"), pool_cfg.get("max_probe", 24), pool_cfg.get("concurrency", 6),
pool_cfg.get("timeout", 30), pool_cfg.get("ttl_hours", 6), False, verbose)
last = list(pool_cfg.get("last_resort", []))
if "fcm/fcm" in last and not fcm_router_up():
last.remove("fcm/fcm")
if "ollama" in last:
i = last.index("ollama")
last[i:i + 1] = [f"ollama/{m}" for m in ollama_models()]
fastest_roles = [r for r, v in roles.items() if v.split(":", 1)[-1] == "fastest"]
if fastest_roles:
return _expand_fastest(roles, fastest_roles, pool, pool_cfg, mapping)
mon_fam = None
mj = mapping.get("monitor_judge")
if mj and "monitor_judge" not in auto_roles:
first = mj if isinstance(mj, str) else mj[0]
if first not in ("none", "scripted"):
mon_fam = resolve(first).family
return assign(roles, pool, pool_cfg.get("chain_length", 4), pool_cfg.get("diverse_roles", True), last, mon_fam)
TRUSTED_ROLES = ("monitor_judge", "qa_peer")
def _expand_fastest(roles: dict[str, str], fastest_roles: list[str], pool: list[Candidate],
pool_cfg: dict[str, Any], mapping: dict[str, Any]) -> dict[str, list[str]]:
"""`auto:fastest`: every such role is pinned to ONE model - the fastest reliable one from the benchmark
(no fallback chain, so all agents really are the same model). Trusted roles left on auto get the best
usable model from a different family."""
from . import bench as B
rows = B.cached(pool_cfg.get("ttl_hours", 6))
if rows is None:
specs = B.candidates_from_pool(pool, pool_cfg.get("bench_top", 8))
print(f"benchmarking {len(specs)} free models on realistic agent calls (once per "
f"{pool_cfg.get('ttl_hours', 6)} h; `clab pool bench` to redo)...", flush=True)
rows = B.bench(specs, pool_cfg.get("bench_calls", 6))
best = B.pick_fastest(rows)
if best is None:
raise RuntimeError("no free model passed the benchmark (see `clab pool bench`); pin one with --role default=...")
print(f"fastest reliable free model: {best['spec']} (median {best['median_s']} s per call, "
f"fail {best['fail_rate']:.0%})", flush=True)
out: dict[str, list[str]] = {r: [best["spec"]] for r in fastest_roles}
others = [c for c in pool if c.ok and c.family not in (best["family"], "unknown", "router:dynamic")
and c.provider != "ollama"]
others.sort(key=lambda c: c.score, reverse=True)
for r in roles:
if r in out:
continue
if r in TRUSTED_ROLES:
out[r] = [c.spec for c in others][: pool_cfg.get("chain_length", 4)] or ["none"]
else:
out[r] = [best["spec"]]
return out
|