"""Dynamic model pool: use *whatever free model is available* for the API keys you configured in free-coding-models (FCM), then fall back to the FCM router, local Ollama and any other keyed provider. Startup pipeline (`clab pool discover`, or automatically for profiles using `auto:` specs): 1. keys - read FCM's config (~/.free-coding-models.json or a copy next to the repo): `apiKeys: {provider: key}`; export them as the env vars our registry uses (no need to duplicate keys in .env). 2. discover - candidates from (a) FCM router sets in that config, (b) each keyed provider's live `/models` catalogue (OpenRouter: `:free` / zero-price only), (c) `free-coding-models --json` if the CLI is on PATH, (d) local Ollama tags. 3. probe - one tiny text call + one tool-call probe per candidate (concurrently, capped); results cached in .cache/model_pool.json (TTL). 4. assign - per role: a ranked fallback chain; agent roles get different families where possible; the trusted monitor gets a model from a family that NO agent chain uses; tails: FCM router (fcm/fcm) and local Ollama models. Agents call models *directly at the provider* (with the FCM keys) so the served model is known and pinned; the FCM router is only a last resort (it injects its own pre-prompt and picks models dynamically, which matters for an experiment). """ from __future__ import annotations import asyncio import json import math import os import re import shutil import subprocess import time import urllib.request from dataclasses import asdict, dataclass, field from pathlib import Path from typing import Any from .providers import REPO_ROOT, family, registry, resolve CACHE_FILE = Path(os.environ.get("CLAB_CACHE", REPO_ROOT / ".cache")) / "model_pool.json" # FCM provider keys -> our registry names (identity when equal) FCM_TO_REGISTRY = {"googleai": "google", "google-ai-studio": "google", "gemini": "google", "nim": "nvidia", "github-models": "github", "hf": "huggingface", "ovhcloud": "ovh", "alibaba": "dashscope", "ollama-cloud": "ollama_cloud", "cloudflare-workers-ai": "cloudflare", "mistral-codestral": "codestral"} NON_CHAT = re.compile(r"embed|whisper|tts|speech|audio|transcri|guard|moderation|rerank|safety|clip|" r"flux|stable-diffusion|sdxl|image-gen|dall-e|ocr|reward|retriever|parse|nv-embed|bge|e5-", re.I) CODER = re.compile(r"coder|code|devstral|codestral|swe|qwen3|gpt-oss|kimi|deepseek|glm|nemotron", re.I) # --------------------------------------------------------------------------- FCM config def fcm_config_path() -> Path | None: cands = [os.environ.get("FCM_CONFIG", ""), str(Path.home() / ".free-coding-models.json"), str(REPO_ROOT.parent / ".free-coding-models.json"), str(REPO_ROOT / ".free-coding-models.json"), str(Path.home() / ".free-coding-models" / "config.json")] for c in cands: if c and Path(c).exists(): return Path(c) return None def load_fcm_config() -> dict[str, Any]: p = fcm_config_path() return json.loads(p.read_text(encoding="utf-8")) if p else {} def fcm_keys(cfg: dict[str, Any] | None = None) -> dict[str, str]: """{registry_provider: key} from FCM `apiKeys` (strings or {key|apiKey|token} objects).""" cfg = cfg if cfg is not None else load_fcm_config() out = {} for prov, v in (cfg.get("apiKeys") or {}).items(): key = v if isinstance(v, str) else next((v.get(k) for k in ("key", "apiKey", "api_key", "token") if isinstance(v, dict) and v.get(k)), None) if key: out[FCM_TO_REGISTRY.get(prov.lower(), prov.lower())] = key return out def inject_fcm_keys(verbose: bool = False) -> dict[str, str]: """Export FCM keys as the env vars our registry reads (never overrides existing env).""" reg = registry()["providers"] applied = {} for prov, key in fcm_keys().items(): envs = reg.get(prov, {}).get("api_key_env", []) if envs and not any(os.environ.get(e) for e in envs): os.environ[envs[0]] = key applied[prov] = envs[0] if verbose and applied: print("FCM keys applied:", ", ".join(f"{p}->{e}" for p, e in applied.items())) return applied def fcm_router_sets(cfg: dict[str, Any] | None = None) -> dict[str, list[dict]]: cfg = cfg if cfg is not None else load_fcm_config() return {n: s.get("models", []) for n, s in ((cfg.get("router") or {}).get("sets") or {}).items()} # --------------------------------------------------------------------------- discovery @dataclass class Candidate: spec: str provider: str model: str family: str source: str fcm_priority: int | None = None ok: bool | None = None tools: str = "?" latency_s: float | None = None error: str = "" score: float = 0.0 size_b: float | None = None def _http_json(url: str, key: str | None, timeout: int = 15) -> Any: req = urllib.request.Request(url, headers={"Authorization": f"Bearer {key}"} if key else {}) with urllib.request.urlopen(req, timeout=timeout) as r: return json.loads(r.read().decode("utf-8")) def _size(model: str) -> float | None: m = re.findall(r"(\d+(?:\.\d+)?)\s*b(?![a-z])", model.lower()) return max(float(x) for x in m) if m else None def provider_catalog(provider: str) -> list[str]: r = resolve(f"{provider}/x") if not r.base_url or not r.has_key: return [] try: data = _http_json(r.base_url.rstrip("/") + "/models", r.api_key) except Exception: return [] items = data.get("data", data) if isinstance(data, dict) else data ids = [] for it in items if isinstance(items, list) else []: mid = it.get("id") if isinstance(it, dict) else (it if isinstance(it, str) else None) if not mid or NON_CHAT.search(mid): continue if provider == "openrouter": # free only pr = (it.get("pricing") or {}) if isinstance(it, dict) else {} if not (mid.endswith(":free") or (str(pr.get("prompt")) in ("0", "0.0") and str(pr.get("completion")) in ("0", "0.0"))): continue if isinstance(it, dict) and it.get("supported_parameters") and "tools" not in it["supported_parameters"]: continue ids.append(mid) return ids def fcm_cli_models() -> list[tuple[str, str, str]]: """(provider, model, tier) from `free-coding-models --json` (best effort, schema not documented).""" exe = shutil.which("free-coding-models") if not exe: return [] try: out = subprocess.run([exe, "--json"], capture_output=True, text=True, timeout=120, encoding="utf-8", errors="replace").stdout data = json.loads(out[out.index("["):]) if "[" in out else [] except Exception: return [] res = [] for d in data if isinstance(data, list) else []: if not isinstance(d, dict): continue prov = next((d[k] for k in ("providerKey", "provider", "origin", "source") if isinstance(d.get(k), str)), None) mid = next((d[k] for k in ("modelId", "model", "id") if isinstance(d.get(k), str)), None) status = str(d.get("status", d.get("up", "up"))).lower() if prov and mid and status not in ("down", "false", "error", "auth"): res.append((FCM_TO_REGISTRY.get(prov.lower(), prov.lower()), mid, str(d.get("tier", "")))) return res def ollama_models() -> list[str]: base = resolve("ollama/x").base_url or "http://localhost:11434/v1" root = re.sub(r"/v1/?$", "", base) try: return [m["name"] for m in _http_json(root + "/api/tags", None, timeout=4).get("models", [])] except Exception: return [] def fcm_router_up() -> bool: base = resolve("fcm/fcm").base_url or "http://localhost:19280/v1" try: return _http_json(re.sub(r"/v1/?$", "", base) + "/health", None, timeout=3) is not None except Exception: return False def gather_candidates(sources: list[str]) -> list[Candidate]: reg = registry()["providers"] # only providers for which a real key is configured (FCM config or env), not built-in placeholders keyed = [p for p, cfg in reg.items() if p not in ("ollama", "fcm", "litellm", "custom", "mockllm") and any(os.environ.get(e) for e in cfg.get("api_key_env", []))] seen: dict[str, Candidate] = {} def add(provider: str, model: str, source: str, prio: int | None = None): if provider not in reg or (provider not in ("ollama",) and provider not in keyed): return spec = f"{provider}/{model}" if spec not in seen: seen[spec] = Candidate(spec, provider, model, family(model), source, prio, size_b=_size(model)) elif prio is not None and seen[spec].fcm_priority is None: seen[spec].fcm_priority = prio if "fcm_sets" in sources: for sname, models in fcm_router_sets().items(): for m in models: add(FCM_TO_REGISTRY.get(m["provider"].lower(), m["provider"].lower()), m["model"], f"fcm-set:{sname}", m.get("priority")) if "fcm_cli" in sources: for p, m, _tier in fcm_cli_models(): add(p, m, "fcm-cli") if "provider_catalogs" in sources: for p in keyed: for m in provider_catalog(p): add(p, m, "catalog") if "ollama" in sources: for m in ollama_models(): add("ollama", m, "ollama") return list(seen.values()) # --------------------------------------------------------------------------- probe + rank def prerank(c: Candidate) -> float: s = 0.0 if c.fcm_priority is not None: s += 3 - 0.2 * c.fcm_priority if c.size_b: s += min(3.0, math.log10(max(c.size_b, 1)) * 1.2) if CODER.search(c.model): s += 1.0 if c.provider == "ollama": s -= 2.0 # local small models are a fallback tier return s async def _probe_all(cands: list[Candidate], concurrency: int, timeout: int) -> None: from .probe import probe_guarded sem = asyncio.Semaphore(concurrency) done = 0 async def one(c: Candidate): nonlocal done async with sem: r = await probe_guarded(c.spec, timeout) c.ok = r["text"] == "ok" or str(r["text"]).startswith("odd") c.tools, c.latency_s, c.error = r["tools"], r["latency_s"], r["error"] done += 1 print(f" [{done}/{len(cands)}] {c.spec}: {'OK' if c.ok else 'FAIL'} tools={c.tools} " f"{'' if c.latency_s is None else str(c.latency_s) + 's'} {c.error[:70]}", flush=True) await asyncio.gather(*[one(c) for c in cands]) def score(c: Candidate) -> float: if not c.ok: return -1e9 s = prerank(c) s += 2.0 if c.tools == "native" else 0.0 if c.latency_s is not None: s -= min(2.0, c.latency_s / 10) return round(s, 3) def discover(sources: list[str] | None = None, max_probe: int = 24, concurrency: int = 6, timeout: int = 30, ttl_hours: float = 6, refresh: bool = False, verbose: bool = True) -> list[Candidate]: inject_fcm_keys(verbose) if not refresh and CACHE_FILE.exists(): d = json.loads(CACHE_FILE.read_text(encoding="utf-8")) if time.time() - d.get("created", 0) < ttl_hours * 3600 and any(c.get("ok") for c in d["candidates"]): return [Candidate(**c) for c in d["candidates"]] sources = sources or ["fcm_sets", "fcm_cli", "provider_catalogs", "ollama"] cands = gather_candidates(sources) if verbose: by = {} for c in cands: by[c.provider] = by.get(c.provider, 0) + 1 print(f"discovered {len(cands)} candidate models: {by}") cands.sort(key=prerank, reverse=True) # keep FCM-set models and local models always; cap the rest per provider for probe budget picked, per_prov = [], {} for c in cands: cap = max(4, max_probe // max(1, len({x.provider for x in cands}))) if c.source.startswith("fcm-set") or c.provider == "ollama" or per_prov.get(c.provider, 0) < cap: picked.append(c) per_prov[c.provider] = per_prov.get(c.provider, 0) + 1 if len(picked) >= max_probe + 8: break if verbose: print(f"probing {len(picked)} models (text + tool call)...") from .agent_loop import run_sync run_sync(_probe_all(picked, concurrency, timeout)) for c in picked: c.score = score(c) picked.sort(key=lambda c: c.score, reverse=True) CACHE_FILE.parent.mkdir(parents=True, exist_ok=True) CACHE_FILE.write_text(json.dumps({"created": time.time(), "candidates": [asdict(c) for c in picked]}, indent=1), encoding="utf-8") return picked # --------------------------------------------------------------------------- assign AGENT_ORDER = ["implementer", "tester", "reviewer", "verifier", "planner", "seller"] def assign(roles: dict[str, str], pool: list[Candidate], chain_length: int = 4, diverse: bool = True, last_resort: list[str] | None = None, monitor_family: str | None = None) -> dict[str, list[str]]: """roles: {role: 'auto:'} (kind: coder | agent | chat | judge). Returns explicit chains.""" good = [c for c in pool if c.ok] if not good and not last_resort: raise RuntimeError("model pool is empty: no free model answered (check keys / `clab fcm status`)") last_resort = last_resort or [] agents = [r for r in roles if r != "monitor_judge"] out: dict[str, list[str]] = {} # monitor first: best text-capable model whose family is not the best agent family; # that family is then excluded from every agent chain. (If the monitor is pinned explicitly, # `monitor_family` is passed in and excluded the same way.) if "monitor_judge" in roles: top_agent_fam = next((c.family for c in good if c.provider != "ollama"), None) fams = [] for c in good: if c.family not in fams and c.family not in ("unknown", "router:dynamic"): fams.append(c.family) choice = next((f for f in fams if f != top_agent_fam), None) or next(iter(fams), None) monitor_family = choice mchain = [c.spec for c in good if c.family == choice][:chain_length] out["monitor_judge"] = mchain or ["none"] agent_pool = [c for c in good if c.family != monitor_family] used_fams: set[str] = set() for role in sorted(agents, key=lambda r: AGENT_ORDER.index(r) if r in AGENT_ORDER else 99): kind = roles[role].split(":", 1)[1] if ":" in roles[role] else "agent" need_tools = role != "seller" and kind != "chat" # sellers act every round (many calls): favour fast models; coders favour capability speed_w = 0.6 if kind == "chat" else 0.0 ranked = sorted(agent_pool, key=lambda c: c.score + (1.0 if kind == "coder" and CODER.search(c.model) else 0) - (3.0 if need_tools and c.tools != "native" else 0) - speed_w * (c.latency_s or 10.0), reverse=True) primary = next((c for c in ranked if not diverse or c.family not in used_fams), ranked[0] if ranked else None) chain = ([primary.spec] if primary else []) + [c.spec for c in ranked if primary is None or c.spec != primary.spec] chain = chain[:chain_length] if primary: used_fams.add(primary.family) tail = [s for s in last_resort if s not in chain and resolve(s).family != monitor_family] out[role] = chain + tail return out def expand_auto(mapping: dict[str, Any], pool_cfg: dict[str, Any], verbose: bool = False) -> dict[str, list[str]]: """Replace `auto:*` role entries with discovered chains (uses the cached pool when fresh).""" auto_roles = {r: (v if isinstance(v, str) else v[0]) for r, v in mapping.items() if (isinstance(v, str) and v.startswith("auto")) or (isinstance(v, list) and v and str(v[0]).startswith("auto"))} if not auto_roles: return {} roles = dict(auto_roles) if "default" in roles: d = roles.pop("default") for r in ("planner", "implementer", "tester", "reviewer", "verifier", "seller"): if r not in mapping: roles[r] = d pool = discover(pool_cfg.get("sources"), pool_cfg.get("max_probe", 24), pool_cfg.get("concurrency", 6), pool_cfg.get("timeout", 30), pool_cfg.get("ttl_hours", 6), False, verbose) last = list(pool_cfg.get("last_resort", [])) if "fcm/fcm" in last and not fcm_router_up(): last.remove("fcm/fcm") if "ollama" in last: i = last.index("ollama") last[i:i + 1] = [f"ollama/{m}" for m in ollama_models()] fastest_roles = [r for r, v in roles.items() if v.split(":", 1)[-1] == "fastest"] if fastest_roles: return _expand_fastest(roles, fastest_roles, pool, pool_cfg, mapping) mon_fam = None mj = mapping.get("monitor_judge") if mj and "monitor_judge" not in auto_roles: first = mj if isinstance(mj, str) else mj[0] if first not in ("none", "scripted"): mon_fam = resolve(first).family return assign(roles, pool, pool_cfg.get("chain_length", 4), pool_cfg.get("diverse_roles", True), last, mon_fam) TRUSTED_ROLES = ("monitor_judge", "qa_peer") def _expand_fastest(roles: dict[str, str], fastest_roles: list[str], pool: list[Candidate], pool_cfg: dict[str, Any], mapping: dict[str, Any]) -> dict[str, list[str]]: """`auto:fastest`: every such role is pinned to ONE model - the fastest reliable one from the benchmark (no fallback chain, so all agents really are the same model). Trusted roles left on auto get the best usable model from a different family.""" from . import bench as B rows = B.cached(pool_cfg.get("ttl_hours", 6)) if rows is None: specs = B.candidates_from_pool(pool, pool_cfg.get("bench_top", 8)) print(f"benchmarking {len(specs)} free models on realistic agent calls (once per " f"{pool_cfg.get('ttl_hours', 6)} h; `clab pool bench` to redo)...", flush=True) rows = B.bench(specs, pool_cfg.get("bench_calls", 6)) best = B.pick_fastest(rows) if best is None: raise RuntimeError("no free model passed the benchmark (see `clab pool bench`); pin one with --role default=...") print(f"fastest reliable free model: {best['spec']} (median {best['median_s']} s per call, " f"fail {best['fail_rate']:.0%})", flush=True) out: dict[str, list[str]] = {r: [best["spec"]] for r in fastest_roles} others = [c for c in pool if c.ok and c.family not in (best["family"], "unknown", "router:dynamic") and c.provider != "ollama"] others.sort(key=lambda c: c.score, reverse=True) for r in roles: if r in out: continue if r in TRUSTED_ROLES: out[r] = [c.spec for c in others][: pool_cfg.get("chain_length", 4)] or ["none"] else: out[r] = [best["spec"]] return out