File size: 19,286 Bytes
bd26047
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
"""Dynamic model pool: use *whatever free model is available* for the API keys you
configured in free-coding-models (FCM), then fall back to the FCM router, local
Ollama and any other keyed provider.

Startup pipeline (`clab pool discover`, or automatically for profiles using `auto:` specs):

  1. keys     - read FCM's config (~/.free-coding-models.json or a copy next to the repo):
                `apiKeys: {provider: key}`; export them as the env vars our registry uses
                (no need to duplicate keys in .env).
  2. discover - candidates from (a) FCM router sets in that config, (b) each keyed provider's
                live `/models` catalogue (OpenRouter: `:free` / zero-price only),
                (c) `free-coding-models --json` if the CLI is on PATH, (d) local Ollama tags.
  3. probe    - one tiny text call + one tool-call probe per candidate (concurrently, capped);
                results cached in .cache/model_pool.json (TTL).
  4. assign   - per role: a ranked fallback chain; agent roles get different families where
                possible; the trusted monitor gets a model from a family that NO agent chain
                uses; tails: FCM router (fcm/fcm) and local Ollama models.

Agents call models *directly at the provider* (with the FCM keys) so the served model is known
and pinned; the FCM router is only a last resort (it injects its own pre-prompt and picks models
dynamically, which matters for an experiment).
"""
from __future__ import annotations

import asyncio
import json
import math
import os
import re
import shutil
import subprocess
import time
import urllib.request
from dataclasses import asdict, dataclass, field
from pathlib import Path
from typing import Any

from .providers import REPO_ROOT, family, registry, resolve

CACHE_FILE = Path(os.environ.get("CLAB_CACHE", REPO_ROOT / ".cache")) / "model_pool.json"

# FCM provider keys -> our registry names (identity when equal)
FCM_TO_REGISTRY = {"googleai": "google", "google-ai-studio": "google", "gemini": "google", "nim": "nvidia",
                   "github-models": "github", "hf": "huggingface", "ovhcloud": "ovh", "alibaba": "dashscope",
                   "ollama-cloud": "ollama_cloud", "cloudflare-workers-ai": "cloudflare", "mistral-codestral": "codestral"}
NON_CHAT = re.compile(r"embed|whisper|tts|speech|audio|transcri|guard|moderation|rerank|safety|clip|"
                      r"flux|stable-diffusion|sdxl|image-gen|dall-e|ocr|reward|retriever|parse|nv-embed|bge|e5-", re.I)
CODER = re.compile(r"coder|code|devstral|codestral|swe|qwen3|gpt-oss|kimi|deepseek|glm|nemotron", re.I)


# --------------------------------------------------------------------------- FCM config
def fcm_config_path() -> Path | None:
    cands = [os.environ.get("FCM_CONFIG", ""), str(Path.home() / ".free-coding-models.json"),
             str(REPO_ROOT.parent / ".free-coding-models.json"), str(REPO_ROOT / ".free-coding-models.json"),
             str(Path.home() / ".free-coding-models" / "config.json")]
    for c in cands:
        if c and Path(c).exists():
            return Path(c)
    return None


def load_fcm_config() -> dict[str, Any]:
    p = fcm_config_path()
    return json.loads(p.read_text(encoding="utf-8")) if p else {}


def fcm_keys(cfg: dict[str, Any] | None = None) -> dict[str, str]:
    """{registry_provider: key} from FCM `apiKeys` (strings or {key|apiKey|token} objects)."""
    cfg = cfg if cfg is not None else load_fcm_config()
    out = {}
    for prov, v in (cfg.get("apiKeys") or {}).items():
        key = v if isinstance(v, str) else next((v.get(k) for k in ("key", "apiKey", "api_key", "token")
                                                 if isinstance(v, dict) and v.get(k)), None)
        if key:
            out[FCM_TO_REGISTRY.get(prov.lower(), prov.lower())] = key
    return out


def inject_fcm_keys(verbose: bool = False) -> dict[str, str]:
    """Export FCM keys as the env vars our registry reads (never overrides existing env)."""
    reg = registry()["providers"]
    applied = {}
    for prov, key in fcm_keys().items():
        envs = reg.get(prov, {}).get("api_key_env", [])
        if envs and not any(os.environ.get(e) for e in envs):
            os.environ[envs[0]] = key
            applied[prov] = envs[0]
    if verbose and applied:
        print("FCM keys applied:", ", ".join(f"{p}->{e}" for p, e in applied.items()))
    return applied


def fcm_router_sets(cfg: dict[str, Any] | None = None) -> dict[str, list[dict]]:
    cfg = cfg if cfg is not None else load_fcm_config()
    return {n: s.get("models", []) for n, s in ((cfg.get("router") or {}).get("sets") or {}).items()}


# --------------------------------------------------------------------------- discovery
@dataclass
class Candidate:
    spec: str
    provider: str
    model: str
    family: str
    source: str
    fcm_priority: int | None = None
    ok: bool | None = None
    tools: str = "?"
    latency_s: float | None = None
    error: str = ""
    score: float = 0.0
    size_b: float | None = None


def _http_json(url: str, key: str | None, timeout: int = 15) -> Any:
    req = urllib.request.Request(url, headers={"Authorization": f"Bearer {key}"} if key else {})
    with urllib.request.urlopen(req, timeout=timeout) as r:
        return json.loads(r.read().decode("utf-8"))


def _size(model: str) -> float | None:
    m = re.findall(r"(\d+(?:\.\d+)?)\s*b(?![a-z])", model.lower())
    return max(float(x) for x in m) if m else None


def provider_catalog(provider: str) -> list[str]:
    r = resolve(f"{provider}/x")
    if not r.base_url or not r.has_key:
        return []
    try:
        data = _http_json(r.base_url.rstrip("/") + "/models", r.api_key)
    except Exception:
        return []
    items = data.get("data", data) if isinstance(data, dict) else data
    ids = []
    for it in items if isinstance(items, list) else []:
        mid = it.get("id") if isinstance(it, dict) else (it if isinstance(it, str) else None)
        if not mid or NON_CHAT.search(mid):
            continue
        if provider == "openrouter":   # free only
            pr = (it.get("pricing") or {}) if isinstance(it, dict) else {}
            if not (mid.endswith(":free") or (str(pr.get("prompt")) in ("0", "0.0") and str(pr.get("completion")) in ("0", "0.0"))):
                continue
            if isinstance(it, dict) and it.get("supported_parameters") and "tools" not in it["supported_parameters"]:
                continue
        ids.append(mid)
    return ids


def fcm_cli_models() -> list[tuple[str, str, str]]:
    """(provider, model, tier) from `free-coding-models --json` (best effort, schema not documented)."""
    exe = shutil.which("free-coding-models")
    if not exe:
        return []
    try:
        out = subprocess.run([exe, "--json"], capture_output=True, text=True, timeout=120,
                             encoding="utf-8", errors="replace").stdout
        data = json.loads(out[out.index("["):]) if "[" in out else []
    except Exception:
        return []
    res = []
    for d in data if isinstance(data, list) else []:
        if not isinstance(d, dict):
            continue
        prov = next((d[k] for k in ("providerKey", "provider", "origin", "source") if isinstance(d.get(k), str)), None)
        mid = next((d[k] for k in ("modelId", "model", "id") if isinstance(d.get(k), str)), None)
        status = str(d.get("status", d.get("up", "up"))).lower()
        if prov and mid and status not in ("down", "false", "error", "auth"):
            res.append((FCM_TO_REGISTRY.get(prov.lower(), prov.lower()), mid, str(d.get("tier", ""))))
    return res


def ollama_models() -> list[str]:
    base = resolve("ollama/x").base_url or "http://localhost:11434/v1"
    root = re.sub(r"/v1/?$", "", base)
    try:
        return [m["name"] for m in _http_json(root + "/api/tags", None, timeout=4).get("models", [])]
    except Exception:
        return []


def fcm_router_up() -> bool:
    base = resolve("fcm/fcm").base_url or "http://localhost:19280/v1"
    try:
        return _http_json(re.sub(r"/v1/?$", "", base) + "/health", None, timeout=3) is not None
    except Exception:
        return False


def gather_candidates(sources: list[str]) -> list[Candidate]:
    reg = registry()["providers"]
    # only providers for which a real key is configured (FCM config or env), not built-in placeholders
    keyed = [p for p, cfg in reg.items() if p not in ("ollama", "fcm", "litellm", "custom", "mockllm")
             and any(os.environ.get(e) for e in cfg.get("api_key_env", []))]
    seen: dict[str, Candidate] = {}

    def add(provider: str, model: str, source: str, prio: int | None = None):
        if provider not in reg or (provider not in ("ollama",) and provider not in keyed):
            return
        spec = f"{provider}/{model}"
        if spec not in seen:
            seen[spec] = Candidate(spec, provider, model, family(model), source, prio, size_b=_size(model))
        elif prio is not None and seen[spec].fcm_priority is None:
            seen[spec].fcm_priority = prio
    if "fcm_sets" in sources:
        for sname, models in fcm_router_sets().items():
            for m in models:
                add(FCM_TO_REGISTRY.get(m["provider"].lower(), m["provider"].lower()), m["model"],
                    f"fcm-set:{sname}", m.get("priority"))
    if "fcm_cli" in sources:
        for p, m, _tier in fcm_cli_models():
            add(p, m, "fcm-cli")
    if "provider_catalogs" in sources:
        for p in keyed:
            for m in provider_catalog(p):
                add(p, m, "catalog")
    if "ollama" in sources:
        for m in ollama_models():
            add("ollama", m, "ollama")
    return list(seen.values())


# --------------------------------------------------------------------------- probe + rank
def prerank(c: Candidate) -> float:
    s = 0.0
    if c.fcm_priority is not None:
        s += 3 - 0.2 * c.fcm_priority
    if c.size_b:
        s += min(3.0, math.log10(max(c.size_b, 1)) * 1.2)
    if CODER.search(c.model):
        s += 1.0
    if c.provider == "ollama":
        s -= 2.0                    # local small models are a fallback tier
    return s


async def _probe_all(cands: list[Candidate], concurrency: int, timeout: int) -> None:
    from .probe import probe_guarded
    sem = asyncio.Semaphore(concurrency)
    done = 0

    async def one(c: Candidate):
        nonlocal done
        async with sem:
            r = await probe_guarded(c.spec, timeout)
        c.ok = r["text"] == "ok" or str(r["text"]).startswith("odd")
        c.tools, c.latency_s, c.error = r["tools"], r["latency_s"], r["error"]
        done += 1
        print(f"  [{done}/{len(cands)}] {c.spec}: {'OK' if c.ok else 'FAIL'} tools={c.tools} "
              f"{'' if c.latency_s is None else str(c.latency_s) + 's'} {c.error[:70]}", flush=True)
    await asyncio.gather(*[one(c) for c in cands])


def score(c: Candidate) -> float:
    if not c.ok:
        return -1e9
    s = prerank(c)
    s += 2.0 if c.tools == "native" else 0.0
    if c.latency_s is not None:
        s -= min(2.0, c.latency_s / 10)
    return round(s, 3)


def discover(sources: list[str] | None = None, max_probe: int = 24, concurrency: int = 6,
             timeout: int = 30, ttl_hours: float = 6, refresh: bool = False, verbose: bool = True) -> list[Candidate]:
    inject_fcm_keys(verbose)
    if not refresh and CACHE_FILE.exists():
        d = json.loads(CACHE_FILE.read_text(encoding="utf-8"))
        if time.time() - d.get("created", 0) < ttl_hours * 3600 and any(c.get("ok") for c in d["candidates"]):
            return [Candidate(**c) for c in d["candidates"]]
    sources = sources or ["fcm_sets", "fcm_cli", "provider_catalogs", "ollama"]
    cands = gather_candidates(sources)
    if verbose:
        by = {}
        for c in cands:
            by[c.provider] = by.get(c.provider, 0) + 1
        print(f"discovered {len(cands)} candidate models: {by}")
    cands.sort(key=prerank, reverse=True)
    # keep FCM-set models and local models always; cap the rest per provider for probe budget
    picked, per_prov = [], {}
    for c in cands:
        cap = max(4, max_probe // max(1, len({x.provider for x in cands})))
        if c.source.startswith("fcm-set") or c.provider == "ollama" or per_prov.get(c.provider, 0) < cap:
            picked.append(c)
            per_prov[c.provider] = per_prov.get(c.provider, 0) + 1
        if len(picked) >= max_probe + 8:
            break
    if verbose:
        print(f"probing {len(picked)} models (text + tool call)...")
    from .agent_loop import run_sync
    run_sync(_probe_all(picked, concurrency, timeout))
    for c in picked:
        c.score = score(c)
    picked.sort(key=lambda c: c.score, reverse=True)
    CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
    CACHE_FILE.write_text(json.dumps({"created": time.time(), "candidates": [asdict(c) for c in picked]}, indent=1),
                          encoding="utf-8")
    return picked


# --------------------------------------------------------------------------- assign
AGENT_ORDER = ["implementer", "tester", "reviewer", "verifier", "planner", "seller"]


def assign(roles: dict[str, str], pool: list[Candidate], chain_length: int = 4, diverse: bool = True,
           last_resort: list[str] | None = None, monitor_family: str | None = None) -> dict[str, list[str]]:
    """roles: {role: 'auto:<kind>'} (kind: coder | agent | chat | judge). Returns explicit chains."""
    good = [c for c in pool if c.ok]
    if not good and not last_resort:
        raise RuntimeError("model pool is empty: no free model answered (check keys / `clab fcm status`)")
    last_resort = last_resort or []
    agents = [r for r in roles if r != "monitor_judge"]
    out: dict[str, list[str]] = {}

    # monitor first: best text-capable model whose family is not the best agent family;
    # that family is then excluded from every agent chain. (If the monitor is pinned explicitly,
    # `monitor_family` is passed in and excluded the same way.)
    if "monitor_judge" in roles:
        top_agent_fam = next((c.family for c in good if c.provider != "ollama"), None)
        fams = []
        for c in good:
            if c.family not in fams and c.family not in ("unknown", "router:dynamic"):
                fams.append(c.family)
        choice = next((f for f in fams if f != top_agent_fam), None) or next(iter(fams), None)
        monitor_family = choice
        mchain = [c.spec for c in good if c.family == choice][:chain_length]
        out["monitor_judge"] = mchain or ["none"]

    agent_pool = [c for c in good if c.family != monitor_family]
    used_fams: set[str] = set()
    for role in sorted(agents, key=lambda r: AGENT_ORDER.index(r) if r in AGENT_ORDER else 99):
        kind = roles[role].split(":", 1)[1] if ":" in roles[role] else "agent"
        need_tools = role != "seller" and kind != "chat"
        # sellers act every round (many calls): favour fast models; coders favour capability
        speed_w = 0.6 if kind == "chat" else 0.0
        ranked = sorted(agent_pool, key=lambda c: c.score + (1.0 if kind == "coder" and CODER.search(c.model) else 0)
                        - (3.0 if need_tools and c.tools != "native" else 0)
                        - speed_w * (c.latency_s or 10.0), reverse=True)
        primary = next((c for c in ranked if not diverse or c.family not in used_fams), ranked[0] if ranked else None)
        chain = ([primary.spec] if primary else []) + [c.spec for c in ranked if primary is None or c.spec != primary.spec]
        chain = chain[:chain_length]
        if primary:
            used_fams.add(primary.family)
        tail = [s for s in last_resort if s not in chain and resolve(s).family != monitor_family]
        out[role] = chain + tail
    return out


def expand_auto(mapping: dict[str, Any], pool_cfg: dict[str, Any], verbose: bool = False) -> dict[str, list[str]]:
    """Replace `auto:*` role entries with discovered chains (uses the cached pool when fresh)."""
    auto_roles = {r: (v if isinstance(v, str) else v[0]) for r, v in mapping.items()
                  if (isinstance(v, str) and v.startswith("auto")) or (isinstance(v, list) and v and str(v[0]).startswith("auto"))}
    if not auto_roles:
        return {}
    roles = dict(auto_roles)
    if "default" in roles:
        d = roles.pop("default")
        for r in ("planner", "implementer", "tester", "reviewer", "verifier", "seller"):
            if r not in mapping:
                roles[r] = d
    pool = discover(pool_cfg.get("sources"), pool_cfg.get("max_probe", 24), pool_cfg.get("concurrency", 6),
                    pool_cfg.get("timeout", 30), pool_cfg.get("ttl_hours", 6), False, verbose)
    last = list(pool_cfg.get("last_resort", []))
    if "fcm/fcm" in last and not fcm_router_up():
        last.remove("fcm/fcm")
    if "ollama" in last:
        i = last.index("ollama")
        last[i:i + 1] = [f"ollama/{m}" for m in ollama_models()]
    fastest_roles = [r for r, v in roles.items() if v.split(":", 1)[-1] == "fastest"]
    if fastest_roles:
        return _expand_fastest(roles, fastest_roles, pool, pool_cfg, mapping)
    mon_fam = None
    mj = mapping.get("monitor_judge")
    if mj and "monitor_judge" not in auto_roles:
        first = mj if isinstance(mj, str) else mj[0]
        if first not in ("none", "scripted"):
            mon_fam = resolve(first).family
    return assign(roles, pool, pool_cfg.get("chain_length", 4), pool_cfg.get("diverse_roles", True), last, mon_fam)


TRUSTED_ROLES = ("monitor_judge", "qa_peer")


def _expand_fastest(roles: dict[str, str], fastest_roles: list[str], pool: list[Candidate],
                    pool_cfg: dict[str, Any], mapping: dict[str, Any]) -> dict[str, list[str]]:
    """`auto:fastest`: every such role is pinned to ONE model - the fastest reliable one from the benchmark
    (no fallback chain, so all agents really are the same model). Trusted roles left on auto get the best
    usable model from a different family."""
    from . import bench as B
    rows = B.cached(pool_cfg.get("ttl_hours", 6))
    if rows is None:
        specs = B.candidates_from_pool(pool, pool_cfg.get("bench_top", 8))
        print(f"benchmarking {len(specs)} free models on realistic agent calls (once per "
              f"{pool_cfg.get('ttl_hours', 6)} h; `clab pool bench` to redo)...", flush=True)
        rows = B.bench(specs, pool_cfg.get("bench_calls", 6))
    best = B.pick_fastest(rows)
    if best is None:
        raise RuntimeError("no free model passed the benchmark (see `clab pool bench`); pin one with --role default=...")
    print(f"fastest reliable free model: {best['spec']} (median {best['median_s']} s per call, "
          f"fail {best['fail_rate']:.0%})", flush=True)
    out: dict[str, list[str]] = {r: [best["spec"]] for r in fastest_roles}
    others = [c for c in pool if c.ok and c.family not in (best["family"], "unknown", "router:dynamic")
              and c.provider != "ollama"]
    others.sort(key=lambda c: c.score, reverse=True)
    for r in roles:
        if r in out:
            continue
        if r in TRUSTED_ROLES:
            out[r] = [c.spec for c in others][: pool_cfg.get("chain_length", 4)] or ["none"]
        else:
            out[r] = [best["spec"]]
    return out