File size: 4,647 Bytes
bd26047
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
"""Probe providers/models: does a text call work, does native tool calling work, latency."""
from __future__ import annotations

import asyncio
import time
from typing import Any

from .providers import get_inspect_model, resolve

DEFAULT_PROBE_MODELS = {
    # one cheap, commonly-available model per provider (edit freely)
    "groq": "groq/llama-3.3-70b-versatile",
    "cerebras": "cerebras/llama-3.3-70b",
    "nvidia": "nvidia/meta/llama-3.3-70b-instruct",
    "openrouter": "openrouter/meta-llama/llama-3.3-70b-instruct:free",
    "google": "google/gemini-2.5-flash",
    "github": "github/openai/gpt-4.1-mini",
    "mistral": "mistral/mistral-small-latest",
    "codestral": "codestral/codestral-latest",
    "sambanova": "sambanova/Meta-Llama-3.3-70B-Instruct",
    "huggingface": "huggingface/meta-llama/Llama-3.3-70B-Instruct",
    "cloudflare": "cloudflare/@cf/meta/llama-3.3-70b-instruct-fp8-fast",
    "ollama": "ollama/gemma2:2b",
    "fcm": "fcm/fcm",
    "litellm": "litellm/default",
    "anthropic": "anthropic/claude-haiku-4-5",
    "openai": "openai/gpt-4.1-mini",
}


def _why(e: BaseException) -> str:
    """Unwrap tenacity RetryError to the real cause (timeout, 401, 404 model not found, ...)."""
    inner = getattr(getattr(e, "last_attempt", None), "exception", lambda: None)()
    e = inner or e
    return f"{type(e).__name__}: {str(e)[:140]}"


async def _probe(spec: str, timeout: int = 45) -> dict[str, Any]:
    from inspect_ai.model import ChatMessageUser, GenerateConfig
    from inspect_ai.tool import ToolInfo, ToolParams
    from inspect_ai.util import JSONSchema
    r = resolve(spec)
    row: dict[str, Any] = {"spec": spec, "family": r.family, "text": "-", "tools": "-", "latency_s": None,
                           "served_by": None, "error": ""}
    if r.needs_key and not r.has_key:
        row["error"] = "no API key"
        return row
    # `timeout` in Inspect bounds the *retry budget*, not a single hanging request; `attempt_timeout`
    # bounds each HTTP attempt (otherwise a cold-starting endpoint can block for the SDK's 10 min).
    kw = dict(max_retries=0, timeout=timeout, max_tokens=64, temperature=0)
    if "attempt_timeout" in GenerateConfig.model_fields:
        kw["attempt_timeout"] = timeout
    cfg = GenerateConfig(**kw)
    try:
        m = get_inspect_model(r)
        t0 = time.time()
        out = await m.generate([ChatMessageUser(content="Reply with exactly: OK")], config=cfg)
        row["latency_s"] = round(time.time() - t0, 2)
        row["text"] = "ok" if "ok" in (out.completion or "").lower() else f"odd: {out.completion[:30]!r}"
        row["served_by"] = getattr(out, "model", None)
    except Exception as e:
        row["text"] = "FAIL"
        row["error"] = _why(e)
        return row
    tool = ToolInfo(name="get_time", description="Get the time in a city",
                    parameters=ToolParams(properties={"city": JSONSchema(type="string", description="city")},
                                          required=["city"]))
    try:
        out = await m.generate([ChatMessageUser(content="What time is it in Paris? Use the tool.")],
                               tools=[tool], config=cfg)
        row["tools"] = "native" if out.message.tool_calls else "json-only"
    except Exception as e:
        row["tools"] = "json-only"
        row["error"] = f"tools: {_why(e)[:100]}"
    return row


async def probe_guarded(spec: str, timeout: int) -> dict[str, Any]:
    """Hard wall-clock cap (two calls + slack) so one stuck endpoint can never block discovery."""
    try:
        return await asyncio.wait_for(_probe(spec, timeout), timeout=2 * timeout + 10)
    except (asyncio.TimeoutError, TimeoutError):
        r = resolve(spec)
        return {"spec": spec, "family": r.family, "text": "FAIL", "tools": "-", "latency_s": None,
                "served_by": None, "error": f"timed out after {2 * timeout + 10}s"}


def probe(specs: list[str], timeout: int = 45, concurrency: int = 6, progress: bool = True) -> list[dict[str, Any]]:
    async def go():
        sem = asyncio.Semaphore(concurrency)
        done = 0

        async def one(s):
            nonlocal done
            async with sem:
                r = await probe_guarded(s, timeout)
            done += 1
            if progress:
                print(f"  [{done}/{len(specs)}] {s}: text={r['text']} tools={r['tools']} "
                      f"{'' if r['latency_s'] is None else str(r['latency_s']) + 's'} {r['error'][:70]}", flush=True)
            return r
        return await asyncio.gather(*[one(s) for s in specs])
    from .agent_loop import run_sync
    return run_sync(go())