Buckets:

download
raw
9.01 kB
#!/usr/bin/env python3
"""Real-model smoke for one hosted provider, run by the CI workflows.
Runs the checks the push-time and nightly workflows need against a real
endpoint: plain generation, streaming, a calculator call through an Agent, and
a yes/no reasoning question. Exit status:
* ``0`` — every check passed, **or the provider refused to serve**. A refusal
(no credit, HTTP 402; a bad or absent key, 401/403; a model id the account
cannot see, 404) says nothing about effGen, so it is printed as
``NOT MEASURED`` with a workflow warning and opens no issue.
* ``1`` — the provider accepted a call and effGen produced the wrong result.
Transient pressure — 429, 503, timeouts — is retried with exponential backoff
and jitter before it counts as anything.
Usage::
python scripts/ci_provider_smoke.py --provider groq --model llama-3.1-8b-instant
python scripts/ci_provider_smoke.py --provider cerebras --model gpt-oss-120b --checks stream
"""
from __future__ import annotations
import argparse
import functools
import os
import random
import sys
import time
PROVIDERS = {
"cerebras": ("CEREBRAS_API_KEY", "effgen.models.cerebras_adapter", "CerebrasAdapter"),
"groq": ("GROQ_API_KEY", "effgen.models.groq_adapter", "GroqAdapter"),
}
RETRIABLE_MARKS = (
"429", "rate limit", "rate_limit", "queue_exceeded", "too_many_requests",
"high traffic", "timeout", "timed out", "503", "service unavailable",
"maximum iterations", "empty response",
)
REFUSAL_MARKS = (
"402", "payment required", "payment_required", "billing", "insufficient",
"no credit", "quota", "401", "403", "unauthorized", "forbidden",
"invalid api key", "invalid_api_key", "authentication", "404",
"model_not_found", "does not exist",
)
#: Words that show the model agreed roses need water.
AFFIRM = ("yes", "they do", "they need", "do need", "need water")
class Refused(RuntimeError):
"""The provider would not serve the call; nothing about effGen was measured."""
def _text(exc: BaseException) -> str:
return str(exc).lower()
def is_retriable(exc: BaseException) -> bool:
return any(mark in _text(exc) for mark in RETRIABLE_MARKS)
def _refusal_types() -> tuple[type[BaseException], ...]:
try:
from effgen.models.errors import ModelAuthError, ModelNotFoundError
except ImportError: # pragma: no cover - the package is installed in CI
return ()
return (ModelAuthError, ModelNotFoundError)
def is_refusal(exc: BaseException) -> bool:
if isinstance(exc, _refusal_types()):
return True
return not is_retriable(exc) and any(mark in _text(exc) for mark in REFUSAL_MARKS)
def with_retry(label: str, fn, attempts: int, base_delay: float = 3.0, max_delay: float = 60.0):
"""Call *fn*; retry transient pressure, refuse on a refusal, raise otherwise."""
last: BaseException | None = None
for attempt in range(attempts):
try:
return fn()
except Exception as exc: # noqa: BLE001 - every kind is classified below
last = exc
if is_refusal(exc):
raise Refused(f"{label}: {exc}") from exc
retriable = is_retriable(exc)
print(f"WARN[{label}] attempt {attempt + 1}/{attempts} "
f"{'RETRIABLE' if retriable else 'FATAL'}: {exc}")
if not retriable or attempt == attempts - 1:
break
delay = random.uniform(0.5, min(max_delay, base_delay * (2 ** attempt)))
print(f" sleeping {delay:.1f}s before retry")
time.sleep(delay)
raise RuntimeError(f"{label} failed after {attempts} attempts: {last}")
def check_generate(adapter, model_id: str) -> str:
from effgen.models.base import GenerationConfig
# A reasoning model spends tokens before any visible content, so the cap
# leaves room for that pass as well as the answer.
out = adapter.generate("Reply with a single word: ALIVE",
config=GenerationConfig(temperature=0.0, max_tokens=512))
text = (out.text or "").strip()
if not text:
raise RuntimeError("empty response: generate returned no visible text")
return f"generate -> {text!r}"
def check_stream(adapter, model_id: str) -> str:
chunks = list(adapter.generate_stream("Count: 1, 2, 3."))
text = "".join(chunks)
if not text:
raise RuntimeError("empty response: stream returned no text")
return f"stream -> {len(chunks)} chunks, {len(text)} chars"
def check_tools(adapter, model_id: str) -> str:
from effgen.core.agent import Agent, AgentConfig
from effgen.tools.builtin.calculator import Calculator
config = AgentConfig(
name=f"smoke-{model_id}",
model=adapter,
tools=[Calculator()],
system_prompt="You are a math assistant. Use the calculator tool. Answer concisely.",
max_iterations=5,
temperature=0.1,
)
response = Agent(config).run("What is 12 * 12?")
text = str(response.output or "")
if not text.strip():
raise RuntimeError("empty response: the agent returned no text")
if "144" not in text:
raise RuntimeError(f"'144' missing from output: {text!r}")
return f"tool-calling -> '144' present ({getattr(response, 'tool_calls', 0) or 0} tool calls)"
def check_reasoning(adapter, model_id: str) -> str:
from effgen.models.base import GenerationConfig
result = adapter.generate(
"If all roses are flowers and all flowers need water, do roses need water? "
"Answer with just 'yes' or 'no' first, then a one-sentence explanation.",
config=GenerationConfig(temperature=0.0, max_tokens=2048),
)
text = (result.text or "").strip()
if not text:
raise RuntimeError("empty response: no visible content "
f"(finish_reason={getattr(result, 'finish_reason', '?')})")
if not any(word in text.lower() for word in AFFIRM):
raise RuntimeError(f"unexpected text: {text!r}")
return f"reasoning -> {text[:80]!r}"
CHECKS = {
"generate": check_generate,
"stream": check_stream,
"tools": check_tools,
"reasoning": check_reasoning,
}
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("--provider", choices=sorted(PROVIDERS), required=True)
parser.add_argument("--model", required=True, help="model id on that provider")
parser.add_argument("--checks", default=",".join(CHECKS),
help="comma-separated subset of: " + ", ".join(CHECKS))
parser.add_argument("--max-attempts", type=int, default=5)
args = parser.parse_args(argv)
key_var, module_name, class_name = PROVIDERS[args.provider]
if not os.getenv(key_var, ""):
print(f"SKIPPED: {key_var} not set for {args.provider}/{args.model}")
return 0
names = [name.strip() for name in args.checks.split(",") if name.strip()]
unknown = [name for name in names if name not in CHECKS]
if unknown:
parser.error(f"unknown check(s): {', '.join(unknown)}. Choose from: {', '.join(CHECKS)}")
import importlib
adapter_cls = getattr(importlib.import_module(module_name), class_name)
label = f"{args.provider}/{args.model}"
try:
# A model id the account cannot see is refused at construction, before
# any call is made; that is a refusal like any other.
adapter = adapter_cls(args.model, enable_rate_limiting=False)
adapter.load()
except Exception as exc: # noqa: BLE001 - classified below
if is_refusal(exc):
print(f"NOT MEASURED[{label}]: the provider refused to serve, so nothing "
f"about effGen was checked. {exc}")
print(f"::warning title=Provider smoke not measured::{label} refused: {str(exc)[:300]}")
return 0
print(f"FAIL[{label}]: could not load the adapter: {exc}")
return 1
try:
for name in names:
check = CHECKS[name]
try:
print(f"OK[{label}]: "
f"{with_retry(name, functools.partial(check, adapter, args.model), args.max_attempts)}")
except Refused as refused:
print(f"NOT MEASURED[{label}]: the provider refused to serve, so nothing "
f"about effGen was checked. {refused}")
print(f"::warning title=Provider smoke not measured::{label} refused: "
f"{str(refused)[:300]}")
return 0
except Exception as exc: # noqa: BLE001 - the verdict is the exit code
print(f"FAIL[{label}]: {exc}")
return 1
finally:
try:
adapter.unload()
except Exception: # noqa: BLE001 - nothing left to report
pass
print(f"All checks passed for {label}: {', '.join(names)}")
return 0
if __name__ == "__main__":
sys.exit(main())

Xet Storage Details

Size:
9.01 kB
·
Xet hash:
4b9a8085d03a6cce70a0dfdb54762990257aebae72d92809fed7d06e0864bb07

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.