Buckets:
AnonymousSpottedToad/human_verified_bucket / codebase /python /ctrl-gaurav_effGen /effgen /cli /battle.py
| """Terminal view for ``effgen battle`` — several models racing one prompt. | |
| On an interactive terminal each contender gets a column that fills in as its | |
| answer streams, headed by the model id and footed by its live token count, | |
| elapsed time and cost, ending in a verdict panel. | |
| When stdout is not a terminal, or ``--json``/``--no-animation``/``NO_COLOR`` | |
| apply, no live frame is drawn: the battle runs to completion and prints one | |
| structured result — the JSON document, or a Markdown report carrying each | |
| model's full answer, the tally and the verdict. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import sys | |
| import threading | |
| from typing import Any | |
| from effgen.eval.battle import BattleResult, Contender, fmt_cost, run_battle | |
| from effgen.ui.render import json_ensure_ascii | |
| from effgen.ui.tables import console_is_interactive | |
| from effgen.ui.theme import color_enabled, get_console | |
| #: Rows a column borrows for its border, metrics and spacing. The rest of the | |
| #: panel height is answer text. | |
| _COLUMN_CHROME_ROWS = 8 | |
| #: Bounds on how tall a column grows, so the race fits a short terminal and does | |
| #: not sprawl on a tall one. | |
| _MIN_COLUMN_HEIGHT = 12 | |
| _MAX_COLUMN_HEIGHT = 28 | |
| def _state_label(contender: Contender) -> str: | |
| """Describe what a contender is doing, so a slow column is never ambiguous. | |
| A model being loaded reads as loading rather than as one that has stalled — | |
| a local model can spend seconds there before its first token. | |
| """ | |
| if contender.state == "loading": | |
| return "loading model…" | |
| if contender.state == "streaming": | |
| return "generating…" if contender.answer else "waiting for first token…" | |
| if contender.state == "failed": | |
| return "failed" | |
| if contender.state == "done": | |
| return "done" | |
| return "queued" | |
| def _footer(contender: Contender) -> str: | |
| """One line of live measurements for a column footer.""" | |
| parts: list[str] = [] | |
| if contender.load_s is not None: | |
| parts.append(f"load {contender.load_s:.1f}s") | |
| if contender.ttft_s is not None: | |
| parts.append(f"ttft {contender.ttft_s:.2f}s") | |
| if contender.latency_s is not None: | |
| parts.append(f"total {contender.latency_s:.2f}s") | |
| if contender.total_tokens is not None: | |
| tokens = f"{contender.total_tokens} tok" | |
| if contender.estimated_tokens: | |
| tokens += " (local count)" | |
| parts.append(tokens) | |
| if contender.state == "done": | |
| parts.append(fmt_cost(contender.cost_usd)) | |
| elif not parts: | |
| parts.append(_state_label(contender)) | |
| return " · ".join(parts) | |
| def _column(contender: Contender, *, width: int, height: int) -> Any: | |
| """Render one contender as a bordered panel. | |
| The measurements sit at the top of the panel, where a narrow column wraps | |
| them instead of cutting them off. An answer longer than the column shows its | |
| most recent text, so a column that is still writing keeps moving; the whole | |
| answer is in the result the command prints or saves. | |
| """ | |
| from rich.console import Group | |
| from rich.panel import Panel | |
| from rich.text import Text | |
| if contender.error: | |
| answer: Any = Text(contender.error, style="red") | |
| elif contender.answer: | |
| budget = max(width, 1) * max(height - _COLUMN_CHROME_ROWS, 1) | |
| answer = Text(contender.answer[-budget:]) | |
| else: | |
| answer = Text(_state_label(contender), style="dim") | |
| border = { | |
| "failed": "red", | |
| "done": "green", | |
| "streaming": "cyan", | |
| }.get(contender.state, "dim") | |
| return Panel( | |
| Group(Text(_footer(contender), style="dim"), Text(""), answer), | |
| title=contender.model, | |
| subtitle=_state_label(contender), | |
| border_style=border, | |
| width=width, | |
| height=height, | |
| ) | |
| def _live_view(contenders: list[Contender], console: Any) -> Any: | |
| """Lay the contenders out side by side, sized to the terminal.""" | |
| from rich.columns import Columns | |
| total = max(console.width, 40) | |
| width = max(24, (total - 2 * len(contenders)) // max(1, len(contenders))) | |
| # Leave room under the columns for the verdict panel that follows them. | |
| height = min(_MAX_COLUMN_HEIGHT, max(_MIN_COLUMN_HEIGHT, console.height - 10)) | |
| return Columns( | |
| [_column(c, width=width, height=height) for c in contenders], | |
| equal=True, | |
| expand=True, | |
| ) | |
| def render_verdict(result: BattleResult, console: Any) -> None: | |
| """Print the measured outcomes, and any judged pick, after the race.""" | |
| from rich.panel import Panel | |
| from rich.table import Table | |
| table = Table(box=None, show_header=False, pad_edge=False) | |
| table.add_column(style="bold") | |
| table.add_column() | |
| for key in ("fastest", "cheapest", "longest"): | |
| entry = result.verdict.get(key) | |
| if entry: | |
| table.add_row(key.capitalize(), f"{entry['model']} — {entry['detail']}") | |
| judge = result.verdict.get("judge") | |
| if judge: | |
| if judge.get("winner"): | |
| reason = judge.get("reasoning") or "" | |
| table.add_row( | |
| "Judged pick", | |
| f"{judge['winner']} — {reason} (judged by {judge['judge_model']})", | |
| ) | |
| elif judge.get("error"): | |
| table.add_row("Judged pick", f"unavailable — {judge['error']}") | |
| failed = [c for c in result.contenders if not c.ok] | |
| if failed: | |
| table.add_row( | |
| "Did not answer", | |
| ", ".join(f"{c.model} ({c.error})" for c in failed), | |
| ) | |
| spent = fmt_cost(result.total_cost()) if result.finishers else "—" | |
| table.add_row( | |
| "Race", | |
| f"{len(result.finishers)}/{len(result.contenders)} answered in " | |
| f"{result.wall_s:.2f}s · total {spent}", | |
| ) | |
| console.print(Panel(table, title="Verdict", border_style="cyan")) | |
| class _Columns: | |
| """Holds each contender in the column position its model was named in. | |
| A contender reports itself as it changes; the columns stay in the order the | |
| models were given rather than the order they happen to report in. Each | |
| contender keeps a column of its own, so entering one model id twice shows | |
| two independent racers instead of two views of the same one. | |
| """ | |
| def __init__(self, models: list[str]) -> None: | |
| self._slots: list[Contender] = [Contender(model=m) for m in models] | |
| self._assigned: dict[int, int] = {} | |
| def update(self, contender: Contender) -> None: | |
| slot = self._assigned.get(id(contender)) | |
| if slot is None: | |
| taken = set(self._assigned.values()) | |
| slot = next( | |
| (i for i, c in enumerate(self._slots) | |
| if c.model == contender.model and i not in taken), | |
| None, | |
| ) | |
| if slot is None: | |
| return | |
| self._assigned[id(contender)] = slot | |
| self._slots[slot] = contender | |
| def current(self) -> list[Contender]: | |
| return list(self._slots) | |
| def _run_live( | |
| prompt: str, | |
| models: list[str], | |
| *, | |
| console: Any, | |
| temperature: float | None, | |
| max_tokens: int | None, | |
| system_prompt: str | None, | |
| judge: str | None, | |
| ) -> BattleResult: | |
| """Race the models with a live side-by-side view, then print the verdict.""" | |
| from rich.live import Live | |
| columns = _Columns(models) | |
| pending = threading.Event() | |
| def _on_update(contender: Contender) -> None: | |
| columns.update(contender) | |
| pending.set() | |
| with Live(console=console, auto_refresh=False, transient=False) as live: | |
| holder: dict[str, Any] = {} | |
| def _race() -> None: | |
| try: | |
| holder["result"] = run_battle( | |
| prompt, | |
| models, | |
| temperature=temperature, | |
| max_tokens=max_tokens, | |
| system_prompt=system_prompt, | |
| judge=judge, | |
| on_update=_on_update, | |
| ) | |
| except BaseException as exc: # noqa: BLE001 - re-raised on the main thread | |
| holder["error"] = exc | |
| finally: | |
| pending.set() | |
| def _draw() -> None: | |
| live.update(_live_view(columns.current(), console), refresh=True) | |
| # The race owns the worker threads; this thread owns the screen, so a | |
| # redraw never happens from inside a contender's callback. | |
| racer = threading.Thread(target=_race, name="battle-race", daemon=True) | |
| racer.start() | |
| _draw() | |
| while racer.is_alive(): | |
| if pending.wait(timeout=0.1): | |
| pending.clear() | |
| _draw() | |
| racer.join() | |
| _draw() | |
| if "error" in holder: | |
| raise holder["error"] | |
| result = holder["result"] | |
| render_verdict(result, console) | |
| return result | |
| def run_battle_command(args: Any) -> int: | |
| """Entry point for ``effgen battle``.""" | |
| from effgen.cli import progress as _progress | |
| prompt = (getattr(args, "prompt", "") or "").strip() | |
| if not prompt: | |
| print("A battle needs a prompt: effgen battle \"<prompt>\" -m a,b", file=sys.stderr) | |
| return 2 | |
| models = [m.strip() for m in (getattr(args, "models", "") or "").split(",") if m.strip()] | |
| if len(models) < 2: | |
| print( | |
| "A battle needs at least two models to compare " | |
| "(-m openai:gpt-5-nano,groq:llama-3.1-8b-instant). " | |
| "Use `effgen models list` to see available ids.", | |
| file=sys.stderr, | |
| ) | |
| return 2 | |
| temperature = getattr(args, "temperature", None) | |
| max_tokens = getattr(args, "max_tokens", None) | |
| system_prompt = getattr(args, "system_prompt", None) | |
| judge = getattr(args, "judge", None) | |
| json_mode = bool(getattr(args, "output_json", False)) | |
| console = get_console(theme_name=getattr(args, "theme", None)) | |
| animated = _progress.animation_enabled( | |
| quiet=getattr(args, "quiet", False), | |
| no_animation=getattr(args, "no_animation", False), | |
| ) | |
| live_capable = ( | |
| not json_mode | |
| and animated | |
| and color_enabled() | |
| and console_is_interactive(console) | |
| ) | |
| try: | |
| if live_capable: | |
| result = _run_live( | |
| prompt, | |
| models, | |
| console=console, | |
| temperature=temperature, | |
| max_tokens=max_tokens, | |
| system_prompt=system_prompt, | |
| judge=judge, | |
| ) | |
| else: | |
| result = run_battle( | |
| prompt, | |
| models, | |
| temperature=temperature, | |
| max_tokens=max_tokens, | |
| system_prompt=system_prompt, | |
| judge=judge, | |
| ) | |
| except KeyboardInterrupt: | |
| print("Battle interrupted.", file=sys.stderr) | |
| return 130 | |
| except ValueError as exc: | |
| print(str(exc), file=sys.stderr) | |
| return 2 | |
| if json_mode: | |
| print(json.dumps(result.to_dict(), indent=2, ensure_ascii=json_ensure_ascii())) | |
| elif not live_capable: | |
| print(result.to_markdown()) | |
| output = getattr(args, "output", None) | |
| if output: | |
| _write_battle_artifact(output, result) | |
| report = getattr(args, "report", None) | |
| if report: | |
| _write_battle_report(report, result) | |
| # A battle reports a comparison; it exits non-zero only when no contender | |
| # answered, which means there is nothing to compare. | |
| return 0 if result.finishers else 1 | |
| def _write_battle_artifact(path: str, result: BattleResult) -> None: | |
| """Save the battle as JSON or Markdown, chosen by the file extension.""" | |
| from pathlib import Path | |
| target = Path(path) | |
| text = result.to_markdown() if target.suffix.lower() in (".md", ".markdown") else result.to_json() | |
| target.parent.mkdir(parents=True, exist_ok=True) | |
| target.write_text(text, encoding="utf-8") | |
| print(f"Saved battle to {target}", file=sys.stderr) | |
| def _write_battle_report(path: str, result: BattleResult) -> None: | |
| """Save the battle as a self-contained HTML report.""" | |
| from effgen.ui.report_html import write_html_report | |
| write_html_report(path, result.to_dict(), kind="battle") | |
| print(f"Wrote HTML report to {path}", file=sys.stderr) | |
Xet Storage Details
- Size:
- 12.2 kB
- Xet hash:
- 065f21e796c96941315cb4c37eae63b8670878be8af0959bd2a35424bba493a9
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.