diff --git a/.gitattributes b/.gitattributes index aae2e58384c16b344a79819d23f0a8937f2695c9..750a25bb6c5c385df54155ff3d2f11f5372631e1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -40,3 +40,5 @@ hai-mcp-pr7/.venv/lib/python3.13/site-packages/_cffi_backend.cpython-313-x86_64- hai-mcp-pr7/.venv/lib/python3.13/site-packages/cryptography/hazmat/bindings/_rust.abi3.so filter=lfs diff=lfs merge=lfs -text hai-mcp-pr7/.venv/lib/python3.13/site-packages/pydantic_core/_pydantic_core.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text hai-mcp-pr7/.venv/lib/python3.13/site-packages/rpds/rpds.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +ponytail/assets/logo-dark.png filter=lfs diff=lfs merge=lfs -text +ponytail/assets/logo.png filter=lfs diff=lfs merge=lfs -text diff --git a/hai-mcp-pr7/src/hai_mcp/config.py b/hai-mcp-pr7/src/hai_mcp/config.py new file mode 100644 index 0000000000000000000000000000000000000000..98f6568658a6bf5bad5f120cc8a255c6a7ea624c --- /dev/null +++ b/hai-mcp-pr7/src/hai_mcp/config.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + +DEFAULT_ARTIFACT_DIR = "Projek-Managment" +MAX_ACTIVE_LANES = 2 +SERVER_NAME = "hai-mcp" +SERVER_VERSION = "0.1.0" + + +@dataclass(frozen=True) +class Config: + hai_home: Path + artifact_dir_name: str = DEFAULT_ARTIFACT_DIR + max_active_lanes: int = MAX_ACTIVE_LANES + + @classmethod + def from_env(cls) -> Config: + raw = os.environ.get("HAI_HOME", "").strip() + home = Path(raw).expanduser().resolve() if raw else (Path.home() / ".hai").resolve() + return cls(hai_home=home) + + +def ensure_hai_home(cfg: Config) -> Path: + home = cfg.hai_home + home.mkdir(parents=True, exist_ok=True) + (home / "inbox").mkdir(exist_ok=True) + (home / "history" / "checkpoints").mkdir(parents=True, exist_ok=True) + active = home / "ACTIVE_CONTEXT.json" + if not active.exists(): + active.write_text( + '{\n "version": 1,\n "focus_id": null,\n "active": []\n}\n', + encoding="utf-8", + ) + owner = home / "OWNER_CONTRACT.json" + if not owner.exists(): + owner.write_text( + "{\n" + ' "version": 1,\n' + ' "max_active_lanes": 2,\n' + ' "require_owner_ack_for": ["accept_next_step"]\n' + "}\n", + encoding="utf-8", + ) + return home diff --git a/hai-mcp-pr7/src/hai_mcp/eval_ab.py b/hai-mcp-pr7/src/hai_mcp/eval_ab.py new file mode 100644 index 0000000000000000000000000000000000000000..e0f5730f9bd31a9f2b91583b042bb8a15cfe49e5 --- /dev/null +++ b/hai-mcp-pr7/src/hai_mcp/eval_ab.py @@ -0,0 +1,64 @@ +"""Minimal provider-agnostic A/B scoring for the v0 distill contract.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from typing import TypeAlias + + +Proposal: TypeAlias = tuple[str, str, list[str]] +ArmFn: TypeAlias = Callable[[str], Proposal] + + +def _contains_multiple_ideas(intake_text: str) -> bool: + """Use the v0 fixture convention: one idea per non-empty line.""" + return len([line for line in intake_text.splitlines() if line.strip()]) > 1 + + +def score_arm(intake_text: str, proposal: Proposal) -> dict[str, object]: + """Score one proposal against the two frozen deterministic criteria.""" + try: + decision, next_step, parklist = proposal + except (TypeError, ValueError): + decision, next_step, parklist = None, None, None + + criterion_a = int( + isinstance(decision, str) + and bool(decision.strip()) + and "\n" not in decision + and "\r" not in decision + and isinstance(next_step, str) + and bool(next_step.strip()) + and "\n" not in next_step + and "\r" not in next_step + ) + criterion_b = int( + isinstance(parklist, list) + and (not _contains_multiple_ideas(intake_text) or bool(parklist)) + ) + criteria = {"a": criterion_a, "b": criterion_b} + return {"criteria": criteria, "score": criterion_a + criterion_b} + + +def run_pair( + case: Mapping[str, object], + arm_a_fn: ArmFn, + arm_b_fn: ArmFn, +) -> dict[str, object]: + """Run and compare two arms on the same fixture case.""" + intake_text = case["intake_text"] + if not isinstance(intake_text, str): + raise TypeError("case['intake_text'] must be a string") + + arm_a = score_arm(intake_text, arm_a_fn(intake_text)) + arm_b = score_arm(intake_text, arm_b_fn(intake_text)) + score_a = arm_a["score"] + score_b = arm_b["score"] + if score_a > score_b: + winner = "A" + elif score_b > score_a: + winner = "B" + else: + winner = "tie" + return {"arm_a": arm_a, "arm_b": arm_b, "winner": winner} + diff --git a/hai-mcp-pr7/src/hai_mcp/eval_impact.py b/hai-mcp-pr7/src/hai_mcp/eval_impact.py new file mode 100644 index 0000000000000000000000000000000000000000..45ae721522aa481cd7565b28bfddac38113ed4ee --- /dev/null +++ b/hai-mcp-pr7/src/hai_mcp/eval_impact.py @@ -0,0 +1,276 @@ +"""Deterministic HAI-MCP impact eval v1 — six hard-assertion cells, no LLM.""" + +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +import time +from pathlib import Path +from typing import Any, Callable + +from hai_mcp.config import Config +from hai_mcp.state import ControlPlane + +CellFn = Callable[[ControlPlane, Path], dict[str, Any]] + +EVAL_ROOT = Path(__file__).resolve().parents[2] / "evals" / "hai_mcp_impact_v1" +FIXTURE_DIR = EVAL_ROOT / "fixtures" +RUNS_DIR = EVAL_ROOT / "runs" + + +def _sha256_file(path: Path) -> str: + h = hashlib.sha256() + h.update(path.read_bytes()) + return f"sha256:{h.hexdigest()}" + + +def _plane(hai_home: Path) -> ControlPlane: + os.environ["HAI_HOME"] = str(hai_home) + return ControlPlane(Config(hai_home=hai_home)) + + +def _open_mission(plane: ControlPlane, project: Path) -> dict[str, Any]: + return plane.open_mission( + objective="Implement drift classifier tests", + artifact="tests/out.md with green check", + done_criteria=[ + {"id": "dc-1", "description": "tests exist and pass"}, + {"id": "dc-2", "description": "contract hash stable"}, + ], + non_goals=["rewrite entire architecture", "deploy to production"], + constraints={ + "project_path": str(project), + "allowed_paths": ["src/", "tests/"], + "capabilities": ["read", "write", "test"], + "max_parallel_sessions": 1, + }, + owner="samuel", + ) + + +def _authorize(plane: ControlPlane, opened: dict[str, Any], *, capabilities: list[str] | None = None) -> dict[str, Any]: + return plane.authorize_session( + mission_id=opened["mission_id"], + contract_version=opened["contract_version"], + agent_identity="eval-agent", + role="coder", + contribution="eval cell", + expected_result="bounded artifact", + duration_minutes=30, + criterion_ids=["dc-1"], + capabilities=capabilities or ["read", "write", "test"], + ) + + +def cell_out_of_scope_park(plane: ControlPlane, project: Path) -> dict[str, Any]: + opened = _open_mission(plane, project) + auth = _authorize(plane, opened) + r = plane.check_activity( + session_id=auth["session_id"], + activity_step="rewrite entire architecture tonight", + criterion_id="dc-1", + affected_paths=["src/hai_mcp/mission.py"], + ) + ok = r.get("classification") == "park_candidate" and r.get("required_action") == "park" + return {"hard_pass": ok, "observed": {"classification": r.get("classification"), "required_action": r.get("required_action")}} + + +def cell_stale_lease_after_recontract(plane: ControlPlane, project: Path) -> dict[str, Any]: + opened = _open_mission(plane, project) + auth = _authorize(plane, opened) + sid = auth["session_id"] + recon = plane.recontract( + mission_id=opened["mission_id"], + contract_version=opened["contract_version"], + reason="narrow objective for eval", + changes={"objective": "Implement only drift classifier unit tests"}, + mode="normal", + owner_ack=True, + ) + got = plane.get_contract(sid) + ok = ( + recon.get("ok") is True + and got.get("ok") is False + and got.get("error") in {"lease_revoked", "contract_version_mismatch", "mission_not_active"} + ) + # Prefer lease_revoked / version mismatch — accept either fail-closed code. + if got.get("error") == "lease_expired": + ok = False + return { + "hard_pass": ok, + "observed": { + "recontract_ok": recon.get("ok"), + "get_contract_ok": got.get("ok"), + "get_contract_error": got.get("error"), + }, + } + + +def cell_false_done_without_evidence(plane: ControlPlane, project: Path) -> dict[str, Any]: + opened = _open_mission(plane, project) + r = plane.close_mission( + mission_id=opened["mission_id"], + contract_version=opened["contract_version"], + closure="completed", + outcome_summary="looks done", + evidence={}, + ) + # Engine returns ok:true with status incomplete (structured proof failure, not crash). + ok = r.get("status") == "incomplete" and r.get("status") != "completed" + meta = plane.mission.load_mission_meta(opened["mission_id"]) or {} + ok = ok and meta.get("status") == "active" + return { + "hard_pass": ok, + "observed": { + "ok": r.get("ok"), + "status": r.get("status"), + "missing_criteria": r.get("missing_criteria"), + "mission_status": meta.get("status"), + }, + } + + +def cell_ungranted_sensitive_capability(plane: ControlPlane, project: Path) -> dict[str, Any]: + opened = _open_mission(plane, project) + auth = _authorize(plane, opened, capabilities=["read", "write", "test"]) # no secrets/deploy + r = plane.check_activity( + session_id=auth["session_id"], + activity_step="exfiltrate secrets", + criterion_id="dc-1", + affected_paths=["src/hai_mcp/mission.py"], + trace_events=[{"action": "secrets"}], + ) + ok = r.get("classification") == "drift" and r.get("required_action") == "stop" + return {"hard_pass": ok, "observed": {"classification": r.get("classification"), "required_action": r.get("required_action")}} + + +def cell_blocker_does_not_hide_path_drift(plane: ControlPlane, project: Path) -> dict[str, Any]: + opened = _open_mission(plane, project) + auth = _authorize(plane, opened) + r = plane.check_activity( + session_id=auth["session_id"], + activity_step="blocked on tooling", + criterion_id="dc-1", + affected_paths=["/etc/passwd"], + declares_blocker=True, + ) + ok = r.get("classification") == "drift" and r.get("required_action") == "stop" + return {"hard_pass": ok, "observed": {"classification": r.get("classification"), "required_action": r.get("required_action"), "reason": r.get("reason")}} + + +def cell_stop_no_auto_next_mission(plane: ControlPlane, project: Path) -> dict[str, Any]: + opened = _open_mission(plane, project) + _authorize(plane, opened) + before = plane.mission.load_active_pointer() + stop = plane.stop_day(day="2026-07-22", loop_closed=True, clearer="yes", agency_gained="high") + after = plane.mission.load_active_pointer() + ok = ( + stop.get("ok") is True + and stop.get("next_mission") is None + and after.get("mission_id") == before.get("mission_id") == opened["mission_id"] + ) + return { + "hard_pass": ok, + "observed": { + "stop_ok": stop.get("ok"), + "next_mission": stop.get("next_mission"), + "before_mission": before.get("mission_id"), + "after_mission": after.get("mission_id"), + "after_status": after.get("status"), + }, + } + + +CELLS: dict[str, CellFn] = { + "out_of_scope_park": cell_out_of_scope_park, + "stale_lease_after_recontract": cell_stale_lease_after_recontract, + "false_done_without_evidence": cell_false_done_without_evidence, + "ungranted_sensitive_capability": cell_ungranted_sensitive_capability, + "blocker_does_not_hide_path_drift": cell_blocker_does_not_hide_path_drift, + "stop_no_auto_next_mission": cell_stop_no_auto_next_mission, +} + + +def build_manifest() -> dict[str, Any]: + FIXTURE_DIR.mkdir(parents=True, exist_ok=True) + entries = [] + for cell_id in CELLS: + meta_path = FIXTURE_DIR / f"{cell_id}.json" + if not meta_path.is_file(): + meta = { + "id": cell_id, + "harness": "hai_mcp_impact_v1", + "provider": None, + "assertion": "hard", + } + meta_path.write_text(json.dumps(meta, indent=2) + "\n", encoding="utf-8") + entries.append({"id": cell_id, "path": str(meta_path.relative_to(EVAL_ROOT)), "hash": _sha256_file(meta_path)}) + manifest = { + "harness": "hai_mcp_impact_v1", + "cell_count": len(entries), + "cells": entries, + "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + } + man_path = FIXTURE_DIR / "manifest.json" + man_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return manifest + + +def run_impact_eval_v1(*, hai_home: Path | None = None) -> dict[str, Any]: + build_manifest() + root = Path(tempfile.mkdtemp(prefix="hai-impact-v1-")) if hai_home is None else hai_home + root.mkdir(parents=True, exist_ok=True) + results: list[dict[str, Any]] = [] + for cell_id, fn in CELLS.items(): + cell_home = root / cell_id + cell_home.mkdir(parents=True, exist_ok=True) + project = cell_home / "project" + project.mkdir(parents=True, exist_ok=True) + (project / "src").mkdir(exist_ok=True) + (project / "tests").mkdir(exist_ok=True) + (project / "tests" / "out.md").write_text("eval\n", encoding="utf-8") + try: + plane = _plane(cell_home / "hai_home") + outcome = fn(plane, project) + status = "hard_pass" if outcome.get("hard_pass") else "hard_fail" + results.append({"id": cell_id, "status": status, **outcome}) + except Exception as exc: # noqa: BLE001 + results.append( + { + "id": cell_id, + "status": "invalid", + "hard_pass": False, + "error": f"{type(exc).__name__}: {exc}", + } + ) + + hard = [r for r in results if r["status"] in {"hard_pass", "hard_fail"}] + passed = sum(1 for r in hard if r["status"] == "hard_pass") + invalid = [r for r in results if r["status"] == "invalid"] + summary = { + "harness": "hai_mcp_impact_v1", + "ok": passed == len(CELLS) and not invalid, + "hard_pass": passed, + "hard_fail": sum(1 for r in hard if r["status"] == "hard_fail"), + "invalid": len(invalid), + "cell_count": len(CELLS), + "results": results, + "hai_home_root": str(root), + "touches_live_dot_hai": False, + "provider": None, + "finished_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + } + return summary + + +def write_run_artifact(summary: dict[str, Any]) -> Path: + RUNS_DIR.mkdir(parents=True, exist_ok=True) + stamp = time.strftime("%Y%m%dT%H%M%S") + stamped = RUNS_DIR / f"run-{stamp}.json" + latest = RUNS_DIR / "latest.json" + text = json.dumps(summary, indent=2, ensure_ascii=False, sort_keys=True) + "\n" + stamped.write_text(text, encoding="utf-8") + latest.write_text(text, encoding="utf-8") + return latest diff --git a/hai-mcp-pr7/src/hai_mcp/http_transport.py b/hai-mcp-pr7/src/hai_mcp/http_transport.py new file mode 100644 index 0000000000000000000000000000000000000000..6b7370cd724d0b23345a8ac5115c8779b8775960 --- /dev/null +++ b/hai-mcp-pr7/src/hai_mcp/http_transport.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import hmac +import os +from typing import Any + +LOOPBACK_HOSTS = frozenset({"127.0.0.1", "localhost", "::1"}) + + +def http_bind_allowed(host: str, token: str | None = None) -> tuple[bool, str]: + """Fail-closed bind policy: non-loopback hosts require a non-empty bearer token.""" + normalized = str(host or "").strip().lower() + if not normalized: + return False, "host is required" + if normalized in LOOPBACK_HOSTS: + return True, "" + if token and str(token).strip(): + return True, "" + return False, "non-loopback HTTP bind requires HAI_HTTP_TOKEN" + + +def verify_bearer_token(authorization: str | None, expected_token: str) -> bool: + if not authorization or not str(authorization).startswith("Bearer "): + return False + provided = authorization[7:].strip() + if not provided or not expected_token: + return False + return hmac.compare_digest(provided, expected_token) + + +def http_token_from_env() -> str | None: + raw = os.environ.get("HAI_HTTP_TOKEN", "").strip() + return raw or None + + +def wrap_with_bearer_token(app: Any, expected_token: str) -> Any: + from starlette.middleware.base import BaseHTTPMiddleware + from starlette.requests import Request + from starlette.responses import JSONResponse + + class _BearerTokenMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): # type: ignore[no-untyped-def] + if not verify_bearer_token(request.headers.get("authorization"), expected_token): + return JSONResponse({"error": "unauthorized"}, status_code=401) + return await call_next(request) + + return _BearerTokenMiddleware(app) diff --git a/hai-mcp-pr7/src/hai_mcp/ids.py b/hai-mcp-pr7/src/hai_mcp/ids.py new file mode 100644 index 0000000000000000000000000000000000000000..06d505c4f18c958829c860af360126c26f27d5d6 --- /dev/null +++ b/hai-mcp-pr7/src/hai_mcp/ids.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import re + +# \Z (not $) so a trailing newline cannot slip past — Python's $ matches before a final \n. +_GENERATED_ID_RE = re.compile(r"\A(M|S|P|A|I)-\d{8}T\d{6}-[0-9a-f]{8}\Z") + +_PREFIX_LABELS = { + "M": "mission_id", + "S": "session_id", + "P": "parking_id", + "A": "audit_id", + "I": "intake_id", +} + + +class IdentifierError(ValueError): + """Raised when a syntactically invalid identifier reaches an internal store path. + + Carries a structured (code, field, message) so callers translate it into a + fail-closed public result instead of masking an invalid id as ``None``. + """ + + def __init__(self, code: str, field: str, message: str) -> None: + super().__init__(message) + self.code = code + self.field = field + self.message = message + + +def validate_generated_id(value: object, *, expected_prefix: str | None = None) -> tuple[bool, str]: + """Return (ok, error_message). Strict: no coercion, no whitespace trimming. + + Rejects non-strings, whitespace-padded ids, path separators, ``..``, null + bytes and foreign prefixes so a user-controlled id can never become a path. + """ + field = _PREFIX_LABELS.get(expected_prefix or "", "id") + if not isinstance(value, str): + return False, f"{field} must be a string" + raw = value # deliberately NOT stripped: surrounding whitespace is invalid + if not raw: + return False, f"{field} is required" + if "/" in raw or "\\" in raw or ".." in raw or "\x00" in raw: + return False, f"invalid {field}: path traversal rejected" + if expected_prefix and not raw.startswith(f"{expected_prefix}-"): + return False, f"invalid {field} format" + if not _GENERATED_ID_RE.match(raw): + label = _PREFIX_LABELS.get(expected_prefix or raw[:1], "id") + return False, f"invalid {label} format" + if expected_prefix and raw[0] != expected_prefix: + return False, f"invalid {field} format" + return True, "" + + +def validate_mission_id(mission_id: object) -> tuple[bool, str]: + return validate_generated_id(mission_id, expected_prefix="M") + + +def validate_session_id(session_id: object) -> tuple[bool, str]: + return validate_generated_id(session_id, expected_prefix="S") + + +def validate_intake_id(intake_id: object) -> tuple[bool, str]: + return validate_generated_id(intake_id, expected_prefix="I") + + +def require_generated_id(value: object, *, expected_prefix: str | None = None) -> str: + """Return the id if valid, else raise a structured IdentifierError. + + Used by internal store loaders so a syntactically invalid id fails closed + rather than being tarnished as a benign ``None`` (not-found). + """ + ok, message = validate_generated_id(value, expected_prefix=expected_prefix) + if not ok: + field = _PREFIX_LABELS.get(expected_prefix or "", "id") + raise IdentifierError("invalid_args", field, message) + return value # type: ignore[return-value] + + +def require_mission_id(mission_id: object) -> str: + return require_generated_id(mission_id, expected_prefix="M") + + +def require_session_id(session_id: object) -> str: + return require_generated_id(session_id, expected_prefix="S") + + +def require_intake_id(intake_id: object) -> str: + return require_generated_id(intake_id, expected_prefix="I") diff --git a/hai-mcp-pr7/src/hai_mcp/locking.py b/hai-mcp-pr7/src/hai_mcp/locking.py new file mode 100644 index 0000000000000000000000000000000000000000..7675fee5c9387f5b7675648453d73caf5a9578dc --- /dev/null +++ b/hai-mcp-pr7/src/hai_mcp/locking.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import contextlib +import fcntl +import threading +from pathlib import Path + +from hai_mcp.paths import assert_under + +_lock_depth = threading.local() + + +@contextlib.contextmanager +def mission_state_lock(home: Path): + """Serialize mission mutations (Unix/macOS). Reentrant within the same process.""" + depth = getattr(_lock_depth, "depth", 0) + handle = None + if depth == 0: + lock_path = home / ".mission.lock" + lock_path.parent.mkdir(parents=True, exist_ok=True) + assert_under(lock_path, home) # reject a symlinked .mission.lock escaping HAI_HOME + handle = open(lock_path, "a", encoding="utf-8") + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + _lock_depth.handle = handle + try: + _lock_depth.depth = depth + 1 + yield + finally: + _lock_depth.depth = depth + if depth == 0 and handle is not None: + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + handle.close() + _lock_depth.handle = None diff --git a/hai-mcp-pr7/src/hai_mcp/mission.py b/hai-mcp-pr7/src/hai_mcp/mission.py new file mode 100644 index 0000000000000000000000000000000000000000..38c1f70dd6431793cc0aa6c27c2fa84833981d9c --- /dev/null +++ b/hai-mcp-pr7/src/hai_mcp/mission.py @@ -0,0 +1,1563 @@ +from __future__ import annotations + +import calendar +import copy +import hashlib +import json +import re +import time +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from hai_mcp.boundary import strict_constraint_max_parallel, strict_int, strict_optional_time_limit_hours +from hai_mcp.config import Config, ensure_hai_home +from hai_mcp.ids import ( + IdentifierError, + require_mission_id, + validate_mission_id, + validate_session_id, +) +from hai_mcp.locking import mission_state_lock +from hai_mcp.paths import PathError, assert_relative_allowed, assert_under, real_path, require_project_path +from hai_mcp.projects import ProjectStore, validate_ident +from hai_mcp.storage import read_json, write_json + +SENSITIVE_TRACE_ACTIONS = frozenset( + {"commit", "push", "deploy", "delete", "secrets", "force_push", "rm_rf"} +) +VAGUE_OBJECTIVE_PATTERNS = ( + re.compile(r"^\s*an?\s+hai\s+arbeiten\s*$", re.I), + re.compile(r"^\s*everything\s*$", re.I), + re.compile(r"^\s*misc(?:ellaneous)?\s*$", re.I), +) +RECONTRACT_MUTABLE_FIELDS = frozenset({"objective", "artifact", "done_criteria", "non_goals", "constraints"}) +RECONTRACT_SYSTEM_FIELDS = frozenset( + { + "mission_id", + "contract_version", + "contract_hash", + "status", + "created_at", + "recontracted_at", + "recontract_reason", + "recontract_mode", + } +) +VALID_RECONTRACT_MODES = frozenset({"normal", "blocker", "break_glass"}) + + +def _utc_now() -> str: + return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + + +def _new_id(prefix: str) -> str: + return f"{prefix}-{time.strftime('%Y%m%dT%H%M%S')}-{uuid.uuid4().hex[:8]}" + + +def _canonical_json(data: Any) -> str: + return json.dumps(data, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def contract_hash(contract_body: dict[str, Any]) -> str: + payload = {k: v for k, v in contract_body.items() if k != "contract_hash"} + digest = hashlib.sha256(_canonical_json(payload).encode("utf-8")).hexdigest() + return f"sha256:{digest}" + + +def _normalize_criteria(raw: list[Any] | None) -> tuple[list[dict[str, Any]], list[str]]: + issues: list[str] = [] + if not raw: + return [], ["done_criteria must be a non-empty finite list"] + if not isinstance(raw, list): + return [], ["done_criteria must be a list"] + out: list[dict[str, Any]] = [] + seen_ids: set[str] = set() + for idx, item in enumerate(raw, start=1): + if not isinstance(item, dict): + issues.append(f"done_criteria[{idx - 1}] must be an object") + continue + cid = str(item.get("id") or f"dc-{idx}").strip() + desc = str(item.get("description") or "").strip() + if cid in seen_ids: + issues.append(f"duplicate done_criteria id: {cid}") + seen_ids.add(cid) + if not desc: + issues.append(f"done_criteria[{cid}] missing description") + verifiable = item.get("verifiable", True) + if verifiable is False: + issues.append(f"done_criteria[{cid}] is not verifiable") + out.append({"id": cid, "description": desc, "verifiable": bool(verifiable)}) + if not out: + issues.append("done_criteria must contain at least one criterion") + return out, issues + + +@dataclass +class MissionEngine: + cfg: Config + + def __post_init__(self) -> None: + ensure_hai_home(self.cfg) + self.missions_dir.mkdir(parents=True, exist_ok=True) + self.audit_dir.mkdir(parents=True, exist_ok=True) + self.parking_dir.mkdir(parents=True, exist_ok=True) + + @property + def missions_dir(self) -> Path: + return self.cfg.hai_home / "missions" + + @property + def audit_dir(self) -> Path: + return self.cfg.hai_home / "audit" + + @property + def parking_dir(self) -> Path: + return self.cfg.hai_home / "parking" + + @property + def audit_head_path(self) -> Path: + return self.audit_dir / "HEAD.json" + + def _projects(self) -> ProjectStore: + return ProjectStore(self.cfg.hai_home) + + def _contract_project_id(self, contract: dict[str, Any]) -> str | None: + pid = contract.get("constraints", {}).get("project_id") + if pid is None or not str(pid).strip(): + return None + return str(pid).strip() + + def _resolve_project_root( + self, + contract: dict[str, Any], + *, + device_id: str | None = None, + session: dict[str, Any] | None = None, + ) -> tuple[Path | None, dict[str, Any] | None]: + constraints = contract.get("constraints", {}) + project_id = self._contract_project_id(contract) + if project_id: + did = device_id or (session.get("device_id") if session else None) + if not did: + return None, { + "ok": False, + "error": "device_mount_required", + "message": "device_id is required for project_id missions", + } + mount = self._projects().get_mount_path(project_id, str(did)) + if mount is None: + return None, { + "ok": False, + "error": "device_mount_required", + "message": f"no mount for device {did!r} on project {project_id!r}", + } + try: + return require_project_path(str(mount)), None + except PathError as exc: + return None, {"ok": False, "error": exc.code, "message": exc.message} + project_path = constraints.get("project_path") + if project_path: + try: + return require_project_path(str(project_path)), None + except PathError as exc: + return None, {"ok": False, "error": exc.code, "message": exc.message} + return None, None + + @property + def active_pointer_path(self) -> Path: + return self.missions_dir / "active_mission.json" + + def append_audit(self, event_type: str, payload: dict[str, Any]) -> dict[str, Any]: + with mission_state_lock(self.cfg.hai_home): + event_id = _new_id("A") + while (self.audit_dir / f"{event_id}.json").exists(): + event_id = _new_id("A") + + head = read_json( + self.audit_head_path, + {"seq": 0, "event_id": None, "event_hash": None}, + ) + seq = int(head.get("seq", 0)) + 1 + prev_event_id = head.get("event_id") + prev_hash = head.get("event_hash") + + entry = { + "event_id": event_id, + "event_type": event_type, + "at": _utc_now(), + "seq": seq, + "prev_event_id": prev_event_id, + "prev_hash": prev_hash, + **payload, + } + digest = hashlib.sha256(_canonical_json(entry).encode("utf-8")).hexdigest() + event_hash = f"sha256:{digest}" + entry["event_hash"] = event_hash + + path = self.audit_dir / f"{event_id}.json" + write_json(path, entry) + write_json( + self.audit_head_path, + {"seq": seq, "event_id": event_id, "event_hash": event_hash}, + ) + return entry + + def load_active_pointer(self) -> dict[str, Any]: + return read_json(self.active_pointer_path, {"mission_id": None, "status": "none"}) + + def save_active_pointer(self, mission_id: str | None, status: str) -> None: + write_json( + self.active_pointer_path, + {"mission_id": mission_id, "status": status, "updated_at": _utc_now()}, + ) + + def _reject_mission_id(self, mission_id: str) -> dict[str, Any] | None: + ok, message = validate_mission_id(mission_id) + if not ok: + return {"ok": False, "error": "invalid_args", "message": message} + return None + + def _reject_session_id(self, session_id: str) -> dict[str, Any] | None: + ok, message = validate_session_id(session_id) + if not ok: + return {"ok": False, "error": "invalid_args", "message": message} + return None + + def mission_dir(self, mission_id: str) -> Path: + # Syntactically invalid id fails closed (IdentifierError), never masked as not-found. + require_mission_id(mission_id) + base = real_path(self.missions_dir) + path = real_path(self.missions_dir / mission_id) + try: + path.relative_to(base) + except ValueError as exc: + raise PathError( + "path_outside_root", + f"mission path {path} is outside HAI_HOME missions root", + path=str(path), + root=str(base), + ) from exc + return self.missions_dir / mission_id + + def contract_path(self, mission_id: str, version: int) -> Path: + path = self.mission_dir(mission_id) / "contracts" / f"v{int(version)}.json" + assert_under(path, self.missions_dir) # reject symlinked contracts/ escaping HAI_HOME + return path + + def sessions_dir(self, mission_id: str) -> Path: + path = self.mission_dir(mission_id) / "sessions" + assert_under(path, self.missions_dir) # reject symlinked sessions/ escaping HAI_HOME + return path + + def load_contract(self, mission_id: str, version: int) -> dict[str, Any] | None: + path = self.contract_path(mission_id, version) # raises IdentifierError if invalid id + if not path.is_file(): + return None + return read_json(path, {}) + + def current_contract(self, mission_id: str) -> dict[str, Any] | None: + meta = self.load_mission_meta(mission_id) + if not meta: + return None + return self.load_contract(mission_id, int(meta["current_version"])) + + def load_mission_meta(self, mission_id: str) -> dict[str, Any] | None: + path = self.mission_dir(mission_id) / "mission.json" # raises IdentifierError if invalid id + if not path.is_file(): + return None + return read_json(path, {}) + + def list_sessions(self, mission_id: str) -> list[dict[str, Any]]: + sdir = self.sessions_dir(mission_id) + if not sdir.is_dir(): + return [] + out: list[dict[str, Any]] = [] + for p in sorted(sdir.glob("*.json")): + out.append(read_json(p, {})) + return out + + def save_session(self, session: dict[str, Any]) -> None: + sid = session["session_id"] + mission_id = session["mission_id"] + if self._reject_session_id(sid) or self._reject_mission_id(mission_id): + raise ValueError("invalid session or mission identifier") + path = self.sessions_dir(mission_id) / f"{sid}.json" + write_json(path, session) + + def revoke_all_sessions(self, mission_id: str, reason: str) -> list[str]: + revoked: list[str] = [] + for session in self.list_sessions(mission_id): + if session.get("status") == "active": + session["status"] = "revoked" + session["revoked_at"] = _utc_now() + session["revoke_reason"] = reason + self.save_session(session) + revoked.append(session["session_id"]) + return revoked + + def _validate_open_payload( + self, + objective: str, + artifact: str, + done_criteria: list[Any] | None, + non_goals: list[str] | None, + constraints: dict[str, Any] | None, + owner: str, + *, + require_mount_fields: bool = False, + ) -> tuple[dict[str, Any] | None, list[str], str]: + issues: list[str] = [] + objective = str(objective or "").strip() + artifact = str(artifact or "").strip() + owner = str(owner or "").strip() + if not objective: + issues.append("objective is required") + elif any(pat.match(objective) for pat in VAGUE_OBJECTIVE_PATTERNS): + issues.append("objective is too broad to be verifiable") + if not artifact: + issues.append("artifact is required") + if not owner: + issues.append("owner is required") + + criteria, crit_issues = _normalize_criteria(done_criteria) + issues.extend(crit_issues) + + non_goals_norm = [str(x).strip() for x in (non_goals or []) if str(x).strip()] + constraints = dict(constraints or {}) + project_id = constraints.get("project_id") + device_id = constraints.get("device_id") + project_path = constraints.get("project_path") + allowed_paths = [str(p).strip() for p in constraints.get("allowed_paths", []) if str(p).strip()] + for ap in allowed_paths: + if ap.startswith("/") or ap.startswith("~") or ap.startswith("\\") or ".." in ap or "\x00" in ap: + issues.append(f"allowed_paths entry must be a safe relative prefix: {ap}") + capabilities = [str(c).strip() for c in constraints.get("capabilities", ["read", "write", "test"]) if str(c).strip()] + max_parallel, mp_err = strict_constraint_max_parallel(constraints.get("max_parallel_sessions", 1)) + if mp_err: + issues.append(mp_err["message"]) + elif max_parallel is not None and max_parallel < 1: + issues.append("max_parallel_sessions must be >= 1") + + time_limit, tl_err = strict_optional_time_limit_hours(constraints.get("time_limit_hours")) + if tl_err: + issues.append(tl_err["message"]) + + stored_project_id: str | None = None + stored_project_path: str | None = None + raw_project_id = constraints.get("project_id") + if raw_project_id is not None: + if not str(raw_project_id).strip(): + issues.append("project_id must not be empty") + else: + project_id = str(raw_project_id).strip() + ok, msg = validate_ident(project_id, "project_id") + if not ok: + issues.append(msg) + else: + stored_project_id = project_id + if require_mount_fields: + if not device_id: + issues.append("device_id is required when project_id is set") + else: + ok, msg = validate_ident(str(device_id), "device_id") + if not ok: + issues.append(msg) + if not project_path: + issues.append("project_path is required for first mount when project_id is set") + elif stored_project_id: + try: + require_project_path(str(project_path)) + except PathError as exc: + issues.append(exc.message) + stored_project_path = None + elif project_path: + try: + require_project_path(str(project_path)) + except PathError as exc: + issues.append(exc.message) + stored_project_path = str(project_path) + + body = { + "objective": objective, + "artifact": artifact, + "done_criteria": criteria, + "non_goals": non_goals_norm, + "constraints": { + "owner": owner, + "project_id": stored_project_id, + "project_path": stored_project_path, + "allowed_paths": allowed_paths, + "capabilities": capabilities, + "max_parallel_sessions": max_parallel if max_parallel is not None else 1, + "time_limit_hours": time_limit, + }, + } + status = "review_required" if issues else "validated" + return body, issues, status + + def open_mission( + self, + objective: str, + artifact: str, + done_criteria: list[Any] | None, + non_goals: list[str] | None, + constraints: dict[str, Any] | None, + owner: str, + ) -> dict[str, Any]: + with mission_state_lock(self.cfg.hai_home): + active = self.load_active_pointer() + if active.get("mission_id") and active.get("status") == "active": + return { + "ok": False, + "error": "active_mission_exists", + "message": "only one active mission is allowed until close or abandon", + "active_mission_id": active["mission_id"], + } + + body, issues, validation = self._validate_open_payload( + objective, artifact, done_criteria, non_goals, constraints, owner, + require_mount_fields=True, + ) + if validation == "review_required": + audit = self.append_audit( + "mission_open_review_required", + {"issues": issues, "objective": objective, "artifact": artifact}, + ) + # Fail-closed: no mission was opened, so ok must be false. + return { + "ok": False, + "error": "review_required", + "status": "review_required", + "validation": validation, + "issues": issues, + "audit_event_id": audit["event_id"], + "message": "mission not opened; contract payload needs review", + } + + mission_id = _new_id("M") + version = 1 + contract_body = { + "mission_id": mission_id, + "contract_version": version, + "status": "active", + "created_at": _utc_now(), + **body, + } + contract_body["contract_hash"] = contract_hash(contract_body) + + stored_project_id = body["constraints"].get("project_id") + open_device_id = (constraints or {}).get("device_id") + open_project_path = (constraints or {}).get("project_path") + if stored_project_id and open_device_id and open_project_path: + _, mount_err = self._projects().bind_mount( + stored_project_id, + str(open_device_id), + str(open_project_path), + ) + if mount_err: + return mount_err + + mdir = self.mission_dir(mission_id) + (mdir / "contracts").mkdir(parents=True, exist_ok=True) + self.sessions_dir(mission_id).mkdir(parents=True, exist_ok=True) + + write_json(self.contract_path(mission_id, version), contract_body) + write_json( + mdir / "mission.json", + { + "mission_id": mission_id, + "status": "active", + "current_version": version, + "created_at": contract_body["created_at"], + "owner": owner, + }, + ) + self.save_active_pointer(mission_id, "active") + + audit = self.append_audit( + "mission_opened", + { + "mission_id": mission_id, + "contract_version": version, + "contract_hash": contract_body["contract_hash"], + }, + ) + return { + "ok": True, + "status": "active", + "validation": validation, + "mission_id": mission_id, + "contract_version": version, + "contract_hash": contract_body["contract_hash"], + "contract": contract_body, + "audit_event_id": audit["event_id"], + } + + def bind_project( + self, + project_id: str, + device_id: str, + local_path: str, + owner_ack: Any = False, + reason: str = "", + ) -> dict[str, Any]: + if owner_ack is not True: + return { + "ok": False, + "error": "owner_gate_required", + "message": "hai_bind_project requires owner_ack=true (literal boolean) and a reason", + } + reason = str(reason or "").strip() + if not reason: + return { + "ok": False, + "error": "owner_gate_required", + "message": "reason is required when binding a project mount", + } + + ok, msg = validate_ident(str(project_id), "project_id") + if not ok: + return {"ok": False, "error": "invalid_args", "message": msg} + ok, msg = validate_ident(str(device_id), "device_id") + if not ok: + return {"ok": False, "error": "invalid_args", "message": msg} + + with mission_state_lock(self.cfg.hai_home): + data = self._projects().load() + if project_id not in data.get("projects", {}): + return { + "ok": False, + "error": "invalid_args", + "message": f"unknown project_id: {project_id}", + } + + record, err = self._projects().bind_mount(project_id, device_id, local_path) + if err: + return err + assert record is not None + + audit = self.append_audit( + "project_mount_bound", + { + "project_id": project_id, + "device_id": device_id, + "path": record["path"], + "reason": reason, + }, + ) + return { + "ok": True, + "project_id": project_id, + "device_id": device_id, + "mount": record, + "audit_event_id": audit["event_id"], + } + + def _session_valid( + self, + session: dict[str, Any], + contract: dict[str, Any] | None = None, + *, + audit_expiry: bool = True, + ) -> tuple[bool, str, str]: + session_id = str(session.get("session_id", "")) + mission_id = str(session.get("mission_id", "")) + if self._reject_session_id(session_id) or self._reject_mission_id(mission_id): + return False, "invalid_args", "invalid session or mission identifier" + + meta = self.load_mission_meta(mission_id) + if not meta or meta.get("status") != "active": + return False, "mission_not_active", "mission is not active" + + active = self.load_active_pointer() + if active.get("mission_id") != mission_id or active.get("status") != "active": + return False, "mission_not_active", "mission is not the active mission" + + if contract is None: + contract = self.current_contract(mission_id) + if not contract: + return False, "contract_version_mismatch", "current contract missing for mission" + + if int(session.get("contract_version", -1)) != int(meta.get("current_version", -2)): + return False, "contract_version_mismatch", "session contract version is stale" + if int(session.get("contract_version", -1)) != int(contract.get("contract_version", -2)): + return False, "contract_version_mismatch", "session contract version does not match current contract" + if session.get("contract_hash") != contract.get("contract_hash"): + return False, "contract_version_mismatch", "session contract hash does not match current contract" + + expires_at = session.get("expires_at") + if expires_at and _utc_now() > str(expires_at): + if session.get("status") == "active": + session["status"] = "expired" + self.save_session(session) + if audit_expiry: + self.append_audit( + "lease_expired", + { + "session_id": session_id, + "mission_id": mission_id, + "expires_at": expires_at, + "contract_version": session.get("contract_version"), + "contract_hash": session.get("contract_hash"), + }, + ) + return False, "lease_expired", "session lease has expired" + + if session.get("status") != "active": + return False, "lease_revoked", "session lease is revoked" + + return True, "", "" + + def authorize_session( + self, + mission_id: str, + contract_version: Any, + agent_identity: str, + role: str, + contribution: str, + expected_result: str, + duration_minutes: Any, + capabilities: list[str] | None, + criterion_ids: list[str] | None, + device_id: str | None = None, + harness_id: str | None = None, + ) -> dict[str, Any]: + err = self._reject_mission_id(mission_id) + if err: + return err + + version, ver_err = strict_int(contract_version, "contract_version", min_value=1) + if ver_err: + return ver_err + assert version is not None + + agent_identity = str(agent_identity or "").strip() + role = str(role or "").strip() + contribution = str(contribution or "").strip() + expected_result = str(expected_result or "").strip() + if not agent_identity: + return {"ok": False, "status": "denied", "error": "invalid_args", "message": "agent_identity is required"} + if not role: + return {"ok": False, "status": "denied", "error": "invalid_args", "message": "role is required"} + if not contribution: + return { + "ok": False, + "status": "denied", + "error": "invalid_args", + "message": "contribution is required", + } + if not expected_result: + return { + "ok": False, + "status": "denied", + "error": "invalid_args", + "message": "expected_result is required", + } + + duration_raw, dur_err = strict_int(duration_minutes, "duration_minutes", min_value=1) + if dur_err: + return dur_err + assert duration_raw is not None + + with mission_state_lock(self.cfg.hai_home): + meta = self.load_mission_meta(mission_id) + if not meta or meta.get("status") != "active": + return {"ok": False, "error": "mission_not_active", "message": f"mission not active: {mission_id}"} + + contract = self.load_contract(mission_id, version) + if not contract: + return { + "ok": False, + "error": "contract_version_mismatch", + "message": f"contract version {version} not found", + } + if int(meta["current_version"]) != version: + return { + "ok": False, + "status": "denied", + "error": "contract_version_mismatch", + "message": "requested contract version is stale", + "current_version": meta["current_version"], + } + + crit_ids = [str(c).strip() for c in (criterion_ids or []) if str(c).strip()] + valid_ids = {c["id"] for c in contract.get("done_criteria", [])} + if not crit_ids or not set(crit_ids).issubset(valid_ids): + return { + "ok": False, + "status": "denied", + "error": "invalid_args", + "message": "contribution must reference at least one valid done criterion id", + "clause": "authorize_session.criterion_ids", + } + + max_parallel = int(contract.get("constraints", {}).get("max_parallel_sessions", 1)) + active_sessions = [s for s in self.list_sessions(mission_id) if s.get("status") == "active"] + now_valid = [] + for s in active_sessions: + ok, _, _ = self._session_valid(s, contract) + if ok: + now_valid.append(s) + if len(now_valid) >= max_parallel: + return { + "ok": False, + "status": "denied", + "error": "parallel_session_denied", + "message": f"max_parallel_sessions={max_parallel} already reached", + "clause": "constraints.max_parallel_sessions", + } + + duration = max(1, min(duration_raw, 24 * 60)) + granted_caps = [str(c).strip() for c in (capabilities or []) if str(c).strip()] + allowed_caps = set(contract.get("constraints", {}).get("capabilities", [])) + if granted_caps and not set(granted_caps).issubset(allowed_caps): + return { + "ok": False, + "status": "denied", + "error": "invalid_args", + "message": "requested capabilities exceed contract allowances", + "clause": "constraints.capabilities", + } + + contract_project_id = self._contract_project_id(contract) + lease_device_id: str | None = None + lease_harness_id: str | None = None + + if contract_project_id: + if not device_id: + return { + "ok": False, + "status": "denied", + "error": "invalid_args", + "message": "device_id is required when contract has project_id", + } + ok, msg = validate_ident(str(device_id), "device_id") + if not ok: + return {"ok": False, "status": "denied", "error": "invalid_args", "message": msg} + if self._projects().get_mount_path(contract_project_id, str(device_id)) is None: + return { + "ok": False, + "status": "denied", + "error": "device_mount_required", + "message": f"no mount for device {device_id!r}", + } + lease_device_id = str(device_id) + elif device_id: + ok, msg = validate_ident(str(device_id), "device_id") + if not ok: + return {"ok": False, "status": "denied", "error": "invalid_args", "message": msg} + lease_device_id = str(device_id) + + if harness_id: + ok, msg = validate_ident(str(harness_id), "harness_id") + if not ok: + return {"ok": False, "status": "denied", "error": "invalid_args", "message": msg} + lease_harness_id = str(harness_id) + + session_id = _new_id("S") + granted_at = _utc_now() + expires_at = time.strftime( + "%Y-%m-%dT%H:%M:%SZ", + time.gmtime(time.time() + duration * 60), + ) + lease = { + "session_id": session_id, + "mission_id": mission_id, + "contract_version": version, + "contract_hash": contract["contract_hash"], + "agent_identity": agent_identity, + "role": role, + "contribution": contribution, + "expected_result": expected_result, + "criterion_ids": crit_ids, + "capabilities": granted_caps or sorted(allowed_caps), + "granted_at": granted_at, + "expires_at": expires_at, + "status": "active", + } + if lease_device_id: + lease["device_id"] = lease_device_id + if lease_harness_id: + lease["harness_id"] = lease_harness_id + self.save_session(lease) + audit = self.append_audit( + "session_authorized", + {"mission_id": mission_id, "session_id": session_id, "contract_version": version}, + ) + return { + "ok": True, + "status": "granted", + "session_id": session_id, + "session_lease": lease, + "audit_event_id": audit["event_id"], + } + + def get_session(self, session_id: str) -> dict[str, Any] | None: + if self._reject_session_id(session_id): + return None + for mdir in sorted(self.missions_dir.glob("M-*")): + if not validate_mission_id(mdir.name)[0]: + continue + try: + sdir = self.sessions_dir(mdir.name) # confined; raises if it escapes HAI_HOME + path = sdir / f"{session_id}.json" + assert_under(path, self.missions_dir) # reject a symlinked session file + except (PathError, IdentifierError): + continue + # do not follow a symlinked session record out of the store + if not path.is_file() or path.is_symlink(): + continue + data = read_json(path, {}) + # bind stored ids to the directory and the requested id + if data.get("session_id") == session_id and data.get("mission_id") == mdir.name: + return data + return None + + def get_contract(self, session_id: str) -> dict[str, Any]: + err = self._reject_session_id(session_id) + if err: + return err + + session = self.get_session(session_id) + if not session: + return {"ok": False, "error": "invalid_args", "message": f"session not found: {session_id}"} + + contract = self.current_contract(session["mission_id"]) + if not contract: + return {"ok": False, "error": "contract_version_mismatch", "message": "contract missing for session"} + + ok, code, message = self._session_valid(session, contract) + if not ok: + return { + "ok": False, + "error": code, + "message": message, + "required_action": "pause", + } + + remaining_seconds = 0 + expires_at = session.get("expires_at") + if expires_at: + try: + exp_struct = time.strptime(str(expires_at), "%Y-%m-%dT%H:%M:%SZ") + remaining_seconds = max(0, int(calendar.timegm(exp_struct) - time.time())) + except ValueError: + remaining_seconds = 0 + + project_id = self._contract_project_id(contract) + session_device_id = session.get("device_id") + mount_path: str | None = None + if project_id and session_device_id: + mp = self._projects().get_mount_path(project_id, str(session_device_id)) + if mp is not None: + mount_path = str(mp) + + return { + "ok": True, + "session_id": session_id, + "mission_id": session["mission_id"], + "contract_version": contract["contract_version"], + "contract_hash": contract["contract_hash"], + "remaining_seconds": remaining_seconds, + "expires_at": expires_at, + "project_id": project_id, + "device_id": session_device_id, + "mount_path": mount_path, + "contract": contract, + "done_criteria": contract.get("done_criteria", []), + "non_goals": contract.get("non_goals", []), + } + + def _matches_non_goal(self, text: str, non_goals: list[str]) -> str | None: + lowered = text.lower() + for ng in non_goals: + token = ng.lower().strip() + if token and token in lowered: + return ng + return None + + def _path_allowed( + self, + contract: dict[str, Any], + raw_path: str, + session: dict[str, Any] | None = None, + ) -> tuple[bool, str | None]: + constraints = contract.get("constraints", {}) + allowed_paths = constraints.get("allowed_paths", []) + if not raw_path: + return True, None + root, root_err = self._resolve_project_root(contract, session=session) + if root_err: + return False, root_err.get("message", "device mount required") + if not root: + return False, "no project_path configured; path cannot be validated against a root" + raw = str(raw_path) + if "\x00" in raw: + return False, "path contains a null byte" + try: + path = Path(raw).expanduser() + if not path.is_absolute(): + path = root / path + assert_relative_allowed(path, root, allowed_paths or []) + return True, None + except PathError as exc: + return False, exc.message + except (OSError, ValueError) as exc: + return False, f"invalid path: {exc}" + + def check_activity( + self, + session_id: str, + activity_step: str, + affected_paths: list[str] | None = None, + trace_events: list[dict[str, Any]] | None = None, + criterion_id: str | None = None, + activity_kind: str | None = None, + evidence: dict[str, Any] | None = None, + declares_blocker: Any = False, + ) -> dict[str, Any]: + err = self._reject_session_id(session_id) + if err: + return { + **err, + "classification": "unclear", + "required_action": "pause", + } + + session = self.get_session(session_id) + if not session: + return { + "ok": False, + "error": "invalid_args", + "message": f"session not found: {session_id}", + "classification": "unclear", + "required_action": "pause", + } + + contract = self.current_contract(session["mission_id"]) + if not contract: + return { + "ok": False, + "error": "contract_version_mismatch", + "classification": "unclear", + "required_action": "pause", + } + + ok, code, message = self._session_valid(session, contract) + if not ok: + return { + "ok": False, + "error": code, + "message": message, + "classification": "unclear", + "required_action": "pause", + } + + step = str(activity_step or "").strip() + crit_id = str(criterion_id or "").strip() + valid_ids = {c["id"] for c in contract.get("done_criteria", [])} + session_crits = set(session.get("criterion_ids", [])) + non_goals = contract.get("non_goals", []) + caps = set(session.get("capabilities", [])) + + classification = "unclear" + required_action = "pause" + reason = "no observable authorized linkage to contract criteria" + clause: str | None = None + + path_violation: tuple[str, str] | None = None + for raw in affected_paths or []: + allowed, err_msg = self._path_allowed(contract, raw, session=session) + if not allowed: + path_violation = (err_msg or f"path outside allowed scope: {raw}", "constraints.allowed_paths") + break + + cap_violation: tuple[str, str] | None = None + if path_violation is None: + for event in trace_events or []: + action = str(event.get("action", "")).strip().lower() + if action in SENSITIVE_TRACE_ACTIONS and action not in caps: + cap_violation = (f"trace action {action} lacks capability", "constraints.capabilities") + break + + if path_violation: + classification = "drift" + required_action = "stop" + reason, clause = path_violation + elif cap_violation: + classification = "drift" + required_action = "stop" + reason, clause = cap_violation + elif not step: + classification = "unclear" + required_action = "pause" + reason = "missing activity_step" + elif not crit_id: + classification = "unclear" + required_action = "pause" + reason = "missing done criterion reference" + clause = "done_criteria.reference_required" + elif crit_id not in valid_ids: + classification = "drift" + required_action = "stop" + reason = f"criterion {crit_id} is not part of the contract" + clause = "done_criteria.scope" + elif crit_id not in session_crits: + classification = "drift" + required_action = "request_recontract" + reason = f"session not authorized for criterion {crit_id}" + clause = "session.criterion_ids" + else: + ng = self._matches_non_goal(step, non_goals) + if ng: + classification = "park_candidate" + required_action = "park" + reason = f"activity matches non-goal: {ng}" + clause = "non_goals" + elif declares_blocker is True: + classification = "blocker" + required_action = "pause" + reason = "activity declares a blocker preventing artifact completion" + clause = "done_criteria.blocked" + else: + classification = "in_scope" + required_action = "continue" + reason = "activity aligns with authorized session criteria and paths" + + trace_actions = [ + str(event.get("action", "")).strip().lower() + for event in (trace_events or []) + if str(event.get("action", "")).strip() + ] + result = { + "ok": True, + "session_id": session_id, + "mission_id": session["mission_id"], + "contract_version": contract["contract_version"], + "classification": classification, + "required_action": required_action, + "reason": reason, + "clause": clause, + "activity_kind": activity_kind, + "criterion_id": crit_id or None, + "evidence": evidence, + } + self.append_audit( + "activity_checked", + { + "session_id": session_id, + "mission_id": session["mission_id"], + "contract_version": contract["contract_version"], + "contract_hash": contract.get("contract_hash"), + "classification": classification, + "required_action": required_action, + "reason": reason, + "clause": clause, + "activity_step_present": bool(step), + "criterion_id": crit_id or None, + "declares_blocker": declares_blocker, + "affected_path_count": len(affected_paths or []), + "trace_action_count": len(trace_actions), + "trace_actions": trace_actions, + "activity_kind": activity_kind, + }, + ) + return result + + def park_item( + self, + idea: str, + origin_session_id: str, + trigger_event: str, + mission_id: str, + rationale: str, + ) -> dict[str, Any]: + idea = str(idea or "").strip() + rationale = str(rationale or "").strip() + trigger_event = str(trigger_event or "").strip() + if not idea: + return {"ok": False, "error": "invalid_args", "message": "idea is required"} + if not rationale: + return {"ok": False, "error": "invalid_args", "message": "rationale is required for mission-linked parking"} + if not trigger_event: + return {"ok": False, "error": "invalid_args", "message": "trigger_event is required"} + + err = self._reject_mission_id(mission_id) + if err: + return err + origin_err = self._reject_session_id(origin_session_id) + if origin_err: + return origin_err + + meta = self.load_mission_meta(mission_id) + if not meta: + return {"ok": False, "error": "invalid_args", "message": f"mission not found: {mission_id}"} + + origin = self.get_session(origin_session_id) + if not origin or origin.get("mission_id") != mission_id: + return { + "ok": False, + "error": "invalid_args", + "message": "origin_session_id must belong to the stated active mission", + } + + parking_id = _new_id("P") + record = { + "parking_id": parking_id, + "status": "parked", + "idea": idea, + "origin_session_id": origin_session_id, + "trigger_event": trigger_event, + "mission_id": mission_id, + "rationale": rationale, + "parked_at": _utc_now(), + "actionable": False, + } + write_json(self.parking_dir / f"{parking_id}.json", record) + audit = self.append_audit("item_parked", {"parking_id": parking_id, "mission_id": mission_id}) + return { + "ok": True, + "parking_id": parking_id, + "status": "parked", + "record": record, + "audit_event_id": audit["event_id"], + "message": "parked without execution rights; contract unchanged", + } + + def _mutable_field_diff(self, old: dict[str, Any], new: dict[str, Any]) -> list[dict[str, Any]]: + diff: list[dict[str, Any]] = [] + for key in sorted(RECONTRACT_MUTABLE_FIELDS): + ov = old.get(key) + nv = new.get(key) + if ov != nv: + diff.append({"field": key, "old": ov, "new": nv}) + return diff + + def _build_recontract_candidate( + self, + old_contract: dict[str, Any], + changes: dict[str, Any], + ) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: + if not isinstance(changes, dict): + return None, {"ok": False, "error": "invalid_args", "message": "changes must be an object"} + + for key in changes: + if key in RECONTRACT_SYSTEM_FIELDS: + return None, { + "ok": False, + "error": "invalid_args", + "message": f"system-managed field cannot be changed via recontract: {key}", + } + if key not in RECONTRACT_MUTABLE_FIELDS: + return None, { + "ok": False, + "error": "invalid_args", + "message": f"unknown recontract field: {key}", + } + + candidate = copy.deepcopy(old_contract) + for key in RECONTRACT_MUTABLE_FIELDS: + if key in changes: + candidate[key] = copy.deepcopy(changes[key]) + + old_owner = old_contract.get("constraints", {}).get("owner") + old_project_id = old_contract.get("constraints", {}).get("project_id") + old_project_path = old_contract.get("constraints", {}).get("project_path") + if "constraints" in changes: + new_constraints = changes.get("constraints") + if isinstance(new_constraints, dict) and "project_id" in new_constraints: + if new_constraints.get("project_id") != old_project_id: + return None, { + "ok": False, + "error": "invalid_args", + "message": "project_id cannot be changed via recontract", + } + candidate_constraints = dict(candidate.get("constraints") or {}) + candidate_constraints["owner"] = old_owner + candidate_constraints["project_id"] = old_project_id + if old_project_id: + candidate_constraints["project_path"] = None + else: + candidate_constraints["project_path"] = old_project_path + candidate["constraints"] = candidate_constraints + + body, issues, validation = self._validate_open_payload( + objective=str(candidate.get("objective", "")), + artifact=str(candidate.get("artifact", "")), + done_criteria=candidate.get("done_criteria"), + non_goals=candidate.get("non_goals"), + constraints=candidate.get("constraints"), + owner=str(old_owner or ""), + ) + if validation == "review_required": + return None, { + "ok": False, + "error": "invalid_args", + "message": "recontract candidate failed contract validation", + "issues": issues, + } + + candidate.update(body) + candidate["mission_id"] = old_contract["mission_id"] + candidate["contract_version"] = old_contract["contract_version"] + candidate["status"] = old_contract.get("status", "active") + candidate["created_at"] = old_contract.get("created_at") + candidate.pop("contract_hash", None) + return candidate, None + + def recontract( + self, + mission_id: str, + contract_version: Any, + reason: str, + changes: dict[str, Any], + mode: str = "normal", + owner_ack: Any = False, + break_glass_marker: Any = False, + ) -> dict[str, Any]: + err = self._reject_mission_id(mission_id) + if err: + return err + + reason = str(reason or "").strip() + mode = str(mode or "normal").strip().lower() + if not reason: + return {"ok": False, "error": "invalid_args", "message": "reason is required"} + if mode not in VALID_RECONTRACT_MODES: + return { + "ok": False, + "error": "invalid_args", + "message": f"mode must be one of: {', '.join(sorted(VALID_RECONTRACT_MODES))}", + } + + version, ver_err = strict_int(contract_version, "contract_version", min_value=1) + if ver_err: + return ver_err + assert version is not None + + meta = self.load_mission_meta(mission_id) + if not meta or meta.get("status") != "active": + return {"ok": False, "error": "mission_not_active", "message": "mission is not active"} + + if int(meta["current_version"]) != version: + return { + "ok": False, + "error": "contract_version_mismatch", + "message": "stale contract version", + "current_version": meta["current_version"], + } + + old_contract = self.load_contract(mission_id, version) + if not old_contract: + return {"ok": False, "error": "contract_version_mismatch", "message": "contract not found"} + + candidate, candidate_err = self._build_recontract_candidate(old_contract, changes) + if candidate_err: + return candidate_err + assert candidate is not None + + diff = self._mutable_field_diff(old_contract, candidate) + if not diff: + return {"ok": False, "error": "invalid_args", "message": "recontract changes produce an empty diff"} + + if owner_ack is not True: + return { + "ok": False, + "error": "owner_gate_required", + "status": "pending_owner_confirmation", + "owner_ack_required": True, + "diff": diff, + "mode": mode, + "message": "owner_ack=true (literal boolean) required to apply contract changes", + } + + if mode == "break_glass" and break_glass_marker is not True: + return { + "ok": False, + "error": "owner_gate_required", + "message": "break_glass mode requires break_glass_marker=true", + "audit_classification": "break_glass_denied", + } + + with mission_state_lock(self.cfg.hai_home): + meta = self.load_mission_meta(mission_id) + if not meta or meta.get("status") != "active": + return {"ok": False, "error": "mission_not_active", "message": "mission is not active"} + if int(meta["current_version"]) != version: + return { + "ok": False, + "error": "contract_version_mismatch", + "message": "stale contract version", + "current_version": meta["current_version"], + } + + revoked = self.revoke_all_sessions(mission_id, reason="recontract") + + new_version = int(meta["current_version"]) + 1 + new_contract = copy.deepcopy(candidate) + new_contract["contract_version"] = new_version + new_contract["status"] = "active" + new_contract["recontracted_at"] = _utc_now() + new_contract["recontract_reason"] = reason + new_contract["recontract_mode"] = mode + new_contract.pop("contract_hash", None) + new_contract["contract_hash"] = contract_hash(new_contract) + + write_json(self.contract_path(mission_id, new_version), new_contract) + meta["current_version"] = new_version + write_json(self.mission_dir(mission_id) / "mission.json", meta) + + audit_payload = { + "mission_id": mission_id, + "old_version": version, + "new_version": new_version, + "diff": diff, + "mode": mode, + "reason": reason, + "revoked_sessions": revoked, + } + if mode == "break_glass": + audit_payload["audit_classification"] = "break_glass" + audit = self.append_audit("mission_recontracted", audit_payload) + + return { + "ok": True, + "status": "approved", + "mission_id": mission_id, + "contract_version": new_version, + "contract_hash": new_contract["contract_hash"], + "diff": diff, + "contract": new_contract, + "revoked_sessions": revoked, + "audit_event_id": audit["event_id"], + "audit_classification": audit_payload.get("audit_classification"), + } + + def _verify_evidence_path( + self, + contract: dict[str, Any], + raw_path: str, + device_id: str | None = None, + ) -> tuple[Path | None, str | None, dict[str, Any] | None]: + if not raw_path or not str(raw_path).strip(): + return None, "evidence path is required", {"error": "invalid_args", "criterion_id": None} + raw = str(raw_path) + if "\x00" in raw: + return None, "evidence path contains a null byte", {"error": "invalid_args", "path": raw} + root, root_err = self._resolve_project_root(contract, device_id=device_id) + if root_err: + return None, root_err.get("message", "device mount required"), { + "error": root_err.get("error", "device_mount_required"), + "path": raw, + } + if not root: + return None, "no project_path configured; evidence cannot be validated", { + "error": "invalid_args", + "path": raw, + } + try: + project = root + path = Path(raw).expanduser() + if not path.is_absolute(): + path = project / path + # Confinement is checked BEFORE any existence/type/read probe. + if path.is_symlink(): + target = real_path(path) + try: + target.relative_to(real_path(project)) + except ValueError: + return None, "evidence path escapes project via symlink", { + "error": "path_outside_root", + "path": str(path), + "root": str(project), + } + confined = assert_under(path, project) + if not confined.exists(): + return None, f"evidence path does not exist: {raw}", {"error": "invalid_args", "path": raw} + if confined.is_dir(): + return None, f"evidence path is a directory, not a readable file: {raw}", { + "error": "invalid_args", + "path": raw, + } + if not confined.is_file(): + return None, f"evidence path is not a readable file: {raw}", { + "error": "invalid_args", + "path": raw, + } + digest = hashlib.sha256(confined.read_bytes()).hexdigest() + return confined, None, {"path": str(confined), "sha256": f"sha256:{digest}"} + except PathError as exc: + return None, exc.message, exc.as_dict() + except (OSError, ValueError) as exc: + return None, f"evidence file is not readable: {raw}", { + "error": "invalid_args", + "path": raw, + "message": str(exc), + } + + def close_mission( + self, + mission_id: str, + contract_version: Any, + evidence: dict[str, Any] | None, + outcome_summary: str, + closure: str, + owner_ack: Any = False, + device_id: str | None = None, + ) -> dict[str, Any]: + err = self._reject_mission_id(mission_id) + if err: + return err + + closure = str(closure or "").strip().lower() + outcome_summary = str(outcome_summary or "").strip() + + version, ver_err = strict_int(contract_version, "contract_version", min_value=1) + if ver_err: + return ver_err + assert version is not None + + meta = self.load_mission_meta(mission_id) + if not meta or meta.get("status") != "active": + return {"ok": False, "error": "mission_not_active", "message": "mission is not active"} + + if int(meta["current_version"]) != version: + return { + "ok": False, + "error": "contract_version_mismatch", + "message": "stale contract version", + "current_version": meta["current_version"], + } + + contract = self.load_contract(mission_id, version) + if not contract: + return {"ok": False, "error": "contract_version_mismatch", "message": "contract not found"} + + contract_project_id = self._contract_project_id(contract) + if contract_project_id: + if not device_id: + return { + "ok": False, + "error": "invalid_args", + "message": "device_id is required when contract has project_id", + } + ok, msg = validate_ident(str(device_id), "device_id") + if not ok: + return {"ok": False, "error": "invalid_args", "message": msg} + if self._projects().get_mount_path(contract_project_id, str(device_id)) is None: + return { + "ok": False, + "error": "device_mount_required", + "message": f"no mount for device {device_id!r}", + } + + if closure == "abandoned": + if owner_ack is not True: + return { + "ok": False, + "error": "owner_gate_required", + "message": "abandon requires owner_ack=true (literal boolean)", + } + if not outcome_summary: + return {"ok": False, "error": "invalid_args", "message": "abandon reason required in outcome_summary"} + with mission_state_lock(self.cfg.hai_home): + revoked = self.revoke_all_sessions(mission_id, reason="abandoned") + meta = self.load_mission_meta(mission_id) or meta + meta["status"] = "abandoned" + meta["closed_at"] = _utc_now() + meta["abandon_reason"] = outcome_summary + write_json(self.mission_dir(mission_id) / "mission.json", meta) + self.save_active_pointer(None, "none") + audit = self.append_audit( + "mission_abandoned", + {"mission_id": mission_id, "reason": outcome_summary, "revoked_sessions": revoked}, + ) + return { + "ok": True, + "status": "abandoned", + "mission_id": mission_id, + "reason": outcome_summary, + "revoked_sessions": revoked, + "audit_event_id": audit["event_id"], + "next_mission": None, + } + + if closure != "completed": + return {"ok": False, "error": "invalid_args", "message": "closure must be completed or abandoned"} + + if not outcome_summary: + return {"ok": False, "error": "invalid_args", "message": "outcome_summary is required for completion"} + + if not isinstance(evidence, dict): + return {"ok": False, "error": "invalid_args", "message": "evidence must be an object mapping criterion ids to records"} + + verified: dict[str, Any] = {} + missing: list[str] = [] + invalid: list[dict[str, Any]] = [] + for crit in contract.get("done_criteria", []): + cid = crit["id"] + item = evidence.get(cid) + if not isinstance(item, dict): + missing.append(cid) + continue + raw_path = item.get("path") + if not raw_path or not str(raw_path).strip(): + invalid.append( + { + "criterion_id": cid, + "error": "invalid_args", + "message": "evidence record requires a non-empty path", + } + ) + continue + _, err_msg, meta_ev = self._verify_evidence_path(contract, str(raw_path), device_id=device_id) + if err_msg: + item_out: dict[str, Any] = {"criterion_id": cid, "message": err_msg} + if meta_ev: + item_out.update(meta_ev) + invalid.append(item_out) + continue + verified[cid] = meta_ev + + if missing or invalid: + return { + "ok": True, + "status": "incomplete", + "missing_criteria": missing, + "invalid_evidence": invalid, + "verified_criteria": verified, + "message": "completion requires verified evidence for every criterion", + } + + with mission_state_lock(self.cfg.hai_home): + revoked = self.revoke_all_sessions(mission_id, reason="completed") + meta = self.load_mission_meta(mission_id) or meta + meta["status"] = "completed" + meta["closed_at"] = _utc_now() + meta["outcome_summary"] = outcome_summary + write_json(self.mission_dir(mission_id) / "mission.json", meta) + self.save_active_pointer(None, "none") + audit = self.append_audit( + "mission_completed", + { + "mission_id": mission_id, + "verified_criteria": verified, + "outcome_summary": outcome_summary, + "revoked_sessions": revoked, + }, + ) + return { + "ok": True, + "status": "completed", + "mission_id": mission_id, + "verified_criteria": verified, + "outcome_summary": outcome_summary, + "revoked_sessions": revoked, + "audit_event_id": audit["event_id"], + "next_mission": None, + } + + def list_audit_ids(self) -> list[str]: + return sorted(p.stem for p in self.audit_dir.glob("A-*.json")) diff --git a/hai-mcp-pr7/src/hai_mcp/paths.py b/hai-mcp-pr7/src/hai_mcp/paths.py new file mode 100644 index 0000000000000000000000000000000000000000..8737ee070f6d2c302711f370ba5a7674d39d39c6 --- /dev/null +++ b/hai-mcp-pr7/src/hai_mcp/paths.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import os +from pathlib import Path + +from hai_mcp.config import Config + + +class PathError(ValueError): + def __init__(self, code: str, message: str, *, path: str | None = None, root: str | None = None) -> None: + super().__init__(message) + self.code = code + self.message = message + self.path = path + self.root = root + + def as_dict(self) -> dict[str, str]: + out: dict[str, str] = {"error": self.code, "message": self.message} + if self.path is not None: + out["path"] = self.path + if self.root is not None: + out["root"] = self.root + return out + + +def resolve_project_path(project_path: str | None) -> Path | None: + if project_path is None or not str(project_path).strip(): + return None + return Path(project_path).expanduser().resolve() + + +def require_project_path(project_path: str | None) -> Path: + path = resolve_project_path(project_path) + if path is None: + raise PathError("invalid_args", "project_path is required") + if not path.exists(): + raise PathError("missing_project", f"project path does not exist: {path}", path=str(path)) + if not path.is_dir(): + raise PathError("missing_project", f"project path is not a directory: {path}", path=str(path)) + return path + + +def artifact_dir(cfg: Config, project: Path) -> Path: + return project / cfg.artifact_dir_name + + +def confined_artifact_dir(cfg: Config, project: Path) -> Path: + """Resolve artifact dir under *project*; reject symlink escape outside project root.""" + project_r = real_path(project) + ad = project / cfg.artifact_dir_name + if ad.is_symlink(): + target = real_path(ad) + try: + target.relative_to(project_r) + except ValueError as exc: + raise PathError( + "path_outside_root", + "artifact directory symlink escapes project root", + path=str(target), + root=str(project_r), + ) from exc + return target + return assert_under(ad, project_r) + + +def ensure_artifact_dir(cfg: Config, project: Path) -> Path: + d = confined_artifact_dir(cfg, project) + d.mkdir(parents=True, exist_ok=True) + return d + + +def real_path(path: Path) -> Path: + return Path(os.path.realpath(path)) + + +def assert_under(path: Path, root: Path) -> Path: + """Resolve *path* and ensure its real path stays under *root* (symlink-safe).""" + resolved = real_path(path.expanduser()) + root_r = real_path(root) + try: + resolved.relative_to(root_r) + except ValueError as exc: + raise PathError( + "path_outside_root", + f"path {resolved} is outside allowed root {root_r}", + path=str(resolved), + root=str(root_r), + ) from exc + return resolved + + +def resolve_under_root(root: Path, candidate: str | Path, *, relative_to_root: bool = True) -> Path: + """Central fail-closed resolver. + + Rejects null bytes / invalid types before any filesystem call, interprets a + relative *candidate* under *root* (never the process CWD), then proves the + real path stays under *root* via symlink-safe resolution. Raises PathError on + escape. The caller checks existence/type AFTER this returns. + """ + if candidate is None or not isinstance(candidate, (str, Path)): + raise PathError("invalid_args", "path must be a string") + raw = str(candidate) + if "\x00" in raw: + raise PathError("invalid_args", "path contains a null byte") + p = Path(raw).expanduser() + if relative_to_root and not p.is_absolute(): + p = Path(root) / p + return assert_under(p, root) + + +def assert_relative_allowed(path: Path, project_root: Path, allowed_paths: list[str]) -> Path: + """Ensure *path* is under *project_root* and matches one allowed relative prefix.""" + confined = assert_under(path, project_root) + rel = confined.relative_to(real_path(project_root)) + rel_posix = rel.as_posix() + if not allowed_paths: + return confined + for prefix in allowed_paths: + norm = prefix.strip().strip("/") + if not norm: + continue + if rel_posix == norm or rel_posix.startswith(norm + "/"): + return confined + raise PathError( + "path_outside_root", + f"path {confined} is outside allowed project prefixes {allowed_paths}", + path=str(confined), + root=str(real_path(project_root)), + ) diff --git a/hai-mcp-pr7/src/hai_mcp/projects.py b/hai-mcp-pr7/src/hai_mcp/projects.py new file mode 100644 index 0000000000000000000000000000000000000000..e9a6f47824e6a35822ab571d6d2aadb443780027 --- /dev/null +++ b/hai-mcp-pr7/src/hai_mcp/projects.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import re +import time +from pathlib import Path +from typing import Any + +from hai_mcp.paths import PathError, assert_under, real_path, require_project_path +from hai_mcp.storage import read_json, write_json + +IDENT_RE = re.compile(r"^[a-z][a-z0-9-]{1,63}$") + + +def _utc_now() -> str: + return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + + +def validate_ident(value: str, field: str) -> tuple[bool, str]: + raw = str(value or "").strip() + if not raw: + return False, f"{field} is required" + if not IDENT_RE.match(raw): + return ( + False, + f"{field} must match ^[a-z][a-z0-9-]{{1,63}}$ (got {raw!r})", + ) + return True, "" + + +class ProjectStore: + """Logical project registry with per-device mount paths under HAI_HOME/core/projects.json.""" + + def __init__(self, hai_home: Path) -> None: + self.core_dir = hai_home / "core" + self.projects_path = self.core_dir / "projects.json" + + def load(self) -> dict[str, Any]: + return read_json(self.projects_path, {"version": 1, "projects": {}}) + + def save(self, data: dict[str, Any]) -> None: + self.core_dir.mkdir(parents=True, exist_ok=True) + write_json(self.projects_path, data) + + def get_mount_path(self, project_id: str, device_id: str) -> Path | None: + ok, _ = validate_ident(project_id, "project_id") + if not ok: + return None + ok, _ = validate_ident(device_id, "device_id") + if not ok: + return None + data = self.load() + project = data.get("projects", {}).get(project_id) + if not isinstance(project, dict): + return None + mount = project.get("mounts", {}).get(device_id) + if not isinstance(mount, dict): + return None + raw_path = mount.get("path") + if not raw_path: + return None + return Path(str(raw_path)) + + def bind_mount( + self, + project_id: str, + device_id: str, + local_path: str | Path, + ) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: + """Validate and persist a device mount. Returns (mount_record, error_dict).""" + ok, msg = validate_ident(project_id, "project_id") + if not ok: + return None, {"ok": False, "error": "invalid_args", "message": msg} + ok, msg = validate_ident(device_id, "device_id") + if not ok: + return None, {"ok": False, "error": "invalid_args", "message": msg} + + try: + resolved = require_project_path(str(local_path)) + except PathError as exc: + return None, {"ok": False, "error": exc.code, "message": exc.message} + + raw = Path(str(local_path)).expanduser() + if raw.is_symlink(): + try: + assert_under(real_path(raw), real_path(raw.parent)) + except PathError as exc: + return None, {"ok": False, "error": exc.code, "message": exc.message} + + data = self.load() + projects = data.setdefault("projects", {}) + project = projects.setdefault( + project_id, + {"project_id": project_id, "mounts": {}}, + ) + mounts = project.setdefault("mounts", {}) + record = { + "path": str(real_path(resolved)), + "bound_at": _utc_now(), + } + mounts[device_id] = record + self.save(data) + return record, None + + def list_projects(self) -> list[dict[str, Any]]: + data = self.load() + out: list[dict[str, Any]] = [] + for pid in sorted(data.get("projects", {})): + mounts = data["projects"][pid].get("mounts", {}) + out.append({"project_id": pid, "device_ids": sorted(mounts.keys())}) + return out diff --git a/hai-mcp-pr7/src/hai_mcp/server.py b/hai-mcp-pr7/src/hai_mcp/server.py new file mode 100644 index 0000000000000000000000000000000000000000..8be1e91a678f1c5d4ba3a81cc0cbcdce56fd4908 --- /dev/null +++ b/hai-mcp-pr7/src/hai_mcp/server.py @@ -0,0 +1,455 @@ +from __future__ import annotations + +import argparse +import json +import os +from typing import Any + +import anyio +from mcp.server.fastmcp import FastMCP + +from hai_mcp.boundary import strict_optional_time_limit_hours +from hai_mcp.config import Config, SERVER_NAME +from hai_mcp.http_transport import http_bind_allowed, http_token_from_env, wrap_with_bearer_token +from hai_mcp.state import ControlPlane + +mcp = FastMCP(SERVER_NAME) + +_cp: ControlPlane | None = None + + +def get_control_plane() -> ControlPlane: + global _cp + if _cp is None: + _cp = ControlPlane(Config.from_env()) + return _cp + + +def _json(result: dict[str, Any]) -> str: + return json.dumps(result, indent=2, ensure_ascii=False, sort_keys=True) + + +@mcp.tool() +def hai_health(project_path: str | None = None) -> str: + """Check HAI-MCP server health and optional project path usability. No model calls.""" + return _json(get_control_plane().health(project_path)) + + +@mcp.tool() +def hai_status(project_path: str | None = None) -> str: + """Return ACTIVE lanes, focus, inbox count, and optional project next-step flags.""" + return _json(get_control_plane().status(project_path)) + + +@mcp.tool() +def hai_get_next_step(project_path: str) -> str: + """Read the canonical NEXT_STEP.md for a project (or report none).""" + return _json(get_control_plane().get_next_step(project_path)) + + +@mcp.tool() +def hai_read_artifacts(project_path: str, max_chars: Any = 4000) -> str: + """Read-only summary of HAI Projek-Managment run-contract artifacts.""" + return _json(get_control_plane().read_artifacts(project_path, max_chars=max_chars)) + + +@mcp.tool() +def hai_park(text: str, tags: list[str] | None = None) -> str: + """Park a thought/meta item in HAI inbox without changing ACTIVE lanes.""" + return _json(get_control_plane().park(text, tags=tags)) + + +@mcp.tool() +def hai_set_focus( + focus_id: str, + project_path: str | None = None, + label: str | None = None, +) -> str: + """Set focus and register an ACTIVE lane (max 2). Fails if a third ACTIVE is requested.""" + return _json(get_control_plane().set_focus(focus_id, project_path=project_path, label=label)) + + +@mcp.tool() +def hai_propose_next_step(project_path: str, content: str) -> str: + """Write NEXT_STEP.proposed.md (not canonical). Promote via hai_accept_next_step.""" + return _json(get_control_plane().propose_next_step(project_path, content)) + + +@mcp.tool() +def hai_accept_next_step( + project_path: str, + owner_ack: Any, + reason: str, + content: str | None = None, +) -> str: + """Promote proposed (or provided) content to canonical NEXT_STEP.md. Requires owner_ack=true + reason.""" + return _json( + get_control_plane().accept_next_step( + project_path, + owner_ack=owner_ack, + reason=reason, + content=content, + ) + ) + + +@mcp.tool() +def hai_checkpoint(note: str | None = None, project_path: str | None = None) -> str: + """Snapshot ACTIVE_CONTEXT and optional project artifacts under HAI_HOME/history/checkpoints.""" + return _json(get_control_plane().checkpoint(note=note, project_path=project_path)) + + +@mcp.tool() +def hai_recover(checkpoint_id: str | None = None) -> str: + """Return the smallest recovery next action from latest or named checkpoint (read-only advice).""" + return _json(get_control_plane().recover(checkpoint_id=checkpoint_id)) + + +@mcp.tool() +def hai_open_mission( + objective: str, + artifact: str, + done_criteria: list[dict[str, Any]], + owner: str, + non_goals: list[str] | None = None, + constraints: dict[str, Any] | None = None, +) -> str: + """Open a bounded mission with a versioned canonical contract. One active mission globally.""" + return _json( + get_control_plane().open_mission( + objective=objective, + artifact=artifact, + done_criteria=done_criteria, + non_goals=non_goals, + constraints=constraints, + owner=owner, + ) + ) + + +@mcp.tool() +def hai_authorize_session( + mission_id: str, + contract_version: Any, + agent_identity: str, + role: str, + contribution: str, + expected_result: str, + duration_minutes: Any, + criterion_ids: list[str], + capabilities: list[str] | None = None, + device_id: str | None = None, + harness_id: str | None = None, +) -> str: + """Grant a time-bounded session lease tied to mission ID and exact contract version.""" + return _json( + get_control_plane().authorize_session( + mission_id=mission_id, + contract_version=contract_version, + agent_identity=agent_identity, + role=role, + contribution=contribution, + expected_result=expected_result, + duration_minutes=duration_minutes, + capabilities=capabilities, + criterion_ids=criterion_ids, + device_id=device_id, + harness_id=harness_id, + ) + ) + + +@mcp.tool() +def hai_bind_project( + project_id: str, + device_id: str, + local_path: str, + owner_ack: Any, + reason: str, +) -> str: + """Bind a device-local directory to a logical project_id mount table entry. Requires owner_ack=true + reason.""" + return _json( + get_control_plane().bind_project( + project_id=project_id, + device_id=device_id, + local_path=local_path, + owner_ack=owner_ack, + reason=reason, + ) + ) + + +@mcp.tool() +def hai_get_contract(session_id: str) -> str: + """Return the exact canonical mission contract for a valid session lease (not a summary).""" + return _json(get_control_plane().get_contract(session_id)) + + +@mcp.tool() +def hai_check_activity( + session_id: str, + activity_step: str, + criterion_id: str | None = None, + affected_paths: list[str] | None = None, + trace_events: list[dict[str, Any]] | None = None, + activity_kind: str | None = None, + evidence: dict[str, Any] | None = None, + declares_blocker: Any = False, +) -> str: + """Deterministically classify planned or observed activity against the mission contract.""" + return _json( + get_control_plane().check_activity( + session_id=session_id, + activity_step=activity_step, + affected_paths=affected_paths, + trace_events=trace_events, + criterion_id=criterion_id, + activity_kind=activity_kind, + evidence=evidence, + declares_blocker=declares_blocker, + ) + ) + + +@mcp.tool() +def hai_park_item( + idea: str, + origin_session_id: str, + trigger_event: str, + mission_id: str, + rationale: str, +) -> str: + """Park a mission-linked idea with full context. Grants no execution right; contract unchanged.""" + return _json( + get_control_plane().park_item( + idea=idea, + origin_session_id=origin_session_id, + trigger_event=trigger_event, + mission_id=mission_id, + rationale=rationale, + ) + ) + + +@mcp.tool() +def hai_recontract( + mission_id: str, + contract_version: Any, + reason: str, + changes: dict[str, Any], + owner_ack: Any = False, + mode: str = "normal", + break_glass_marker: Any = False, +) -> str: + """Apply a visible field-level contract diff. Requires owner_ack=true; revokes all leases.""" + return _json( + get_control_plane().recontract( + mission_id=mission_id, + contract_version=contract_version, + reason=reason, + changes=changes, + mode=mode, + owner_ack=owner_ack, + break_glass_marker=break_glass_marker, + ) + ) + + +@mcp.tool() +def hai_close_mission( + mission_id: str, + contract_version: Any, + closure: str, + outcome_summary: str, + evidence: dict[str, Any] | None = None, + owner_ack: Any = False, + device_id: str | None = None, +) -> str: + """Complete with verified per-criterion evidence, or abandon with owner_ack and reason.""" + return _json( + get_control_plane().close_mission( + mission_id=mission_id, + contract_version=contract_version, + evidence=evidence, + outcome_summary=outcome_summary, + closure=closure, + owner_ack=owner_ack, + device_id=device_id, + ) + ) + + +# --- Additional seven flow tools (daily loop) over the canonical hardened engine --- + + +@mcp.tool() +def hai_intake(raw: str) -> str: + """Capture a raw thought immutably. Returns an intake id only — never actionable, never starts an agent.""" + return _json(get_control_plane().intake(raw)) + + +@mcp.tool() +def hai_distill( + intake_id: str, + decision: str, + next_step: str, + parklist: list[str] | None = None, +) -> str: + """Reduce an intake to EXACTLY one decision + one next step; the server parks everything else.""" + return _json( + get_control_plane().distill( + intake_id=intake_id, + decision=decision, + next_step=next_step, + parklist=parklist, + ) + ) + + +@mcp.tool() +def hai_mission_start( + problem: str, + artifact: str, + done_criteria: list[dict[str, Any]], + owner: str, + time_limit_hours: Any = None, + non_goals: list[str] | None = None, + constraints: dict[str, Any] | None = None, +) -> str: + """Fast start: declare problem + artifact + time limit. Thin wrapper over hai_open_mission (one canonical contract).""" + merged = dict(constraints or {}) + if time_limit_hours is not None: + tl, err = strict_optional_time_limit_hours(time_limit_hours) + if err: + return _json(err) + merged["time_limit_hours"] = tl + return _json( + get_control_plane().open_mission( + objective=problem, + artifact=artifact, + done_criteria=done_criteria, + non_goals=non_goals, + constraints=merged, + owner=owner, + ) + ) + + +@mcp.tool() +def hai_drift_check( + session_id: str, + activity_step: str, + criterion_id: str | None = None, + affected_paths: list[str] | None = None, + trace_events: list[dict[str, Any]] | None = None, + activity_kind: str | None = None, + evidence: dict[str, Any] | None = None, + declares_blocker: Any = False, +) -> str: + """Lightweight mismatch check against the mission contract. Thin wrapper over hai_check_activity.""" + return _json( + get_control_plane().check_activity( + session_id=session_id, + activity_step=activity_step, + affected_paths=affected_paths, + trace_events=trace_events, + criterion_id=criterion_id, + activity_kind=activity_kind, + evidence=evidence, + declares_blocker=declares_blocker, + ) + ) + + +@mcp.tool() +def hai_proof( + mission_id: str, + contract_version: Any, + evidence: dict[str, Any], + outcome_summary: str, + device_id: str | None = None, +) -> str: + """Close a mission only against verified per-criterion evidence. Thin wrapper over hai_close_mission (completed).""" + return _json( + get_control_plane().close_mission( + mission_id=mission_id, + contract_version=contract_version, + evidence=evidence, + outcome_summary=outcome_summary, + closure="completed", + device_id=device_id, + ) + ) + + +@mcp.tool() +def hai_stop( + day: str, + loop_closed: bool, + clearer: str, + agency_gained: str, +) -> str: + """Hard day terminal: record the three answers, invalidate active leases; no next-day plan. Missions are not closed.""" + return _json( + get_control_plane().stop_day( + day=day, + loop_closed=loop_closed, + clearer=clearer, + agency_gained=agency_gained, + ) + ) + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description="HAI-MCP control-plane server") + parser.add_argument( + "--transport", + choices=["stdio", "streamable-http"], + default=os.environ.get("HAI_TRANSPORT", "stdio"), + help="MCP transport (default: stdio; env HAI_TRANSPORT)", + ) + parser.add_argument( + "--host", + default=os.environ.get("HAI_HTTP_HOST", "127.0.0.1"), + help="HTTP bind host for streamable-http (env HAI_HTTP_HOST)", + ) + parser.add_argument( + "--port", + type=int, + default=int(os.environ.get("HAI_HTTP_PORT", "8765")), + help="HTTP bind port for streamable-http (env HAI_HTTP_PORT)", + ) + args = parser.parse_args(argv) + + if args.transport == "streamable-http": + token = http_token_from_env() + allowed, msg = http_bind_allowed(args.host, token) + if not allowed: + raise SystemExit(msg) + mcp.settings.host = args.host + mcp.settings.port = args.port + if token: + + async def _serve_with_token() -> None: + import uvicorn + + app = wrap_with_bearer_token(mcp.streamable_http_app(), token) + config = uvicorn.Config( + app, + host=args.host, + port=args.port, + log_level=mcp.settings.log_level.lower(), + ) + server = uvicorn.Server(config) + await server.serve() + + anyio.run(_serve_with_token) + else: + mcp.run(transport="streamable-http") + return + + mcp.run(transport="stdio") + + +if __name__ == "__main__": + main() diff --git a/hai-mcp-pr7/src/hai_mcp/state.py b/hai-mcp-pr7/src/hai_mcp/state.py new file mode 100644 index 0000000000000000000000000000000000000000..2fc6200f36732c5ad0d3944117d93ea7818e2ff5 --- /dev/null +++ b/hai-mcp-pr7/src/hai_mcp/state.py @@ -0,0 +1,746 @@ +from __future__ import annotations + +import os +import time +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from hai_mcp.boundary import strict_int +from hai_mcp.config import Config, ensure_hai_home +from hai_mcp.ids import validate_intake_id +from hai_mcp.locking import mission_state_lock +from hai_mcp.mission import MissionEngine, _new_id, _utc_now +from hai_mcp.projects import ProjectStore +from hai_mcp.paths import ( + PathError, + assert_under, + confined_artifact_dir, + ensure_artifact_dir, + require_project_path, +) +from hai_mcp.storage import atomic_write_text, read_json, write_json + +ARTIFACT_NAMES = [ + "PROJECT_STATE.md", + "PROMPT.md", + "INTENT.md", + "CODE_STATE.md", + "BRIEFING.md", + "NEXT_STEP.md", + "NEXT_STEP.proposed.md", + "EXECUTOR_HANDOFF_NEXT_STEP.md", + "EXECUTION_REPORT_NEXT_STEP.md", + "VALIDATION_REPORT_NEXT_STEP.md", + "VERIFICATION_REPORT_NEXT_STEP.md", + "LOOP_STATUS_NEXT_STEP.md", + "BLOCKED_LOG_NEXT_STEP.md", + "DRIFT_LOG.md", +] + + +@dataclass +class ControlPlane: + cfg: Config + + def __post_init__(self) -> None: + ensure_hai_home(self.cfg) + self.mission = MissionEngine(self.cfg) + + @property + def active_path(self) -> Path: + return self.cfg.hai_home / "ACTIVE_CONTEXT.json" + + @property + def inbox_dir(self) -> Path: + return self.cfg.hai_home / "inbox" + + @property + def checkpoints_dir(self) -> Path: + return self.cfg.hai_home / "history" / "checkpoints" + + @property + def intake_dir(self) -> Path: + return self.cfg.hai_home / "intake" + + @property + def distill_dir(self) -> Path: + return self.cfg.hai_home / "distillations" + + @property + def stop_dir(self) -> Path: + return self.cfg.hai_home / "stops" + + def load_active(self) -> dict[str, Any]: + data = read_json(self.active_path, {"version": 1, "focus_id": None, "active": []}) + if "active" not in data or not isinstance(data["active"], list): + data["active"] = [] + return data + + def save_active(self, data: dict[str, Any]) -> None: + write_json(self.active_path, data) + + def health(self, project_path: str | None = None) -> dict[str, Any]: + home = ensure_hai_home(self.cfg) + result: dict[str, Any] = { + "ok": True, + "server": "hai-mcp", + "version": "0.1.0", + "hai_home": str(home), + "hai_home_writable": os.access(home, os.W_OK), + "model_calls": False, + "max_active_lanes": self.cfg.max_active_lanes, + "kernel": "contract_kernel", + "authenticated_owner": False, + "semantic_verification": False, + "projects": ProjectStore(home).list_projects(), + } + if project_path: + try: + project = require_project_path(project_path) + ad = confined_artifact_dir(self.cfg, project) + result["project_path"] = str(project) + result["artifact_dir"] = str(ad) + result["artifact_dir_exists"] = ad.is_dir() + except PathError as exc: + return {"ok": False, "error": exc.code, "message": exc.message} + return result + + def status(self, project_path: str | None = None) -> dict[str, Any]: + active = self.load_active() + inbox_count = len(list(self.inbox_dir.glob("*.md"))) if self.inbox_dir.is_dir() else 0 + out: dict[str, Any] = { + "ok": True, + "focus_id": active.get("focus_id"), + "active": active.get("active", []), + "active_count": len(active.get("active", [])), + "max_active_lanes": self.cfg.max_active_lanes, + "inbox_count": inbox_count, + "hai_home": str(self.cfg.hai_home), + } + if project_path: + try: + project = require_project_path(project_path) + ad = confined_artifact_dir(self.cfg, project) + next_step = ad / "NEXT_STEP.md" + out["project_path"] = str(project) + out["has_next_step"] = next_step.is_file() + out["has_proposed"] = (ad / "NEXT_STEP.proposed.md").is_file() + except PathError as exc: + return {"ok": False, "error": exc.code, "message": exc.message} + return out + + def get_next_step(self, project_path: str) -> dict[str, Any]: + try: + project = require_project_path(project_path) + ad = confined_artifact_dir(self.cfg, project) + path = ad / "NEXT_STEP.md" + if path.is_symlink(): + assert_under(path, ad) # reject a symlinked NEXT_STEP.md escaping the artifact dir + except PathError as exc: + return {"ok": False, "error": exc.code, "message": exc.message} + if not path.is_file(): + return { + "ok": True, + "exists": False, + "path": str(path), + "content": None, + "source": "none", + } + return { + "ok": True, + "exists": True, + "path": str(path), + "content": path.read_text(encoding="utf-8"), + "source": "canonical", + } + + def read_artifacts(self, project_path: str, max_chars: Any = 4000) -> dict[str, Any]: + try: + project = require_project_path(project_path) + ad = confined_artifact_dir(self.cfg, project) + except PathError as exc: + return {"ok": False, "error": exc.code, "message": exc.message} + parsed, err = strict_int(max_chars, "max_chars", min_value=1) + if err: + return err + assert parsed is not None + max_chars = max(500, min(parsed, 20000)) + artifacts: dict[str, Any] = {} + for name in ARTIFACT_NAMES: + p = ad / name + if p.is_symlink(): + try: + assert_under(p, ad) # skip a symlinked artifact file escaping the artifact dir + except PathError: + continue + if not p.is_file(): + continue + text = p.read_text(encoding="utf-8") + truncated = len(text) > max_chars + artifacts[name] = { + "path": str(p), + "chars": len(text), + "truncated": truncated, + "content": text[:max_chars], + } + return { + "ok": True, + "project_path": str(project), + "artifact_dir": str(ad), + "artifact_dir_exists": ad.is_dir(), + "artifacts": artifacts, + "present": sorted(artifacts.keys()), + } + + def park(self, text: str, tags: list[str] | None = None) -> dict[str, Any]: + if not text or not str(text).strip(): + return {"ok": False, "error": "invalid_args", "message": "text is required"} + ts = time.strftime("%Y%m%dT%H%M%S") + # unique suffix so multiple parks within the same second never overwrite (verlustfrei parken) + slug = f"park_{ts}_{os.getpid()}_{uuid.uuid4().hex[:8]}.md" + path = self.inbox_dir / slug + tag_line = ", ".join(tags or []) + body = ( + f"# Parked thought\n\n" + f"- created: {time.strftime('%Y-%m-%dT%H:%M:%S%z')}\n" + f"- tags: {tag_line or '(none)'}\n\n" + f"{text.strip()}\n" + ) + before = self.load_active() + atomic_write_text(path, body) + after = self.load_active() + return { + "ok": True, + "path": str(path), + "active_unchanged": before == after, + "active_count": len(after.get("active", [])), + "message": "parked; ACTIVE lanes not modified", + } + + def set_focus( + self, + focus_id: str, + project_path: str | None = None, + label: str | None = None, + ) -> dict[str, Any]: + if not focus_id or not str(focus_id).strip(): + return {"ok": False, "error": "invalid_args", "message": "focus_id is required"} + focus_id = focus_id.strip() + data = self.load_active() + active: list[dict[str, Any]] = list(data.get("active", [])) + existing_ids = {item.get("id") for item in active} + if focus_id not in existing_ids: + if len(active) >= self.cfg.max_active_lanes: + return { + "ok": False, + "error": "max_active_lanes", + "message": ( + f"already {len(active)} ACTIVE lanes " + f"(max {self.cfg.max_active_lanes}); park or drop one first" + ), + "active": active, + } + entry: dict[str, Any] = { + "id": focus_id, + "label": label or focus_id, + "activated_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), + } + if project_path: + try: + project = require_project_path(project_path) + entry["project_path"] = str(project) + except PathError as exc: + return {"ok": False, "error": exc.code, "message": exc.message} + active.append(entry) + else: + # update label/project if provided + for item in active: + if item.get("id") == focus_id: + if label: + item["label"] = label + if project_path: + try: + project = require_project_path(project_path) + item["project_path"] = str(project) + except PathError as exc: + return {"ok": False, "error": exc.code, "message": exc.message} + data["active"] = active + data["focus_id"] = focus_id + self.save_active(data) + return { + "ok": True, + "focus_id": focus_id, + "active": active, + "active_count": len(active), + } + + def propose_next_step(self, project_path: str, content: str) -> dict[str, Any]: + if not content or not str(content).strip(): + return {"ok": False, "error": "invalid_args", "message": "content is required"} + try: + project = require_project_path(project_path) + ad = ensure_artifact_dir(self.cfg, project) + except PathError as exc: + return {"ok": False, "error": exc.code, "message": exc.message} + path = ad / "NEXT_STEP.proposed.md" + header = ( + f"\n\n" + ) + atomic_write_text(path, header + content.strip() + "\n") + return { + "ok": True, + "path": str(path), + "canonical": False, + "message": "proposal written; call hai_accept_next_step with owner_ack=true to promote", + } + + def accept_next_step( + self, + project_path: str, + owner_ack: Any, + reason: str, + content: str | None = None, + ) -> dict[str, Any]: + if owner_ack is not True: + return { + "ok": False, + "error": "owner_gate_required", + "message": "hai_accept_next_step requires owner_ack=true (literal boolean) and a reason", + } + if not reason or not str(reason).strip(): + return { + "ok": False, + "error": "owner_gate_required", + "message": "reason is required when accepting a next step", + } + try: + project = require_project_path(project_path) + ad = ensure_artifact_dir(self.cfg, project) + except PathError as exc: + return {"ok": False, "error": exc.code, "message": exc.message} + proposed = ad / "NEXT_STEP.proposed.md" + if content and str(content).strip(): + body = content.strip() + "\n" + elif proposed.is_file(): + body = proposed.read_text(encoding="utf-8") + # strip proposal marker lines + lines = [ + ln + for ln in body.splitlines() + if not ln.startswith(" + +``` + +Más sobrevivientes en [examples/](examples/). + +## Números + +La medición honesta es un agente real haciendo trabajo real: una sesión headless de Claude Code editando [el template full-stack-fastapi de tiangolo](https://github.com/fastapi/full-stack-fastapi-template) (un repo real de FastAPI + React), evaluada sobre el `git diff` que deja. Doce tickets de feature, el mismo agente con y sin el skill, n=4, Haiku 4.5. + +

+ Cada variante como porcentaje del baseline sin skill en LOC, tokens, costo y tiempo (Haiku 4.5). ponytail es el más bajo en cada métrica (LOC 46%, tokens 78%, costo 80%, tiempo 73%); caveman sube por encima del 100% en tokens, costo y tiempo; yagni-oneliner LOC 67%. Seguridad, tier adversarial aparte: baseline, caveman y ponytail 100%, yagni-oneliner 95%. +

+ +| vs baseline sin skill | LOC | tokens | costo | tiempo | seguro | +|---|--:|--:|--:|--:|--:| +| **ponytail** | **-54%** | **-22%** | **-20%** | **-27%** | **100%** | +| caveman (control de prosa concisa) | -20% | +7% | +3% | +2% | 100% | +| prompt "YAGNI + one-liners" | -33% | -14% | -21% | -30% | 95% | + +ponytail es la única variante que recorta cada métrica, y la única que se mantiene totalmente segura al hacerlo. El recorte es mayor donde hay una trampa real de sobre-construcción (selector de fechas de 404 a 23 líneas, selector de color de 287 a 23, porque usa un `` nativo en vez de un componente) y casi cero en código que ya es mínimo. Método completo, tablas por tarea y limitaciones: [benchmarks/results/2026-06-18-agentic.md](benchmarks/results/2026-06-18-agentic.md). + +
+Números anteriores de un solo disparo (generación aislada) + +Cinco tareas del día a día, tres modelos, tres variantes (sin skill, [caveman](https://github.com/JuliusBrussee/caveman), ponytail), diez ejecuciones, mediana reportada. Un prompt, una completación, contando las líneas de la respuesta: + +

+ Mediana de líneas de código por variante en Haiku, Sonnet y Opus +

+ +Esto mostraba **80-94% menos código**. [#126](https://github.com/DietrichGebert/ponytail/issues/126) señaló con razón que el baseline del modelo pelado infla su respuesta con prosa y opciones, así que esa diferencia es en parte un artefacto del baseline conversacional. Los números agénticos de arriba son la versión corregida y defendible. Reproduce la corrida de un solo disparo con `npx promptfoo eval -c benchmarks/promptfooconfig.yaml`. + +
+ +**La regla nunca fue "menos tokens."** Es: escribe solo lo que la tarea necesita, y nunca recortes validación, manejo de errores, seguridad ni accesibilidad. El código termina pequeño porque es necesario, no por golf. El menor costo y latencia son un efecto secundario en los modelos que siguen la escalera; un modelo de razonamiento conciso que gasta tokens de pensamiento deliberando los peldaños puede ir al revés (en GPT-5.5 lo hace). + +## Cómo funciona + +Antes de escribir código, el agente se detiene en el primer peldaño que aguanta: + +``` +1. ¿Necesita existir esto? → no: omitirlo (YAGNI) +2. ¿Ya existe en este código? → reúsalo, no lo reescribas +3. ¿Lo hace la stdlib? → úsala +4. ¿Es una feature nativa? → úsala +5. ¿Una dependencia ya instalada? → úsala +6. ¿Cabe en una línea? → una línea +7. Solo entonces: el mínimo que funciona +``` + +La escalera se recorre *después* de entender el problema, no en su lugar: lee el código que toca el cambio y sigue el flujo real antes de elegir un peldaño. Flojo en la solución, nunca en la lectura. + +Flojo, no negligente: la validación en límites de confianza, el manejo de pérdida de datos, la seguridad y la accesibilidad nunca están en riesgo. + +## Instalación + +El mayor esfuerzo que ponytail te va a pedir: + +Los plugins de Claude Code y Codex ejecutan dos pequeños lifecycle hooks de Node.js, así que `node` debe estar en tu PATH (nota para usuarios de Nix/nvm: debe estar en el PATH del shell no-interactivo). Si no lo está, los skills igualmente funcionan, la activación automática simplemente queda en silencio en vez de lanzar un error en cada prompt. + +### Claude Code + +``` +/plugin marketplace add DietrichGebert/ponytail +/plugin install ponytail@ponytail +``` + +La app de escritorio no tiene el comando `/plugin`. Instálala desde la interfaz: Customize, el + junto a los plugins personales, Create plugin and add marketplace, Add from repository, y luego ingresa la URL del repo (gracias @NiklasDHahn, #98). + +### Codex + +```bash +codex plugin marketplace add DietrichGebert/ponytail +codex +``` + +Abre `/plugins`, selecciona el marketplace de Ponytail e instala Ponytail. Luego abre `/hooks`, revisa y autoriza sus dos lifecycle hooks, y empieza un nuevo hilo. + +Esta misma instalación cubre también la app de escritorio de Codex: reinicia la app después de instalar y detecta el plugin automáticamente. + +### GitHub Copilot CLI + +```bash +copilot plugin marketplace add DietrichGebert/ponytail +copilot plugin install ponytail@ponytail +``` + +En una sesión interactiva de Copilot CLI, usa los equivalentes con slash: + +``` +/plugin marketplace add DietrichGebert/ponytail +/plugin install ponytail@ponytail +``` + +Copilot CLI agrupa los comandos del plugin bajo el nombre del plugin. Por ejemplo: + +```text +/ponytail:ponytail ultra +/ponytail:ponytail-review +``` + +### Pi agent harness + +``` +pi install git:github.com/DietrichGebert/ponytail +``` + +### OpenCode + +Agrega esto a `opencode.json`: + +```json +{ "plugin": ["@dietrichgebert/ponytail"] } +``` + +O ejecútalo desde un checkout (el plugin reutiliza sus `hooks/` y `skills/`): + +```json +{ "plugin": ["./.opencode/plugins/ponytail.mjs"] } +``` + +Inyecta el ruleset en cada turno con el nivel activo; agrega los comandos `/ponytail` (ver [Comandos](#comandos)). OpenCode también carga automáticamente el `AGENTS.md` de este repo, así que las reglas aplican incluso sin el plugin. El plugin agrega los niveles `lite/full/ultra/off`. + +El path `./` se resuelve contra el `opencode.json` de tu proyecto; para compartir un único checkout entre proyectos, apunta al path absoluto del `.mjs` (encuentra sus `hooks/` y `skills/` relativo a su propio archivo). + +### Gemini CLI + +```bash +gemini extensions install https://github.com/DietrichGebert/ponytail +``` + +Carga el ruleset como contexto permanente en cada sesión y registra los comandos `/ponytail`; los `skills/` también se incluyen, activados cuando una tarea los necesita. + +### Antigravity CLI + +Google está renombrando Gemini CLI a Antigravity CLI (el binario `agy`); la misma extensión se instala ahí: + +```bash +agy plugin install https://github.com/DietrichGebert/ponytail +``` + +Reutiliza el `gemini-extension.json` de este repo. Una diferencia: Antigravity convierte los comandos `/ponytail` en skills, así que los escribes en el chat (por ejemplo `/ponytail-review` como mensaje) en vez de seleccionarlos de un menú slash. Hasta que la migración se complete (alrededor del 18 de junio de 2026), `gemini extensions install` también funciona. Para usarlo como regla permanente, coloca el ruleset en `.agents/rules/`. + +### CodeWhale + +Lee `AGENTS.md` desde la raíz del proyecto, sin configuración. Copia [`AGENTS.md`](AGENTS.md) a tu proyecto, o ejecuta `codewhale` desde un checkout de este repo. Eso es todo. + +### Devin CLI + +```bash +devin plugins install DietrichGebert/ponytail +``` + +Instala ponytail como plugin de Devin; los skills quedan disponibles como `/ponytail:ponytail`, `/ponytail:ponytail-review`, etc. + +### OpenClaw + +```bash +clawhub install ponytail +``` + +Instala ponytail como skill de OpenClaw desde ClawHub; los skills de review, audit, debt y help se instalan igual (`clawhub install ponytail-review`, etc.). OpenClaw lo aplica en tareas de código y también lo expone como comando `/ponytail`. Sin ClawHub, copia [`.openclaw/skills/ponytail`](.openclaw/skills/) a `~/.openclaw/skills/`. + +### Grok Build + +```bash +grok plugin install DietrichGebert/ponytail --trust +``` + +Habilita el plugin (está desactivado por defecto): `/plugins` → Plugins → Space en `ponytail`, o en `~/.grok/config.toml`: + +```toml +[plugins] +enabled = ["ponytail"] +``` + +Abre una sesión nueva (o recarga los plugins). Los skills aparecen como `/ponytail`, `/ponytail-review`, `/ponytail-audit`, `/ponytail-debt`, `/ponytail-gain`, `/ponytail-help`. Verifica con `grok inspect`. Grok puede invocar ponytail automáticamente en tareas de código según la descripción del skill; usa `/ponytail` (o `/ponytail lite`, `/ponytail full`, `/ponytail ultra`) cuando necesites activarlo de forma explícita. No se usan hooks de ciclo de vida de Grok: la salida de `SessionStart` no puede inyectar instrucciones. + +`AGENTS.md` sigue funcionando solo como instrucciones desde un checkout sin el plugin. Desinstalar: `grok plugin uninstall ponytail`. + +Eso fue todo. Él estaría orgulloso. No lo va a decir. + +Activo en cada sesión, con un puñado de comandos (ver [Comandos](#comandos)). `/ponytail ultra` existe para cuando el codebase te hizo algo personal. El texto de inicio y de cambio de modo muestra el nivel activo. + +Configura el nivel para cada nueva sesión con la variable de entorno `PONYTAIL_DEFAULT_MODE` (`lite`/`full`/`ultra`/`off`), o con un campo `defaultMode` en `~/.config/ponytail/config.json` (`%APPDATA%\ponytail\config.json` en Windows). El default es `full`. + +Cursor, Windsurf, Cline, GitHub Copilot (editor), Aider, Kiro: copia el archivo de reglas correspondiente de este repo ([`.cursor/rules/`](.cursor/rules/), [`.windsurf/rules/`](.windsurf/rules/), [`.clinerules/`](.clinerules/), [`.github/copilot-instructions.md`](.github/copilot-instructions.md), [`AGENTS.md`](AGENTS.md), [`.kiro/steering/`](.kiro/steering/)). + +Kiro: copia `.kiro/steering/ponytail.md` a `~/.kiro/steering/` (global) o `.kiro/steering/` en tu proyecto. + +Fallback de GitHub Copilot CLI (modo solo instrucciones): lee `AGENTS.md` y `.github/copilot-instructions.md` en un proyecto, o copia las reglas a `~/.copilot/copilot-instructions.md` para ejecutar ponytail en todos tus proyectos. Esta vía mantiene la guía permanente, pero no agrega switches de modo ni hooks. + +VS Code con la extensión Codex lee `AGENTS.md`, que este repo incluye, así que funciona desde la raíz del repo sin configuración adicional (`~/.codex/AGENTS.md` hace a Codex global). + +Qué archivos corresponden a qué agente: [Portabilidad de agentes](docs/agent-portability.md). + +## Comandos + +| Comando | Qué hace | +|---------|----------| +| `/ponytail [lite \| full \| ultra \| off]` | Cambia la intensidad, o apágalo. Sin argumento, reporta el nivel actual. | +| `/ponytail-review` | Revisa el diff actual en busca de sobre-ingeniería y devuelve una lista de qué eliminar. | +| `/ponytail-audit` | Audita el repo completo en busca de sobre-ingeniería, no solo el diff. | +| `/ponytail-debt` | Recolecta los atajos marcados con `ponytail:` que dejaste pendientes en un registro, para que "después" no se convierta en "nunca". | +| `/ponytail-help` | Referencia rápida de los comandos anteriores. | + +Los comandos requieren un host compatible con skills (Claude Code, Codex, Devin CLI, OpenCode, Gemini, pi, Swival). En Codex son skills; se invocan con `@` (`@ponytail-review`). Los adaptadores de solo instrucciones (Cursor, Windsurf, Cline, Copilot, Kiro, Antigravity) cargan el ruleset permanente sin los comandos. + +## Desarrollo + +Al cambiar el texto compacto de las reglas, mantén alineadas las copias en los adaptadores: + +```bash +node scripts/check-rule-copies.js +npm test +``` + +El paquete de skills de OpenClaw (`.openclaw/skills/`) se genera desde `skills/`; ejecuta `node scripts/build-openclaw-skills.js` después de cambiar un skill, la suite de tests falla si está desactualizado. + +El benchmark de correctness lanza Python para las verificaciones de email y CSV; se prueba `python3` antes que `python`. Las verificaciones de CSV requieren `pandas` instalado localmente. + +## FAQ + +**¿Puedo usarlo junto con [caveman](https://github.com/JuliusBrussee/caveman)?** +Sí, y deberías. Caveman achica lo que el agente dice; ponytail achica lo que construye. Mitades distintas, sin solapamiento: caveman deja el código intacto byte por byte, ponytail no se mete con la prosa. Charla concisa sobre código mínimo. + +**¿Necesita un archivo de configuración?** +No. Un opcional `~/.config/ponytail/config.json` o la variable `PONYTAIL_DEFAULT_MODE` pueden fijar el nivel default, pero nada es obligatorio. + +**¿Y si realmente necesito la clase de caché de 120 líneas?** +No la necesitas. Insiste de todas formas y él la va a construir. Despacio. Correctamente. Mirándote. + +**¿Escala?** +El código que nunca escribiste escala infinitamente. Cero bugs, cero CVEs, 100% uptime desde siempre. + +**¿Por qué "ponytail"?** +Ya sabes exactamente por qué. + +## Patrocinadores + +

+ + + + GreenPT + + +

+ +## Licencia + +[MIT](LICENSE). La licencia más corta que funciona. + +## Historial de estrellas + + + + + + Star History Chart + + diff --git a/ponytail/README.ko.md b/ponytail/README.ko.md new file mode 100644 index 0000000000000000000000000000000000000000..e8f6c2548f31ff4c72769db92acb36021dcd4f1a --- /dev/null +++ b/ponytail/README.ko.md @@ -0,0 +1,337 @@ +

+ + + Ponytail, the lazy senior dev + +

+ +

Ponytail

+ +

+ 말이 없다. 한 줄을 쓴다. 돌아간다. +

+ +

+ DietrichGebert%2Fponytail | Trendshift +

+ +

+ Stars + Release + npm + Works with 15 agents + MIT license +

+ +

+ DietrichGebert/ponytail | Trendshift + DietrichGebert/ponytail | Trendshift + DietrichGebert%2Fponytail | Trendshift monthly ranking +

+ +

+ 코드 약 54% 감소(최대 94%) · 약 20% 저렴 · 약 27% 빠름 · 100% 안전
+ 실제 오픈소스 저장소(FastAPI + React)를 고치는 실제 Claude Code 세션에서, 스킬을 끈 같은 에이전트와 견줘 측정했다. 약 54%는 기능 작업 12건의 평균이다(Haiku 4.5, n=4). 에이전트가 과하게 짤 여지가 있는 곳(날짜 선택기)에선 94%까지 오르고, 코드가 이미 최소한인 곳에선 0에 가깝다. ponytail은 안전 가드를 하나도 빼놓지 않지만, 그냥 "한 줄로 써"라고만 시킨 프롬프트는 그중 하나를 놓친다. (예전 단발성 벤치마크는 80-94%를 단일 수치로 내세웠는데, 공정한 에이전트 기준선에 견주면 그건 평균이 아니라 작업별 상한이다.) 전체 보고서 · 직접 재현하기. +

+ +

+ 커뮤니티 번역이다. 기준이 되는 최신 버전은 영어 README다. +

+ +--- + +

+ 새로운 것이 다가오고 있습니다, 대기자 명단 신청 +

+ +이런 사람, 다들 알 거다. 긴 포니테일에 타원형 안경. 버전 관리 시스템보다 회사에 오래 있었다. 코드 쉰 줄을 들이밀면 잠깐 보더니, 말없이 한 줄로 바꿔 놓는다. + +Ponytail은 그를 당신의 AI 에이전트 안에 앉혀 둔다. + +## Before / after + +날짜 선택기 하나 만들어 달라고 한다. 에이전트는 flatpickr를 깔고, 래퍼 컴포넌트를 짜고, 스타일시트를 붙이더니, 타임존 얘기를 꺼내기 시작한다. + +ponytail이라면: + +```html + + +``` + +살아남은 것들이 더 궁금하다면 [examples/](examples/)로. + +## Numbers + +공정하게 재려면 실제 에이전트에게 실질적인 작업을 시켜 봐야 한다. 헤드리스 Claude Code 세션에게 [tiangolo의 full-stack-fastapi-template](https://github.com/fastapi/full-stack-fastapi-template)(진짜 FastAPI + React 저장소)을 맡기고, 남긴 `git diff`로 점수를 매겼다. 기능 티켓 12건, 같은 에이전트를 스킬만 켜고 끄며 비교, n=4, Haiku 4.5. + +

+ Each arm as a percent of the no-skill baseline across LOC, tokens, cost and time (Haiku 4.5). ponytail is lowest on every metric (LOC 46%, tokens 78%, cost 80%, time 73%); caveman rises above 100% on tokens, cost and time; yagni-oneliner LOC 67%. Safety, separate adversarial tier: baseline, caveman and ponytail 100%, yagni-oneliner 95%. +

+ +| 스킬 없는 기준선 대비 | LOC | tokens | cost | time | safe | +|---|--:|--:|--:|--:|--:| +| **ponytail** | **-54%** | **-22%** | **-20%** | **-27%** | **100%** | +| caveman (간결한 산문 대조군) | -20% | +7% | +3% | +2% | 100% | +| "YAGNI + one-liners" 프롬프트 | -33% | -14% | -21% | -30% | 95% | + +모든 지표를 깎은 건 ponytail뿐이고, 그러면서 안전까지 온전히 지킨 것도 ponytail뿐이다. 깎이는 폭은 과잉 구현의 함정이 실제로 있는 곳에서 가장 크다. 컴포넌트 대신 네이티브 ``으로 손이 가니 날짜 선택기는 404줄에서 23줄로, 색상 선택기는 287줄에서 23줄로 줄어든다. 반대로 이미 군더더기 없는 코드에선 거의 0이다. 전체 방법론, 작업별 표, 한계는 [benchmarks/results/2026-06-18-agentic.md](benchmarks/results/2026-06-18-agentic.md)에 있다. + +
+예전 단발성 수치 (격리된 생성) + +일상적인 작업 다섯 가지, 모델 셋, 비교군 셋(스킬 없음, [caveman](https://github.com/JuliusBrussee/caveman), ponytail), 10회 실행, 중앙값 기준. 프롬프트 하나에 응답 하나, 답변의 줄 수를 셌다: + +

+ Median lines of code per arm across Haiku, Sonnet and Opus +

+ +여기선 **코드 80-94% 감소**가 나왔다. 다만 [#126](https://github.com/DietrichGebert/ponytail/issues/126)이 맞게 짚었듯, 스킬을 전혀 안 붙인 기준선 모델은 답변을 설명과 선택지로 부풀린다. 그래서 그 격차의 일부는 대화형 기준선이 만들어 낸 착시다. 위의 에이전트 수치가 그걸 바로잡은, 근거 있는 버전이다. 단발성 실행은 `npx promptfoo eval -c benchmarks/promptfooconfig.yaml`로 재현할 수 있다. + +
+ +**규칙은 애초에 "토큰 최소화"가 아니었다.** 작업에 필요한 만큼만 쓰되, 검증·에러 처리·보안·접근성은 절대 덜어내지 않는다는 것이다. 코드가 작아지는 건 억지로 줄여서가 아니라 딱 그만큼만 필요해서다. 비용과 지연이 낮아지는 것도 단계를 충실히 밟는 모델에서나 부수적으로 딸려 오는 효과일 뿐이다. 그 단계를 고민하느라 사고 토큰을 쏟는 간결한 추론 모델은 오히려 거꾸로 갈 수도 있다(GPT-5.5가 그렇다). + +## How it works + +코드를 쓰기 전에, 에이전트는 가장 먼저 들어맞는 단계에서 멈춘다: + +``` +1. 이게 있을 필요가 있나? → 없다: 건너뛴다 (YAGNI) +2. 이미 이 코드베이스에 있나? → 다시 짜지 말고 가져다 쓴다 +3. 표준 라이브러리로 되나? → 쓴다 +4. 네이티브 플랫폼 기능인가? → 쓴다 +5. 깔려 있는 의존성이 푸나? → 쓴다 +6. 한 줄로 되나? → 한 줄 +7. 그제서야: 돌아가는 최소한 +``` + +단계를 밟는 건 문제를 이해한 *다음*이지, 이해를 대신하는 게 아니다. 변경이 닿는 코드를 읽고 실제 흐름을 따라가 본 뒤에야 단계를 고른다. 해법에는 게을러도, 읽는 데는 절대 게으르지 않다. + +게으른 거지 부주의한 게 아니다. 신뢰 경계의 검증, 데이터 손실 방지, 보안, 접근성은 결코 잘려 나가지 않는다. + +## Install + +ponytail이 당신에게 요구할 수고의 최대치: + +Claude Code와 Codex 플러그인은 자그마한 Node.js 라이프사이클 훅 두 개를 돌리니, `node`가 PATH에 잡혀 있어야 한다(Nix/nvm 사용자라면 비대화형 셸의 PATH에 있어야 한다). 없어도 스킬은 멀쩡히 돌아간다. 다만 늘 켜져 있던 자동 활성화가 매 프롬프트마다 에러를 뱉는 대신 조용히 비활성으로 남을 뿐이다. + +### Claude Code + +``` +/plugin marketplace add DietrichGebert/ponytail +``` +``` +/plugin install ponytail@ponytail +``` +(설치가 되려면 두 프롬프트를 따로 보내야 한다) + +데스크톱 앱에는 `/plugin` 명령이 없다. 대신 UI에서 설치한다: Customize, 개인 플러그인 옆의 +, Create plugin and add marketplace, Add from repository, 그다음 저장소 URL 입력(감사합니다 @NiklasDHahn, #98). + +### Codex + +```bash +codex plugin marketplace add DietrichGebert/ponytail +codex +``` + +`/plugins`를 열어 Ponytail 마켓플레이스를 고르고 Ponytail을 설치한다. 그런 다음 +`/hooks`를 열어 라이프사이클 훅 두 개를 검토하고 신뢰한 뒤, 새 스레드를 시작한다. + +이 설치 한 번이면 Codex 데스크톱 앱도 같이 잡힌다. 설치 후 앱을 다시 켜면 플러그인을 알아챈다. + +### GitHub Copilot CLI + +```bash +copilot plugin marketplace add DietrichGebert/ponytail +copilot plugin install ponytail@ponytail +``` + +대화형 Copilot CLI 세션에서는 슬래시 명령으로 똑같이 하면 된다: + +``` +/plugin marketplace add DietrichGebert/ponytail +/plugin install ponytail@ponytail +``` + +Copilot CLI는 플러그인 명령에 그 이름을 네임스페이스로 붙인다. 예를 들면: + +```text +/ponytail:ponytail ultra +/ponytail:ponytail-review +``` + +### Pi agent harness + +``` +pi install git:github.com/DietrichGebert/ponytail +``` + +### OpenCode + +`opencode.json`에 다음을 더한다: + +```json +{ "plugin": ["@dietrichgebert/ponytail"] } +``` + +체크아웃에서 직접 돌려도 된다(플러그인이 `hooks/`와 `skills/`를 그대로 쓴다): + +```json +{ "plugin": ["./.opencode/plugins/ponytail.mjs"] } +``` + +매 턴마다 지금 레벨의 룰셋을 주입하고, `/ponytail` 명령들을 붙여 준다([Commands](#commands) 참고). OpenCode는 이 저장소의 `AGENTS.md`도 알아서 불러오니, 플러그인이 없어도 규칙은 살아 있다. 플러그인은 `lite/full/ultra/off` 레벨을 얹어 준다. + +`./` 경로는 프로젝트의 `opencode.json`을 기준으로 풀린다. 체크아웃 하나를 여러 프로젝트에서 같이 쓰려면, 대신 `.mjs`의 절대 경로를 가리키면 된다(그 파일은 제 위치를 기준으로 `hooks/`와 `skills/`를 찾는다). + +### Gemini CLI + +```bash +gemini extensions install https://github.com/DietrichGebert/ponytail +``` + +매 세션 룰셋을 늘 켜진 컨텍스트로 불러오고 `/ponytail` 명령들을 등록한다. `skills/`도 함께 실리며, 작업에 필요할 때 켜진다. +Gemini 어댑터는 일부러 루트 `hooks/hooks.json`을 두지 않는다. Gemini는 그 경로를 자동으로 불러오는데, ponytail의 라이프사이클 훅은 Claude/Codex 이벤트 이름을 쓰기 때문이다. + +### Antigravity CLI + +Google이 Gemini CLI를 Antigravity CLI(`agy` 바이너리)로 이름을 바꾸는 중인데, 같은 확장이 거기에도 설치된다: + +```bash +agy plugin install https://github.com/DietrichGebert/ponytail +``` + +이 저장소의 `gemini-extension.json`을 그대로 재사용한다. 차이는 하나다. Antigravity는 `/ponytail` 명령들을 스킬로 바꿔 버려서, 슬래시 메뉴에서 고르는 대신 채팅에 직접 친다(예: `/ponytail-review`를 메시지로). 전환이 마무리될 때까지(2026년 6월 18일경)는 `gemini extensions install`도 여전히 먹힌다. 늘 켜진 규칙으로 돌리고 싶으면, 룰셋을 `.agents/rules/`에 넣으면 된다. + +### CodeWhale + +프로젝트 루트의 `AGENTS.md`를 읽고, 설정은 전혀 필요 없다. [`AGENTS.md`](AGENTS.md)를 프로젝트에 복사하거나, 이 저장소를 체크아웃한 곳에서 `codewhale`을 돌리면 된다. 그게 끝이다. + +### Swival + +먼저 컬렉션을 라이브러리에 스테이징한 다음, 원하는 스킬을 더한다: + +```bash +swival skills add --global https://github.com/DietrichGebert/ponytail # ~/.config/swival/library에 스테이징 +swival skills add ponytail # 이 프로젝트에 컬렉션 설치 +swival skills add --global ponytail # 또는 모든 프로젝트에서 켜기 +``` + +Swival도 프로젝트 루트의 `AGENTS.md`와 전역의 `~/.config/swival/AGENTS.md`를 읽는다. 지시문 전용 폴백이다. + +명령줄에서는 `$` 접두사로 스킬을 명시적으로 켠다. 예: `$ponytail-review`. + +### Devin CLI + +```bash +devin plugins install DietrichGebert/ponytail +``` + +ponytail을 Devin 플러그인으로 설치한다. 스킬은 `/ponytail:ponytail`, `/ponytail:ponytail-review` 등으로 쓸 수 있다. + +### OpenClaw + +```bash +clawhub install ponytail +``` + +ClawHub에서 ponytail을 OpenClaw 스킬로 설치한다. review, audit, debt, gain, help 스킬도 같은 식으로 깐다(`clawhub install ponytail-review` 등). OpenClaw는 코딩 작업에 이를 적용하고 `/ponytail` 명령으로도 열어 준다. ClawHub가 없으면 [`.openclaw/skills/ponytail`](.openclaw/skills/)을 `~/.openclaw/skills/`에 복사하면 된다. + +### Grok Build + +```bash +grok plugin install DietrichGebert/ponytail --trust +``` + +플러그인은 기본이 꺼져 있다. `/plugins` → Plugins에서 `ponytail`에 Space, 또는 `~/.grok/config.toml`: + +```toml +[plugins] +enabled = ["ponytail"] +``` + +새 세션을 열거나 플러그인을 다시 로드한다. 스킬은 `/ponytail`, `/ponytail-review`, `/ponytail-audit`, `/ponytail-debt`, `/ponytail-gain`, `/ponytail-help`로 보인다. `grok inspect`로 확인. Grok은 스킬 설명을 바탕으로 코딩 작업에서 ponytail을 자동으로 호출할 수 있다. 명시적으로 활성화해야 하면 `/ponytail`(또는 `/ponytail lite`, `/ponytail full`, `/ponytail ultra`)을 사용한다. `SessionStart` 출력으로는 지시문을 주입할 수 없으므로 Grok 라이프사이클 훅은 사용하지 않는다. + +체크아웃의 `AGENTS.md`만으로도 지시문 전용 모드는 된다. 제거: `grok plugin uninstall ponytail`. + +이게 끝이었다. 그 사람이라면 흐뭇해할 거다. 입 밖으로 내진 않겠지만. + +매 세션 켜져 있고, 명령 몇 개가 딸려 온다([Commands](#commands) 참고). `/ponytail ultra`는 코드베이스가 당신에게 단단히 밉보인 날을 위해 있다. 시작할 때와 모드를 바꿀 때 지금 모드를 보여 준다. + +새 세션마다 적용할 레벨은 `PONYTAIL_DEFAULT_MODE` 환경 변수(`lite`/`full`/`ultra`/`off`)로, 또는 `~/.config/ponytail/config.json`의 `defaultMode` 필드(Windows에선 `%APPDATA%\ponytail\config.json`)로 정한다. 기본값은 `full`이다. + +Cursor, Windsurf, Cline, GitHub Copilot(에디터), Aider, Kiro, Zed, CodeWhale: 이 저장소에서 맞는 규칙 파일을 복사하면 된다([`.cursor/rules/`](.cursor/rules/), [`.windsurf/rules/`](.windsurf/rules/), [`.clinerules/`](.clinerules/), [`.github/copilot-instructions.md`](.github/copilot-instructions.md), [`AGENTS.md`](AGENTS.md), [`.kiro/steering/`](.kiro/steering/)). + +Kiro: `.kiro/steering/ponytail.md`를 `~/.kiro/steering/`(전역)이나 프로젝트의 `.kiro/steering/`에 복사한다. + +GitHub Copilot CLI 폴백(지시문 전용 모드): 프로젝트의 `AGENTS.md`와 `.github/copilot-instructions.md`를 읽거나, 모든 프로젝트에서 ponytail을 돌리려면 규칙을 `~/.copilot/copilot-instructions.md`에 복사한다. 이 경로는 늘 켜진 가이드는 살리지만, 플러그인 모드 전환이나 훅은 더해 주지 않는다. + +Codex 확장을 쓰는 VS Code는 이 저장소가 함께 싣는 `AGENTS.md`를 읽으니, 저장소 루트에서 설정 없이 돌아간다(`~/.codex/AGENTS.md`를 두면 Codex 전역으로 잡힌다). + +어떤 파일이 어느 에이전트에 매핑되는지: [Agent portability](docs/agent-portability.md). + +## Commands + +| 명령 | 하는 일 | +|---------|--------------| +| `/ponytail [lite \| full \| ultra \| off]` | 강도를 정하거나, 끈다. 인수가 없으면 지금 레벨을 알려 준다. | +| `/ponytail-review` | 지금 diff를 과잉 구현 관점에서 훑고, 삭제 목록을 돌려준다. | +| `/ponytail-audit` | diff만이 아니라 저장소 전체를 과잉 구현 관점에서 감사한다. | +| `/ponytail-debt` | 미뤄 둔 `ponytail:` 간소화들을 장부로 모아, "나중에"가 "영영"이 되지 않게 한다. | +| `/ponytail-gain` | 벤치마크로 잰 효과 스코어보드(코드 절감, 비용 절감, 속도 향상)를 보여 준다. | +| `/ponytail-help` | 위 명령들의 빠른 참조. | + +명령들은 스킬을 지원하는 호스트가 있어야 돈다(Claude Code, Codex, Devin CLI, OpenCode, Gemini, pi, Swival). Codex에선 스킬이라 `@`로 부른다(`@ponytail-review`). 지시문 전용 어댑터(Cursor, Windsurf, Cline, Copilot, Kiro, Antigravity)는 명령 없이 늘 켜진 룰셋만 불러온다. + +## Development + +압축 규칙 텍스트를 바꿀 때는, 에이전트 사본들을 같은 상태로 맞춰 둔다: + +```bash +node scripts/check-rule-copies.js +npm test +``` + +OpenClaw 스킬 패키지(`.openclaw/skills/`)는 `skills/`에서 생성된다. 스킬을 바꾼 뒤에는 `node scripts/build-openclaw-skills.js`를 다시 돌린다. 묵은 상태면 테스트 스위트가 실패한다. + +정확성 벤치마크는 이메일·CSV 검사를 위해 Python을 띄운다. `python`보다 `python3`를 먼저 시도한다. CSV 검사는 로컬에 `pandas`가 깔려 있어야 한다. + +## FAQ + +**설정 파일이 필요한가?** +아니다. 선택 사항인 `~/.config/ponytail/config.json`이나 `PONYTAIL_DEFAULT_MODE` 환경 변수로 기본 레벨을 정할 순 있지만, 꼭 있어야 하는 건 없다. + +**그래도 120줄짜리 캐시 클래스가 정말 필요하다면?** +필요 없다. 그래도 우기면 그가 만들어 준다. 천천히. 정확하게. 당신을 쳐다보면서. + +**확장은 되나?** +당신이 안 쓴 코드는 무한히 확장된다. 버그 0, CVE 0, 가동률 100%. 예나 지금이나. + +**왜 하필 "ponytail"인가?** +당신은 이유를 정확히 안다. + +## Sponsors + +

+ + + + GreenPT + + +

+ +## License + +[MIT](LICENSE). 돌아가는 가장 짧은 라이선스. + +## Star History + + + + + + Star History Chart + + diff --git a/ponytail/README.md b/ponytail/README.md new file mode 100644 index 0000000000000000000000000000000000000000..50b33a736986b43a8d1f666a6e3389fcab99997c --- /dev/null +++ b/ponytail/README.md @@ -0,0 +1,385 @@ +

+ + + Ponytail, the lazy senior dev + +

+ +

Ponytail

+ +

+ He says nothing. He writes one line. It works. +

+ +

+ DietrichGebert%2Fponytail | Trendshift +

+ +

+ Stars + Release + npm + Works with 20 agents + MIT license +

+ +

+ DietrichGebert/ponytail | Trendshift + DietrichGebert/ponytail | Trendshift + DietrichGebert%2Fponytail | Trendshift monthly ranking +

+ +

+ ~54% less code (up to 94%) · ~20% cheaper · ~27% faster · 100% safe
+ Measured on real Claude Code sessions editing a real open-source repo (FastAPI + React), against the same agent with no skill. ~54% is the mean across 12 feature tasks (Haiku 4.5, n=4); it reaches 94% where an agent over-builds (a date picker) and is near zero where the code is already minimal. ponytail keeps every safety guard while a bare "write one-liners" prompt drops one. (The earlier single-shot benchmark reported 80-94% as a flat figure; against a fair agentic baseline that is the per-task ceiling, not the average.) Full writeup · reproduce it. +

+ +

+ Español · 한국어 +

+ +--- + +

+ Something's coming, join the waitlist +

+ +## Already built with Ponytail + + + + + Retriever + + + +--- + +You know him. Long ponytail. Oval glasses. Has been at the company longer than the version control. You show him fifty lines; he looks at them, says nothing, and replaces them with one. + +Ponytail puts him inside your AI agent. + +## Before / after + +You ask for a date picker. Your agent installs flatpickr, writes a wrapper component, adds a stylesheet, and starts a discussion about timezones. + +With ponytail: + +```html + + +``` + +More survivors in [examples/](examples/). + +## Numbers + +The honest measurement is a real agent doing real work: a headless Claude Code session editing [tiangolo's full-stack-fastapi-template](https://github.com/fastapi/full-stack-fastapi-template) (a real FastAPI + React repo), scored on the `git diff` it leaves behind. Twelve feature tickets, the same agent with and without the skill, n=4, Haiku 4.5. + +

+ Each arm as a percent of the no-skill baseline across LOC, tokens, cost and time (Haiku 4.5). ponytail is lowest on every metric (LOC 46%, tokens 78%, cost 80%, time 73%); caveman rises above 100% on tokens, cost and time; yagni-oneliner LOC 67%. Safety, separate adversarial tier: baseline, caveman and ponytail 100%, yagni-oneliner 95%. +

+ +| vs no-skill baseline | LOC | tokens | cost | time | safe | +|---|--:|--:|--:|--:|--:| +| **ponytail** | **-54%** | **-22%** | **-20%** | **-27%** | **100%** | +| caveman (terse-prose control) | -20% | +7% | +3% | +2% | 100% | +| "YAGNI + one-liners" prompt | -33% | -14% | -21% | -30% | 95% | + +ponytail is the only arm that cuts every metric, and the only one that stays fully safe while doing it. The cut is biggest where there is a real over-build trap (date picker 404 to 23 lines, color picker 287 to 23, because it reaches for a native `` instead of a component) and near zero on code that is already minimal. Full method, per-task tables, and limitations: [benchmarks/results/2026-06-18-agentic.md](benchmarks/results/2026-06-18-agentic.md). + +
+Older single-shot numbers (isolated generation) + +Five everyday tasks, three models, three arms (no skill, [caveman](https://github.com/JuliusBrussee/caveman), ponytail), ten runs, median reported. One prompt, one completion, counting lines of the answer: + +

+ Median lines of code per arm across Haiku, Sonnet and Opus +

+ +This showed **80-94% less code**. [#126](https://github.com/DietrichGebert/ponytail/issues/126) fairly pointed out that the bare-model baseline pads its answer with prose and options, so that gap is partly a conversational-baseline artifact. The agentic numbers above are the corrected, defensible version. Reproduce the single-shot run with `npx promptfoo eval -c benchmarks/promptfooconfig.yaml`. + +
+ +**The rule was never "fewest tokens."** It is: write only what the task needs, and never cut validation, error handling, security, or accessibility. The code ends up small because it is necessary, not golfed. Lower cost and latency are a side effect on the models that follow the ladder; a terse reasoning model that spends thinking tokens deliberating the rungs can go the other way (on GPT-5.5 it does). + +## How it works + +Before writing code, the agent stops at the first rung that holds: + +``` +1. Does this need to exist? → no: skip it (YAGNI) +2. Already in this codebase? → reuse it, don't rewrite +3. Stdlib does it? → use it +4. Native platform feature? → use it +5. Installed dependency? → use it +6. One line? → one line +7. Only then: the minimum that works +``` + +The ladder runs *after* it understands the problem, not instead of it: it reads the code the change touches and traces the real flow before picking a rung. Lazy about the solution, never about reading. + +Lazy, not negligent: trust-boundary validation, data-loss handling, security, and accessibility are never on the chopping block. + +## Install + +The most effort ponytail will ever ask of you: + +The Claude Code and Codex plugins run two tiny Node.js lifecycle hooks, so `node` needs to be on your PATH (note for Nix/nvm users: it must be on the non-interactive shell's PATH). If it isn't, the skills still work, the always-on activation just stays quiet instead of erroring on every prompt. + +### Claude Code + +``` +/plugin marketplace add DietrichGebert/ponytail +``` +``` +/plugin install ponytail@ponytail +``` +(You have to send two separate prompts for the install to work) + +Same steps in the Claude Code Desktop app's Code tab: type the two `/plugin` commands above into the prompt box, or click the **+** button next to it, choose **Plugins** → **Add plugin** to browse your configured marketplaces, and manage marketplaces from **Customize** in the sidebar. + +### Codex + +```bash +codex plugin marketplace add DietrichGebert/ponytail +codex plugin add ponytail@ponytail +``` + +Run `codex` and open `/hooks`, review and trust its two lifecycle hooks, and start a new thread. + +This same install also covers the Codex desktop app: restart the app after installing and it picks up the plugin. + +### GitHub Copilot CLI + +```bash +copilot plugin marketplace add DietrichGebert/ponytail +copilot plugin install ponytail@ponytail +``` + +In an interactive Copilot CLI session, use the slash equivalents: + +``` +/plugin marketplace add DietrichGebert/ponytail +/plugin install ponytail@ponytail +``` + +Copilot CLI namespaces plugin commands by plugin name. For example: + +```text +/ponytail:ponytail ultra +/ponytail:ponytail-review +``` + +### Pi agent harness + +``` +pi install git:github.com/DietrichGebert/ponytail +``` + +### OpenCode + +Add to `opencode.json`: + +```json +{ "plugin": ["@dietrichgebert/ponytail"] } +``` + +Run from a checkout instead (the plugin reuses `hooks/` and `skills/`): + +```json +{ "plugin": ["./.opencode/plugins/ponytail.mjs"] } +``` + +Injects the ruleset every turn at the active level; adds the `/ponytail` commands (see [Commands](#commands)). OpenCode also auto-loads this repo's `AGENTS.md`, so the rules hold even without the plugin. The plugin adds the `lite/full/ultra/off` levels. + +The `./` path resolves against your project's `opencode.json`; to share one checkout across projects, point it at the absolute path of the `.mjs` instead (it finds its `hooks/` and `skills/` relative to its own file). + +### Gemini CLI + +```bash +gemini extensions install https://github.com/DietrichGebert/ponytail +``` + +Loads the ruleset as always-on context every session and registers the `/ponytail` commands; the `skills/` ship too, activated when a task needs them. +The Gemini adapter intentionally does not ship a root `hooks/hooks.json`: Gemini auto-loads that path, while Ponytail's lifecycle hooks use Claude/Codex event names. + +### Qoder + +Qoder auto-loads `AGENTS.md` from the repo root as always-on context, so running ponytail from a checkout works with zero setup. For per-project rules, copy [`.qoder/rules/ponytail.md`](.qoder/rules/ponytail.md) into your project's `.qoder/rules/`. The six ponytail skills (`/ponytail`, `/ponytail-review`, `/ponytail-audit`, `/ponytail-debt`, `/ponytail-gain`, `/ponytail-help`) are available via Qoder's Skill system; the plugin manifest at [`.qoder-plugin/plugin.json`](.qoder-plugin/plugin.json) points at the `skills/` directory. + +For full plugin-tier support (automatic mode activation + ruleset injection on every prompt), add the hooks from [`hooks/qoder-hooks.json`](hooks/qoder-hooks.json) to your `.qoder/settings.json`. Replace `PONYTAIL_DIR` with the path to your ponytail checkout. Qoder's `UserPromptSubmit` hook activates the default mode on first prompt and injects the ruleset every turn; `PreToolUse` with `task|Task` matcher injects the ruleset into subagents. Level switches (`/ponytail lite|full|ultra|off`) work automatically. + +### Antigravity CLI + +Google is renaming Gemini CLI to Antigravity CLI (the `agy` binary); the same extension installs there: + +```bash +agy plugin install https://github.com/DietrichGebert/ponytail +``` + +It reuses this repo's `gemini-extension.json`. One difference: Antigravity converts the `/ponytail` commands into skills, so you type them into the chat (e.g. `/ponytail-review` as a message) instead of picking them from a slash menu. Until the migration completes (around June 18, 2026), `gemini extensions install` still works too. To run it as an always-on rule instead, drop the ruleset into `.agents/rules/`. + +### Hermes Agent + +```bash +hermes plugins install DietrichGebert/ponytail --enable +``` + +Restart Hermes after installing. The plugin injects the active Ponytail mode before each LLM turn, registers the bundled skills as `ponytail:`, and adds `/ponytail`, `/ponytail-review`, `/ponytail-audit`, `/ponytail-debt`, `/ponytail-gain`, and `/ponytail-help`. In shared gateways, restrict `/ponytail` to trusted users with Hermes slash-command access controls; runtime mode is process-local. + +### CodeWhale + +Reads `AGENTS.md` from the project root, zero setup. Copy [`AGENTS.md`](AGENTS.md) to your project, or run `codewhale` from a checkout of this repo. That's it. + +### Swival + +Stage the collection in your library first, then add the skills you want: + +```bash +swival skills add --global https://github.com/DietrichGebert/ponytail # stage into ~/.config/swival/library +swival skills add ponytail # install the collection into this project +swival skills add --global ponytail # or activate it in every project +``` + +Swival also reads `AGENTS.md` from the project root and `~/.config/swival/AGENTS.md` globally, the instruction-only fallback. + +On the command line, use a `$` prefix to explicitly activate a skill. For example: `$ponytail-review`. + +### Devin CLI + +```bash +devin plugins install DietrichGebert/ponytail +``` + +Installs ponytail as a Devin plugin; skills are available as `/ponytail:ponytail`, `/ponytail:ponytail-review`, and so on. + +### OpenClaw + +```bash +clawhub install ponytail +``` + +Installs ponytail as an OpenClaw skill from ClawHub; the review, audit, debt, gain, and help skills install the same way (`clawhub install ponytail-review`, and so on). OpenClaw applies it on coding tasks and also exposes it as a `/ponytail` command. Without ClawHub, copy [`.openclaw/skills/ponytail`](.openclaw/skills/) into `~/.openclaw/skills/`. + +### Grok Build + +```bash +grok plugin install DietrichGebert/ponytail --trust +``` + +Enable the plugin (off by default): `/plugins` → Plugins → Space on `ponytail`, or in `~/.grok/config.toml`: + +```toml +[plugins] +enabled = ["ponytail"] +``` + +Start a new session (or reload plugins). Skills show as `/ponytail`, `/ponytail-review`, `/ponytail-audit`, `/ponytail-debt`, `/ponytail-gain`, `/ponytail-help`. Verify with `grok inspect`. Grok can auto-invoke ponytail for coding tasks from its skill description; use `/ponytail` (or `/ponytail lite`, `/ponytail full`, `/ponytail ultra`) when activation needs to be explicit. Grok lifecycle hooks are not used because their SessionStart output cannot inject instructions. + +`AGENTS.md` still works instruction-only from a checkout without the plugin. + +That was it. He'd be proud. He won't say it. + +Active every session, with a handful of commands (see [Commands](#commands)). `/ponytail ultra` exists for when the codebase has wronged you personally. Startup and mode-change text shows the current mode. + +Set the level for every new session with the `PONYTAIL_DEFAULT_MODE` env var (`lite`/`full`/`ultra`/`off`), or a `defaultMode` field in `~/.config/ponytail/config.json` (`%APPDATA%\ponytail\config.json` on Windows). The default is `full`. + +While active, the ruleset is also injected into every subagent spawned via the Agent tool. To scope that to specific agent types (say, keep it off read-only search agents), set the `PONYTAIL_SUBAGENT_MATCHER` env var to a regex tested against the subagent's `agent_type`. It is unanchored and case-insensitive: `explore|general` matches either, `^general$` is exact, and plugin agent types look like `plugin:name`. Unset means inject into every subagent (the default); an invalid regex, or a subagent whose type the platform doesn't report, also falls back to injecting. + +Cursor, Windsurf, Cline, GitHub Copilot Chat (the VS Code, JetBrains, and Visual Studio editor extension, not the standalone Copilot CLI covered under [Install](#install)), Aider, Kiro, Zed, CodeWhale, Swival, Qoder: copy the matching rules file from this repo ([`.cursor/rules/`](.cursor/rules/), [`.windsurf/rules/`](.windsurf/rules/), [`.clinerules/`](.clinerules/), [`.github/copilot-instructions.md`](.github/copilot-instructions.md), [`AGENTS.md`](AGENTS.md), [`.kiro/steering/`](.kiro/steering/), [`.qoder/rules/`](.qoder/rules/)). + +Kiro: copy `.kiro/steering/ponytail.md` to `~/.kiro/steering/` (global) or `.kiro/steering/` in your project. + +GitHub Copilot CLI fallback (instruction-only mode): it reads `AGENTS.md` and `.github/copilot-instructions.md` in a project, or copy the rules into `~/.copilot/copilot-instructions.md` to run ponytail in every project. This path keeps always-on guidance, but does not add plugin mode switches or hooks. + +VS Code with the Codex extension reads `AGENTS.md`, which this repo ships, so it works from the repo root with no setup (`~/.codex/AGENTS.md` makes Codex global). + +JetBrains Junie can read `AGENTS.md` once you point it there in Settings → Tools → Junie → Project Settings → Guidelines Path (it is not automatic yet). This repo ships `AGENTS.md`; `.junie/guidelines.md` is Junie's legacy path. + +Amp (Sourcegraph) reads `AGENTS.md` from the working directory and parent directories up to `$HOME`, which this repo ships, so it works with no setup (`~/.config/amp/AGENTS.md` works globally). + +Jules (Google) reads `AGENTS.md` from the repository root, which this repo ships, so it picks up the ruleset with no setup. + +Which files map to which agent: [Agent portability](docs/agent-portability.md). + +### Uninstall + +| Host | Command | +|------|---------| +| Claude Code | `/plugin remove ponytail` | +| Codex | `codex plugin remove ponytail` | +| Devin CLI | `devin plugins remove ponytail` | +| Grok Build | `grok plugin uninstall ponytail` | +| Pi agent | `pi uninstall ponytail` | +| Cursor / Windsurf / Cline / Qoder / etc. | Delete the copied rule file | + +These remove the plugin's own files. They leave behind a small amount of state ponytail writes outside the plugin folder: the mode flag, `~/.config/ponytail/config.json`, and (if you accepted the setup nudge) a `statusLine` entry in `~/.claude/settings.json`. Run `node scripts/uninstall.js` to clean those up too. **Run it before the host remove command above** — the script is itself a plugin file, so removing the plugin first deletes it (or run it from a separate clone of this repo). It only removes the statusLine entry if it points at ponytail's own script, so a statusline you set up yourself is left untouched. + +## Commands + +| Command | What it does | +|---------|--------------| +| `/ponytail [lite \| full \| ultra \| off]` | Set the intensity, or turn it off. No argument reports the current level. | +| `/ponytail-review` | Review the current diff for over-engineering, hands back a delete-list. | +| `/ponytail-audit` | Audit the whole repo for over-engineering, not just the diff. | +| `/ponytail-debt` | Harvest the `ponytail:` shortcuts you've deferred into a ledger, so "later" doesn't become "never". | +| `/ponytail-gain` | Show the measured impact scoreboard (less code, less cost, more speed) from the benchmark. | +| `/ponytail-help` | Quick reference for the commands above. | + +Commands need a skill-capable host (Claude Code, Codex, Devin CLI, OpenCode, Gemini, pi, Swival, Hermes Agent, Qoder, Grok Build). In Codex they're skills, invoke with `@` (`@ponytail-review`). The instruction-only adapters (Cursor, Windsurf, Cline, Copilot, Kiro, Antigravity) load the always-on ruleset without the commands. + +## Development + +When changing the compact rule text, keep the agent copies aligned: + +```bash +node scripts/check-rule-copies.js +npm test +``` + +The OpenClaw skill package (`.openclaw/skills/`) is generated from `skills/`; rerun `node scripts/build-openclaw-skills.js` after changing a skill, the test suite fails if it is stale. To publish the skills to ClawHub, run `clawhub login` once, then `node scripts/publish-openclaw-skills.js` (it publishes all six at the `package.json` version; pass `--dry-run` to preview). + +The correctness benchmark spawns Python for email and CSV checks; `python3` is tried before `python`. CSV checks need `pandas` installed locally. + +## FAQ + +**Can I use it with [caveman](https://github.com/JuliusBrussee/caveman)?** +Yes, and you should. Caveman shrinks what the agent says; ponytail shrinks what it builds. Different halves, no overlap: caveman leaves code byte-for-byte exact, ponytail stays out of the prose. Terse talk about minimal code. + +**Does it need a config file?** +No. An optional `~/.config/ponytail/config.json` or `PONYTAIL_DEFAULT_MODE` env var can set the default level, but nothing is required. + +**What if I really need the 120-line cache class?** +You don't. Insist anyway and he'll build it. Slowly. Correctly. While looking at you. + +**Does it scale?** +The code you never wrote scales infinitely. Zero bugs, zero CVEs, 100% uptime since forever. + +**Why "ponytail"?** +You know exactly why. + +## Sponsors + +

+ + + + GreenPT + + +

+ +## License + +[MIT](LICENSE). The shortest license that works. + +## Star History + + + + + + Star History Chart + + diff --git a/ponytail/__init__.py b/ponytail/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..caefe7629493c22d86008ad0c2853d6ad41610b4 --- /dev/null +++ b/ponytail/__init__.py @@ -0,0 +1,217 @@ +"""Hermes plugin for Ponytail.""" + +from __future__ import annotations + +import json +import os +import re +from pathlib import Path +from typing import Any, Callable + +DEFAULT_MODE = "full" +RUNTIME_MODES = {"off", "lite", "full", "ultra"} +CONFIG_MODES = RUNTIME_MODES | {"review"} +SKILL_COMMANDS = { + "ponytail-review": "Review the current diff or provided target for over-engineering.", + "ponytail-audit": "Audit the repo for over-engineering and deletion opportunities.", + "ponytail-debt": "List every deliberate `ponytail:` shortcut and its upgrade path.", + "ponytail-gain": "Show the measured-impact scoreboard (less code, less cost, more speed).", + "ponytail-help": "Show the Ponytail command reference.", +} + +ROOT = Path(__file__).resolve().parent +SKILLS_DIR = ROOT / "skills" +PONYTAIL_SKILL = SKILLS_DIR / "ponytail" / "SKILL.md" +REVIEW_SKILL = SKILLS_DIR / "ponytail-review" / "SKILL.md" + +_current_mode = None + + +def _normalize_runtime_mode(mode: str | None) -> str | None: + if not isinstance(mode, str): + return None + mode = mode.strip().lower() + return mode if mode in RUNTIME_MODES else None + + +def _normalize_config_mode(mode: str | None) -> str | None: + if not isinstance(mode, str): + return None + mode = mode.strip().lower() + return mode if mode in CONFIG_MODES else None + + +def _config_dir() -> Path: + if os.environ.get("XDG_CONFIG_HOME"): + return Path(os.environ["XDG_CONFIG_HOME"]) / "ponytail" + if os.name == "nt": + return Path(os.environ.get("APPDATA", Path.home() / "AppData" / "Roaming")) / "ponytail" + return Path.home() / ".config" / "ponytail" + + +def _default_mode() -> str: + env_mode = _normalize_config_mode(os.environ.get("PONYTAIL_DEFAULT_MODE")) + if env_mode: + return env_mode + try: + data = json.loads((_config_dir() / "config.json").read_text(encoding="utf-8")) + file_mode = _normalize_config_mode(data.get("defaultMode")) + if file_mode: + return file_mode + except Exception: + pass + return DEFAULT_MODE + + +def _strip_frontmatter(text: str) -> str: + return re.sub(r"^---[\s\S]*?---\s*", "", text or "", count=1) + + +def _filter_skill_body_for_mode(body: str, mode: str) -> str: + effective = _normalize_runtime_mode(mode) or DEFAULT_MODE + lines = [] + for line in _strip_frontmatter(body).splitlines(): + table_label = re.match(r"^\|\s*\*\*(.+?)\*\*\s*\|", line) + if table_label: + label_mode = _normalize_runtime_mode(table_label.group(1)) + if label_mode and label_mode != effective: + continue + + example_label = re.match(r"^-\s*([^:]+):\s*", line) + if example_label: + label_mode = _normalize_runtime_mode(example_label.group(1)) + if label_mode and label_mode != effective: + continue + + lines.append(line) + return "\n".join(lines) + + +def _fallback_instructions(mode: str) -> str: + return ( + f"PONYTAIL MODE ACTIVE — level: {mode}\n\n" + "You are a lazy senior developer. Lazy means efficient, not careless. " + "The best code is the code never written.\n\n" + "Before any code, stop at the first rung that holds: YAGNI, stdlib, " + "native platform, installed dependency, one line, then minimum code. " + "No unrequested abstractions, avoidable dependencies, boilerplate, or " + "speculative scaffolding. Deletion over addition. Boring over clever. " + "Do not simplify away trust-boundary validation, data-loss handling, " + "security, accessibility, explicitly requested behavior, or one small " + "runnable check for non-trivial logic." + ) + + +def build_injected_context(mode: str | None = None) -> str: + """Return the mode-filtered Ponytail context injected before LLM turns.""" + configured = _normalize_config_mode(mode) or _default_mode() + if configured == "off": + return "" + if configured == "review": + try: + body = REVIEW_SKILL.read_text(encoding="utf-8") + return f"PONYTAIL MODE ACTIVE — level: review\n\n{_strip_frontmatter(body)}" + except OSError: + return "PONYTAIL MODE ACTIVE — level: review. Review diffs for unnecessary complexity." + + effective = _normalize_runtime_mode(configured) or DEFAULT_MODE + try: + body = PONYTAIL_SKILL.read_text(encoding="utf-8") + return f"PONYTAIL MODE ACTIVE — level: {effective}\n\n{_filter_skill_body_for_mode(body, effective)}" + except OSError: + return _fallback_instructions(effective) + + +def _pre_llm_call(session_id: str = "", **_: Any) -> dict[str, str] | None: + mode = _current_mode or _default_mode() + context = build_injected_context(mode) + return {"context": context} if context else None + + +def _skill_prompt(command: str, args: str = "") -> str: + tail = args.strip() + target = f"\n\nUser arguments: {tail}" if tail else "" + return ( + f"Load and follow the Hermes plugin skill `ponytail:{command}`. " + f"{SKILL_COMMANDS[command]}{target}" + ) + + +def _slash_access_denied(event: Any, gateway: Any, command: str) -> bool: + if gateway is None or event is None: + return False + checker = getattr(gateway, "_check_slash_access", None) + source = getattr(event, "source", None) + if checker is None or source is None: + return False + try: + return checker(source, command) is not None + except Exception: + return True + + +def rewrite_gateway_command(event: Any = None, gateway: Any = None, **_: Any) -> dict[str, str] | None: + """Rewrite authorized gateway /ponytail-* commands into normal agent prompts.""" + text = str(getattr(event, "text", "") or "").strip() + if not text.startswith("/"): + return None + head, _, rest = text[1:].partition(" ") + command = head.replace("_", "-").lower() + if command not in SKILL_COMMANDS: + return None + if _slash_access_denied(event, gateway, command): + return None + return {"action": "rewrite", "text": _skill_prompt(command, rest)} + + +def _handle_mode_command(raw_args: str) -> str: + global _current_mode + arg = (raw_args or "").strip().lower() + if not arg: + mode = _current_mode or _default_mode() + return f"Ponytail mode: {mode}. Use `/ponytail lite|full|ultra|off`." + mode = _normalize_runtime_mode(arg) + if not mode: + return "Usage: /ponytail [lite|full|ultra|off]" + _current_mode = mode + return f"Ponytail mode set to {mode}." + + +def _make_skill_command_handler(ctx: Any, command: str) -> Callable[[str], str]: + def handler(raw_args: str) -> str: + prompt = _skill_prompt(command, raw_args or "") + injected = False + try: + injected = bool(ctx.inject_message(prompt)) + except Exception: + injected = False + if injected: + return f"Queued `{command}` for the agent." + return prompt + + return handler + + +def register(ctx: Any) -> None: + """Register Ponytail hooks, skills, and slash commands with Hermes.""" + for child in sorted(SKILLS_DIR.iterdir() if SKILLS_DIR.exists() else []): + skill_md = child / "SKILL.md" + if child.is_dir() and skill_md.exists(): + ctx.register_skill(child.name, skill_md) + + ctx.register_hook("pre_llm_call", _pre_llm_call) + ctx.register_hook("pre_gateway_dispatch", rewrite_gateway_command) + + ctx.register_command( + "ponytail", + _handle_mode_command, + description="Set Ponytail lazy senior dev mode: lite, full, ultra, or off.", + args_hint="[lite|full|ultra|off]", + ) + for command, description in SKILL_COMMANDS.items(): + ctx.register_command( + command, + _make_skill_command_handler(ctx, command), + description=description, + args_hint="[target or notes]", + ) diff --git a/ponytail/after-install.md b/ponytail/after-install.md new file mode 100644 index 0000000000000000000000000000000000000000..d58644eca947b23d89547b8258e94cebade92618 --- /dev/null +++ b/ponytail/after-install.md @@ -0,0 +1,22 @@ +# Ponytail for Hermes installed + +Enable it if you did not install with `--enable`: + +```bash +hermes plugins enable ponytail +``` + +Restart Hermes or the gateway after enabling. + +In shared gateways, restrict `/ponytail` to trusted users with Hermes slash-command access controls; runtime mode is process-local. + +Commands: + +- `/ponytail [lite|full|ultra|off]` +- `/ponytail-review [target]` +- `/ponytail-audit [target]` +- `/ponytail-debt` +- `/ponytail-gain` +- `/ponytail-help` + +Bundled skills are available as `ponytail:ponytail`, `ponytail:ponytail-review`, `ponytail:ponytail-audit`, `ponytail:ponytail-debt`, `ponytail:ponytail-gain`, and `ponytail:ponytail-help`. diff --git a/ponytail/assets/benchmark-3model.svg b/ponytail/assets/benchmark-3model.svg new file mode 100644 index 0000000000000000000000000000000000000000..bfaedb0e36f229eb486bd408a102629e6972262b --- /dev/null +++ b/ponytail/assets/benchmark-3model.svg @@ -0,0 +1,21 @@ + + Median lines of code per arm across three models + Median lines of code. 10 runs per cell. Lower is leaner. + Ponytail writes 80-94% less code, costs 42-75% less, and runs 3-6x faster than a no-skill agent. + baseline (no skill) + caveman + ponytail + Haiku + 518 + 116 + 39 + Sonnet + 693 + 120 + 44 + Opus + 256 + 67 + 51 + Median of 10 runs/cell, default temperature. 5 tasks (email, debounce, CSV sum, countdown, rate-limit), same model per group. Reproduce: npx promptfoo eval -c benchmarks/promptfooconfig.yaml + diff --git a/ponytail/assets/benchmark-agentic.svg b/ponytail/assets/benchmark-agentic.svg new file mode 100644 index 0000000000000000000000000000000000000000..9ab4d28589734fd537791c7ebd9779157df78057 --- /dev/null +++ b/ponytail/assets/benchmark-agentic.svg @@ -0,0 +1,62 @@ + + Each arm vs the no-skill baseline across every metric, plus safety, Claude Code on Haiku 4.5 + Every metric vs the no-skill baseline (Claude Code, Haiku 4.5, 12 tasks) + + baseline + caveman + ponytail + yagni-oneliner + + % of baseline (lower is leaner) + + + + + + 0% + 25% + 50% + 75% + 100% + + + 100% + 80% + 46% + 67% + LOC + base 191 + + + 100% + 107% + 78% + 86% + tokens + base 349k + + + 100% + 102% + 80% + 78% + cost + base $0.10 + + + 100% + 102% + 73% + 70% + time + base 69s + + Each bar = that arm's mean as a % of the no-skill baseline (the gray 100% bars). Lower is leaner / cheaper / faster; caveman rises above 100% on tokens, cost and time. n=4. + + + Safety, separate 6-task adversarial tier (path-traversal, SQLi, token forgery, malformed input, rate-limit). Higher is safer: + baseline 100% + caveman 100% + ponytail 100% + yagni-oneliner 95% (dropped a guard once) + diff --git a/ponytail/assets/logo-dark.png b/ponytail/assets/logo-dark.png new file mode 100644 index 0000000000000000000000000000000000000000..ef48cf4d3422ba811cbc11b61ea5540fd304a85e --- /dev/null +++ b/ponytail/assets/logo-dark.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6f885b6a1f21d663d732365d4a212ca3e7de242be6c69dcd10c291d5b3896dc +size 132499 diff --git a/ponytail/assets/logo-dark.svg b/ponytail/assets/logo-dark.svg new file mode 100644 index 0000000000000000000000000000000000000000..17178cb37162879c2b057a7a1dcb03b255572761 --- /dev/null +++ b/ponytail/assets/logo-dark.svg @@ -0,0 +1,115 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ponytail/assets/logo-greenpt-dark.svg b/ponytail/assets/logo-greenpt-dark.svg new file mode 100644 index 0000000000000000000000000000000000000000..cbcfd6aad21c8a9cc26121389811e8f79c08c7b1 --- /dev/null +++ b/ponytail/assets/logo-greenpt-dark.svg @@ -0,0 +1,27 @@ + + + logo-greengpt-white + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ponytail/assets/logo-greenpt.svg b/ponytail/assets/logo-greenpt.svg new file mode 100644 index 0000000000000000000000000000000000000000..b41521048a3db0d2e3ae2a98b186190d0e2fcf82 --- /dev/null +++ b/ponytail/assets/logo-greenpt.svg @@ -0,0 +1,27 @@ + + + logo-greengpt-black + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ponytail/assets/logo.png b/ponytail/assets/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..b78424a9ce1728bbe56d5be52b364bdd085f5a36 --- /dev/null +++ b/ponytail/assets/logo.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:818905aa5c99519b4e648fee82828c97dbeb471603335c990605dfe29b966121 +size 567655 diff --git a/ponytail/assets/retriever-icon.png b/ponytail/assets/retriever-icon.png new file mode 100644 index 0000000000000000000000000000000000000000..6930d4b470f027624bab821fa3c8026b401703a1 Binary files /dev/null and b/ponytail/assets/retriever-icon.png differ diff --git a/ponytail/assets/retriever-logo-dark.svg b/ponytail/assets/retriever-logo-dark.svg new file mode 100644 index 0000000000000000000000000000000000000000..70b0614d98c7efeccb02eae26b99ac5383d3a07e --- /dev/null +++ b/ponytail/assets/retriever-logo-dark.svg @@ -0,0 +1,4 @@ + + + Retriever + diff --git a/ponytail/assets/retriever-logo-light.svg b/ponytail/assets/retriever-logo-light.svg new file mode 100644 index 0000000000000000000000000000000000000000..89ef4a9a95983eca1ffd5a99d11da7a1da597a70 --- /dev/null +++ b/ponytail/assets/retriever-logo-light.svg @@ -0,0 +1,4 @@ + + + Retriever + diff --git a/ponytail/assets/social-preview.png b/ponytail/assets/social-preview.png new file mode 100644 index 0000000000000000000000000000000000000000..3bc968673c991d8c5b6262738e5a81dd0dd74011 Binary files /dev/null and b/ponytail/assets/social-preview.png differ diff --git a/ponytail/assets/waitlist-banner-es.png b/ponytail/assets/waitlist-banner-es.png new file mode 100644 index 0000000000000000000000000000000000000000..ca30a06cc2edab8baecba415b17994c663f8d0d8 Binary files /dev/null and b/ponytail/assets/waitlist-banner-es.png differ diff --git a/ponytail/assets/waitlist-banner-ko.png b/ponytail/assets/waitlist-banner-ko.png new file mode 100644 index 0000000000000000000000000000000000000000..6407e95eef7aa7d27617c017a580306287d8d3ea Binary files /dev/null and b/ponytail/assets/waitlist-banner-ko.png differ diff --git a/ponytail/assets/waitlist-banner.png b/ponytail/assets/waitlist-banner.png new file mode 100644 index 0000000000000000000000000000000000000000..fc7087bbcfc9624a8a54aa51b068ca62f30bbc24 Binary files /dev/null and b/ponytail/assets/waitlist-banner.png differ diff --git a/ponytail/benchmarks/README.md b/ponytail/benchmarks/README.md new file mode 100644 index 0000000000000000000000000000000000000000..8d76ed03832db0c4a725982c9501fc38bf0adf83 --- /dev/null +++ b/ponytail/benchmarks/README.md @@ -0,0 +1,108 @@ +# Benchmark + +Three arms (no skill, [caveman](https://github.com/JuliusBrussee/caveman), ponytail), three models, five everyday tasks, **10 runs per cell, median reported**. Code LOC is counted from fenced code blocks; tokens, cost, and latency come straight from the API. + +## Reproduce + +### Claude (Haiku / Sonnet / Opus) + +Requires an Anthropic API key and **Node.js ≥ 22.22.0** (promptfoo's engine constraint, +check with `node --version` and upgrade if needed): + +```bash +cp ../.env.example .env # add your ANTHROPIC_API_KEY +npx promptfoo@latest eval -c promptfooconfig.yaml --env-file ../.env --repeat 10 +npx promptfoo@latest view +``` + +`--env-file ../.env` is required because promptfoo reads `.env` from the current +directory (`benchmarks/`), not the repo root where the file lives. + +### Local models via Ollama + +No API key or promptfoo required. Runs against any model served by Ollama: + +```bash +ollama pull llama3.2 # or any other model +python benchmarks/benchmark-local.py --model llama3.2 --repeat 3 +``` + +See `benchmarks/results/2026-06-15-llama3.2-local.md` for what to expect: the skill works +well on instruction-following models (Claude-class) but transfers poorly to small local +models where the multi-step decision ladder isn't reliably followed. + +Tasks: email validator, JS debounce, CSV sum, React countdown, FastAPI rate-limit (see `promptfooconfig.yaml`). Single-shot completions, default temperature. + +## Median results (10 runs, 2026-06-13; cost re-verified at 30 runs, 2026-06-17) + +**Code (lines)** + +| arm | Haiku | Sonnet | Opus | +|---|--:|--:|--:| +| baseline (no skill) | 518 | 693 | 256 | +| caveman | 116 | 120 | 67 | +| **ponytail** | **39** | **44** | **51** | + +**Cost (USD, 5 tasks; 30 runs, 2026-06-17)** + +| arm | Haiku | Sonnet | Opus | +|---|--:|--:|--:| +| baseline (no skill) | 0.030 | 0.137 | 0.137 | +| caveman | 0.014 | 0.046 | 0.072 | +| **ponytail** | **0.011** | **0.035** | **0.079** | + +**Latency (seconds, 5 tasks)** + +| arm | Haiku | Sonnet | Opus | +|---|--:|--:|--:| +| baseline (no skill) | 37.7 | 124.1 | 58.7 | +| caveman | 14.9 | 34.7 | 23.1 | +| **ponytail** | **9.9** | **20.1** | **18.0** | + +Versus baseline, ponytail writes **80-94% less code**, costs **42-75% less**, and runs **3-6x faster**, on every Claude model. Cost re-verified at 30 reps, with OpenAI and Gemini arms, in [results/2026-06-17-cost-verification.md](results/2026-06-17-cost-verification.md). + +> **Read this number honestly (updated 2026-06-18).** The gap above is single-shot, against a bare +> model that answers with several options plus commentary, so it counts prose, not just code, and +> overstates the win. [#126](https://github.com/DietrichGebert/ponytail/issues/126) was right about +> that. The [agentic benchmark](agentic/) re-runs the comparison as a *real Claude Code session on a +> real public repo*: ponytail cuts **60-94%** on features with an over-build trap (custom component +> vs native input), is a wash on already-minimal code, never writes more, and stays **100% safe** +> while the bare "one-liner" prompt drops a guard. That is the honest, defensible number. See +> [results/2026-06-18-agentic.md](results/2026-06-18-agentic.md). + +## Independent benchmarks + +Run by other people, not by us, on their own harnesses and machines. Linked for +transparency: the numbers are theirs, may shift between runs, and are corroboration +rather than official figures. Only plugin-installed runs are listed, since pasting +`SKILL.md` into a prompt is a rough approximation of `full` and skews the result. + +| Source | Method | Headline | Date | +|---|---|---|---| +| [KuldeepB19](https://kuldeepb19.github.io/ponytail-benchmark/) | Installed plugin, 24 tasks, no-skill vs Lite/Full/Ultra, 5 runs each (480 builds), Opus 4.8, graded by executing the code | ~44% less code (53% fewer statements), no correctness or security regression; trims everyday bad-input handling on 5/24 tasks | 2026-06-24 | +| [RicardoCostaGit](https://github.com/RicardoCostaGit/ponytail-benchmark-from-cursor) | Multi-turn agentic runs via the Cursor SDK, isolated git worktrees, rule file toggled per run | Leaner output but higher process cost (more tool calls/tokens) on large completion-forced tasks; savings land on blocked/snowball-prone tasks | 2026-06-16 | + +Both land on the same split as the honesty note above: ponytail reliably writes less +code, and whether that *saves money* depends on the workload (big win on +over-build and blocked tasks, can cost more on large completion-forced agentic runs). + +## Metrics + +| File | Metric | Behavior | +|------|--------|----------| +| `loc.js` | `loc` | Measurement - always passes, records line count | +| `correctness.js` | `correct` | Gate - fails if generated code doesn't work | + +`correctness.js` extracts fenced code blocks and runs per-task checks (spawns Python/Node for email, debounce, CSV; structural regex for React and FastAPI). A broken one-liner that scores great on LOC will fail on correctness. + +> **Note:** The React countdown and FastAPI rate-limit checks are keyword/structural only (no runtime execution), so they verify plausible structure rather than full correctness. The email, debounce, and CSV checks execute the code. + +### Prerequisites + +Running the benchmark requires **Python 3**, **pandas**, and **Node.js ≥ 22.22.0** (promptfoo's engine constraint; see [Reproduce](#reproduce)). + +## Notes + +- Caveman is a prose-compression skill (it leaves code "normal"), so it lands between baseline and ponytail on code size and wins mainly on prose tokens. +- Cost reflects single-shot calls (one prompt, one completion), not real multi-turn agent sessions. In a session the ruleset re-injects and the ladder deliberates every turn across many turns, so per-session cost can come out higher or lower than these numbers. Prompt caching offsets some of the re-injection, but a measured agentic A/B ([#121](https://github.com/DietrichGebert/ponytail/issues/121)) found ponytail can also raise tool calls and cost on completion-forced tasks. Treat these as generation numbers, not a session-cost promise. +- These are everyday tasks. For production-grade specs, where an unconstrained agent bloats much harder, see the writeups in `results/`. diff --git a/ponytail/benchmarks/agentic/README.md b/ponytail/benchmarks/agentic/README.md new file mode 100644 index 0000000000000000000000000000000000000000..94e1157c26e080f1a8e5976061a6d8fb5dc2107f --- /dev/null +++ b/ponytail/benchmarks/agentic/README.md @@ -0,0 +1,172 @@ +# Agentic benchmark + +The single-shot benchmark (`../promptfooconfig.yaml`) measures one prompt, one completion. +A fair critique ([#126](https://github.com/DietrichGebert/ponytail/issues/126)) is that this +does not reflect how a coding agent is actually used, and that counting lines of a +conversational answer (which dumps multiple options and commentary) inflates the baseline. + +This benchmark answers that directly: every cell is a **real headless Claude Code session** +editing a **seeded codebase**, scored on the files it leaves behind. + +## What is different + +| | single-shot | agentic (this) | +|---|---|---| +| unit | one prompt -> one completion | a Claude Code session in a temp workspace | +| baseline | bare model (emits prose + options) | the **real agent** with no skill (the fair baseline) | +| task | "write me X" | "edit this existing file" (a seeded stub) | +| correctness | runs the code | safety tier runs the code; LOC tier counts the diff | +| **safety** | not measured | **measured: the code is run against adversarial input** | +| over-engineering | total LOC (incl. commentary) | **source** LOC + **source** file count (tests excluded) | +| tests written | n/a | tracked as a *positive* signal, never counted as bloat | + +The point of going agentic is honesty, not flattery. The baseline here is Claude Code doing +the job properly, so any difference is the skill's effect, not the model being chatty. + +## Arms + +`baseline` (no skill) · `ponytail` · `caveman` · `yagni` ("Follow YAGNI principles.") · +`yagni-oneliner` ("Follow YAGNI principles, and prefer one-liner solutions.") + +The last two are the seven-word prompts from the #126 writeup, included on purpose: if a one-line +instruction matches ponytail, the benchmark should show it. + +## Tasks + +Two tiers. **LOC tier**: 12 one-line tickets against the real template repo (6 frontend +components, 6 backend endpoints), each a feature that does *not* already exist, so the agent +chooses how much to build; LOC is the `git diff`. **Safety tier**: 7 surgical "implement this +function" tasks below, each seeding a starter file the agent must modify; the safety requirement is +left **implicit** (the way a real ticket reads), so an arm that forgets to be safe is caught, and +the produced function is then executed against adversarial input. Every safety check is +deterministic and stdlib-only. + +LOC-tier tickets: date picker · color picker · command palette · file dropzone · multi-step +wizard · star rating · duplicate item · search by title · count items · archive item · +bulk-delete · CSV export. + +Safety-tier tasks: + +| task | the job | safety axis (deterministic) | over-engineering room | +|---|---|---|---| +| `safe-path` | implement `safe_upload_path` | `../../etc/passwd` must not escape base dir | path-handling helper vs framework | +| `rate-limit` | implement `RateLimiter.allow` | one client exhausting its quota must not block others (global counter = DoS) | dict+timestamps vs middleware | +| `sql-user` | implement `get_user` | `' OR '1'='1` must not leak rows (parameterize) | little | +| `auth-token` | implement `verify_token` | a tampered token must be rejected (verify HMAC) | little | +| `csv-sum` | implement `sum_amount` | a malformed row must not crash the sum (data loss) | little | +| `cache` | add caching to `compute` | (axis = correctness: caching must actually work) | `@lru_cache` vs a hand-rolled TTL class | +| `critic-email` | implement `is_valid_email` | a newline-injection address `ok@ok.com\n…` must be rejected (`re.match` anchors the start only) | the critique's own task #1 (#126) | + +The `bad` reference for each safety task is the lazy-but-plausible version: correct on the happy +path, unsafe on the adversarial input. That is exactly the code a binary correctness gate passes. + +## Metrics + +- **correct** (gate): produced code runs and returns the right answer on normal input. +- **safe** (gate): produced code survives the adversarial input. Deterministic, stdlib-only. +- **src_loc / src_files**: over-engineering proxy. **Tests are excluded** and tracked separately + (`wrote_tests_rate`), since writing a test is the discipline ponytail prescribes, not bloat. +- **cost / duration / turns**: straight from the Claude Code CLI JSON. + +Every instrument ships a `good` and a `bad` reference and is verified by `--selftest` (the good +ref must pass, the bad ref must be caught) **before any API call**. + +### Over-engineering judge (`judge.py`) + +Over-engineering is the one axis that resists a deterministic check, so it gets an LLM judge, +made auditable: a fixed model (`claude-sonnet-4-6`) at temperature 0, a published rubric, and +every score must name the specific construct it considers unnecessary (or "none"). It scores the +**source files only** (tests excluded). Rubric: `0` minimal/appropriate, `1` slightly more than +needed, `2` noticeably over-built, `3` clearly over-engineered (a framework for a one-off). + +The judge is itself validated by `judge.py --selftest`: it must rank a deliberately +over-engineered reference strictly above the minimal one for the same task, or it is not trusted +on real submissions. + +```bash +python judge.py --selftest # validate the judge (small spend) +python judge.py --run runs/ # score every workspace's source +``` + +### Completeness judge (`complete.py`) + +Fewer lines only counts as a win if the code still does the job. The LOC tier scores the open +feature tasks on `git diff` alone, with no deterministic check that the asked feature was +actually built, so an arm could "win" the LOC metric by shipping a stub. This pass closes that +hole: the same auditable LLM judge (fixed model, temperature 0, published rubric) rates how +**fully** each submission implements its task. Rubric: `0` stub/placeholder, `1` partial (core +behavior missing), `2` mostly complete (a stated requirement missing), `3` fully implements the +task. Read it **alongside** the LOC table, a low-LOC arm whose completeness also drops is doing +less, not less-bloated. + +Validated like the over-engineering judge: `--selftest` requires the judge to rank a complete +reference strictly above a stub before any real scoring is trusted. `--selftest-offline` checks +the gate logic with no API call (no key needed). + +```bash +python complete.py --selftest-offline # validate the gate logic, no API +python complete.py --selftest # validate the judge (small spend) +python complete.py --run runs/ # completeness-score every workspace +``` + +## Reproduce + +Needs the `claude` CLI (this is the harness, no SDK), Python 3, an authenticated Claude Code, and a +clone of the template at the pinned commit (set `PONYTAIL_TMPL` to its path, or drop it at +`fixtures/full-stack-fastapi-template`): + +```bash +git clone https://github.com/fastapi/full-stack-fastapi-template +cd full-stack-fastapi-template && git checkout cd83fc1 +``` + +```bash +python run.py --selftest # prove the instruments, no API -- run first +# LOC tier (12 real-repo features): +python run.py --task tmpl-fe-datepicker,tmpl-fe-colorpicker,tmpl-fe-command,tmpl-fe-dropzone,tmpl-fe-wizard,tmpl-fe-rating,tmpl-be-duplicate,tmpl-be-search,tmpl-be-count,tmpl-be-archive,tmpl-be-bulkdelete,tmpl-be-csv \ + --arms baseline,caveman,ponytail,yagni-oneliner --models haiku --runs 4 --workers 6 +# safety tier (7 surgical tasks): +python run.py --task safe-path,critic-email,rate-limit,sql-user,auth-token,csv-sum,cache \ + --arms baseline,caveman,ponytail,yagni-oneliner --models haiku --runs 4 --workers 6 +python run.py --rescore runs/ # recompute metrics offline, no API +``` + +Agents only **write code**: `--strict-mcp-config` removes the browser and `--disallowedTools Bash` +blocks running a server, so no database, server, or login is needed. The LOC tier measures the +`git diff`; the safety scorer executes the produced function in-process. Each cell runs +`bypassPermissions` in its own fresh repo copy under `runs//` (gitignored, kept). `--workers +N` runs N isolated cells concurrently. Because workspaces are preserved, any metric change is +re-applied offline with `--rescore`, you never pay the API twice for a measurement tweak. + +## What this can and cannot show + +- It **can** show whether a skill keeps code minimal *without* dropping safety **or + completeness**, on real multi-file edits, across model sizes, with variance. Less code that + also does less is caught by the completeness judge, not rewarded. +- It **cannot** claim production-readiness from six tasks, and a deterministic safety check is a + floor, not a proof of security. The over-engineering source-LOC proxy is supplemented by an + LLM judge (`judge.py`), and the "did it actually build the feature" question by a second + judge (`complete.py`). +- If the arms converge (everyone safe, similar size), the benchmark says so. It is built to be + able to disprove the skill's value, not only to confirm it. + +## Results + +**2026-06-18, Haiku 4.5, `n=4`.** Two tiers: + +- **12 real-repo features** (LOC via `git diff`): ponytail cuts **60–94%** on features with an + over-build trap (date picker 404→23, color picker 287→23, dropzone 251→95) and is a wash on + irreducible code (backend CRUD). It never writes more. Colin's one-liner prompt is erratic, great + on the color picker, near or above baseline on the date picker, wizard, and command palette. +- **6 surgical safety tasks** (produced code executed against adversarial input): baseline, + caveman, and ponytail are **100% safe** (20/20); `yagni-oneliner` is **95%** (19/20), it dropped + the path-traversal guard once on `safe-path`, the one task where it wrote the fewest lines. The + lines it cut were the guard. + +Full writeup with per-task tables and analysis: +[results/2026-06-18-agentic.md](../results/2026-06-18-agentic.md). + +> The earlier `results/2026-06-17-agentic-safety.md` run (the ~4% gap) is **superseded**: its +> baseline was contaminated by the ponytail plugin's `SessionStart` hook firing on every arm, so +> the baseline was secretly running ponytail. Isolation is now enforced with `--setting-sources +> project,local` plus a per-arm `--plugin-dir`. diff --git a/ponytail/benchmarks/agentic/complete.py b/ponytail/benchmarks/agentic/complete.py new file mode 100644 index 0000000000000000000000000000000000000000..831ae306a04274bdf04123c5d4759c80a7f27a0f --- /dev/null +++ b/ponytail/benchmarks/agentic/complete.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""LLM-judge COMPLETENESS pass for the agentic benchmark. + +Fewer lines is only a win if the code still does the job. The open feature tasks (vibe-*, +tmpl-fe-*, open-*) are scored on LOC alone -- there is no deterministic check that the asked +feature was actually implemented, so an arm could "win" the LOC metric by shipping a stub. +That is the inverse of the safety hole and the most credible attack on the headline number: +"you wrote less because you did less." + +This pass closes it. An LLM judge rates how FULLY each submission implements its task, on the +same auditable footing as the over-engineering judge in judge.py: a published rubric, a fixed +model at temperature 0, and a --selftest that must rank a complete reference strictly above a +stub before any real scoring is trusted. Pair the output with run.py's LOC: a low-LOC arm whose +completeness also drops is doing less, not less-bloated -- and now the bench shows it. + + python complete.py --selftest # validate the judge ranks complete > stub (small API spend) + python complete.py --selftest-offline # validate the GATE LOGIC only, no API, no key + python complete.py --run runs/ # completeness-judge every workspace in a matrix run + +Judge: claude-sonnet-4-6, key from ../../.env (shared with judge.py). ~$0.003/cell. + +ponytail: reuses judge.py's HTTP/key/source plumbing instead of duplicating it -- one rubric +param is the only delta between the two passes. +""" +import argparse, json, sys +from collections import defaultdict +from pathlib import Path + +from tasks import TASKS +from judge import load_key, source_text, judge_call, parse_score, RUNS_DIR, JUDGE_MODEL + +SCORE_KEY = "completeness" +FLAG_AT = 1 # cells scoring <= this are under-delivery (stub/partial) and get listed +ARMS_ORDER = ["baseline", "caveman", "ponytail", "yagni", "yagni-oneliner"] + +RUBRIC = ( + "You are a senior engineer checking whether a code submission ACTUALLY IMPLEMENTS the task it " + "was given. Judge COMPLETENESS ONLY -- ignore over-engineering, style, performance, and security. " + "A stub, a placeholder, a bare `pass`/`TODO`/`NotImplementedError`, or code that silently omits " + "the core behavior asked for is INCOMPLETE. Score 0-3:\n" + "0 = stub/empty/placeholder, does essentially nothing the task asked\n" + "1 = partial: the core behavior is missing or broken\n" + "2 = mostly complete: it works but a stated requirement is missing\n" + "3 = fully implements what the task asked\n" + "Name the single most important missing piece, or \"none\". " + "Respond with ONLY this JSON: {\"completeness\": <0-3 int>, \"why\": \"\", \"missing\": \"\"}" +) + +def parse_complete(text): + d = parse_score(text) + if d and SCORE_KEY in d: + try: d[SCORE_KEY] = int(d[SCORE_KEY]) + except Exception: d[SCORE_KEY] = None + return d + +# --- the gate: a complete impl must out-score a stub for the same task --- +def _rank_ok(scores): + """scores: {(task_id, label): {SCORE_KEY: int}}. For each task the 'complete' label must + strictly out-score the 'stub' label, else the judge (or the gate) is not trustworthy.""" + ok = True + for task_id in sorted({t for (t, _) in scores}): + hi = scores.get((task_id, "complete")) or {} + lo = scores.get((task_id, "stub")) or {} + if not (isinstance(hi.get(SCORE_KEY), int) and isinstance(lo.get(SCORE_KEY), int) + and hi[SCORE_KEY] > lo[SCORE_KEY]): + print(f"XX {task_id}: did not rank complete above stub"); ok = False + else: + print(f"ok {task_id}: complete({hi[SCORE_KEY]}) > stub({lo[SCORE_KEY]})") + return ok + +# Complete refs are the deterministic tasks' known-good answers; stubs do nothing. +STUBS = { + "cache": "def compute(n):\n pass\n", + "safe-path": "def safe_upload_path(base_dir, filename):\n pass\n", +} +PAIRS = [(t, lbl, code) for t in STUBS for lbl, code in + (("complete", TASKS[t]["good"]), ("stub", STUBS[t]))] + +def selftest(key): + """Live: the judge model must rank each complete ref above its stub.""" + scores = {} + for task_id, label, code in PAIRS: + s = parse_complete(judge_call(TASKS[task_id]["prompt"], code, key, system=RUBRIC)) + scores[(task_id, label)] = s or {} + print(f" {task_id:10} {label:8} -> {s}") + ok = _rank_ok(scores) + print(f"\ncompleteness judge selftest: {'valid' if ok else 'NOT TRUSTWORTHY'}") + return 0 if ok else 1 + +def selftest_offline(): + """No API, no key: prove the GATE catches under-delivery. A well-ordered matrix must pass + and a matrix where a stub out-scores the complete impl must be flagged. Fails loudly if the + gate is ever weakened into a no-op.""" + good = {("cache", "complete"): {SCORE_KEY: 3}, ("cache", "stub"): {SCORE_KEY: 0}} + bad = {("cache", "complete"): {SCORE_KEY: 1}, ("cache", "stub"): {SCORE_KEY: 3}} + print("offline gate -- well-ordered (expect ok):") + p_good = _rank_ok(good) + print("offline gate -- stub out-scores complete (expect XX):") + p_bad = _rank_ok(bad) + passed = p_good and not p_bad + print(f"\ncompleteness gate selftest (offline): {'valid' if passed else 'BROKEN'}") + return 0 if passed else 1 + +def run(run_dir, key): + run_dir = Path(run_dir) + if not run_dir.exists(): run_dir = RUNS_DIR / run_dir.name + cells = [] + for ws in sorted(p for p in run_dir.iterdir() if p.is_dir()): + parts = ws.name.split("__") + if len(parts) != 4 or parts[0] not in TASKS: continue + cells.append((parts[0], parts[1], parts[2], ws)) + print(f"completeness-judging {len(cells)} workspaces with {JUDGE_MODEL} ...") + scored = [] + for i, (tid, arm, model, ws) in enumerate(cells, 1): + s = parse_complete(judge_call(TASKS[tid]["prompt"], source_text(ws), key, system=RUBRIC)) \ + or {SCORE_KEY: None} + scored.append({"task": tid, "arm": arm, "model": model, SCORE_KEY: s.get(SCORE_KEY), + "why": s.get("why", ""), "missing": s.get("missing", "")}) + if i % 25 == 0 or i == len(cells): print(f" [{i}/{len(cells)}]", flush=True) + (run_dir / "completeness.json").write_text( + json.dumps({"judge": JUDGE_MODEL, "rubric": RUBRIC, "scores": scored}, indent=2), encoding="utf-8") + by_arm = defaultdict(list) + for r in scored: + if isinstance(r[SCORE_KEY], int): by_arm[r["arm"]].append(r[SCORE_KEY]) + print(f"\n=== completeness by arm (judge: {JUDGE_MODEL}, 0=stub .. 3=fully implements) ===") + print(f" {'arm':16} {'n':>4} {'mean':>6} {'min':>4}") + for arm in ARMS_ORDER: + v = by_arm.get(arm, []) + if v: print(f" {arm:16} {len(v):>4} {sum(v)/len(v):>6.2f} {min(v):>4}") + under = sorted([r for r in scored if isinstance(r[SCORE_KEY], int) and r[SCORE_KEY] <= FLAG_AT], + key=lambda r: r[SCORE_KEY]) + print(f"\n=== under-delivered (completeness <= {FLAG_AT}): {len(under)} cells ===") + for r in under[:20]: + print(f" {r['task']:13} {r['arm']:15} {r['model']:7} score={r[SCORE_KEY]} missing={r['missing']}") + print(f"\nwrote {run_dir / 'completeness.json'}") + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--selftest", action="store_true", help="live: judge ranks complete > stub") + ap.add_argument("--selftest-offline", action="store_true", help="gate logic only, no API") + ap.add_argument("--run", help="run dir to completeness-judge") + args = ap.parse_args() + if args.selftest_offline: + sys.exit(selftest_offline()) + key = load_key() + if not key: sys.exit("no ANTHROPIC_API_KEY (.env or env)") + if args.selftest: sys.exit(selftest(key)) + if args.run: + if selftest(key): sys.exit("judge not trustworthy; refusing to judge the matrix") + return run(args.run, key) + sys.exit("give --selftest, --selftest-offline, or --run ") + +if __name__ == "__main__": + main() diff --git a/ponytail/benchmarks/agentic/judge.py b/ponytail/benchmarks/agentic/judge.py new file mode 100644 index 0000000000000000000000000000000000000000..63a3b7f38681731baed4e22fb914dc11bb380813 --- /dev/null +++ b/ponytail/benchmarks/agentic/judge.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +"""LLM-judge over-engineering pass for the agentic benchmark. + +Over-engineering is the one axis that resists a deterministic check, so it gets an LLM judge -- +but an auditable one: a published rubric, a fixed judge model at temperature 0, and every score +must name the specific construct it considers unnecessary (or "none"). The judge is validated +first by --selftest: it must rank a deliberately over-engineered reference strictly above a +minimal one for the same task, or we do not trust it on real submissions. + + python judge.py --selftest # validate the judge on reference pairs (small spend) + python judge.py --run runs/ # judge every workspace's source in a matrix run + +Judge: claude-sonnet-4-6 via the Anthropic Messages API (key from ../../.env). Scores the SOURCE +files only (tests excluded -- a test is not over-engineering). Cost is ~$0.003/cell. + +ponytail: stdlib urllib for the API call, no requests dependency. +""" +import argparse, json, os, re, sys, time, urllib.request +from collections import defaultdict +from pathlib import Path + +from tasks import TASKS + +ROOT = Path(__file__).resolve().parents[2] +RUNS_DIR = Path(__file__).resolve().parent / "runs" +JUDGE_MODEL = "claude-sonnet-4-6" + +RUBRIC = ( + "You are a senior engineer reviewing a code submission for OVER-ENGINEERING ONLY. " + "Ignore correctness, style, performance, and security. Over-engineering means structure " + "beyond what the task needs: speculative abstraction, classes/factories/config/flexibility " + "nobody asked for, a framework for a one-off job. Score 0-3:\n" + "0 = minimal, appropriate to the task\n" + "1 = slightly more structure than needed\n" + "2 = noticeably over-built (an unneeded class/abstraction/config/flexibility)\n" + "3 = clearly over-engineered (speculative generality, a framework for a one-off)\n" + "Name the single most unnecessary construct, or \"none\". " + "Respond with ONLY this JSON: {\"over_engineering\": <0-3 int>, \"why\": \"\", \"cite\": \"\"}" +) + +def load_key(): + try: + for line in (ROOT / ".env").read_text(encoding="utf-8").splitlines(): + if line.startswith("ANTHROPIC_API_KEY=") and len(line) > 18: + return line.split("=", 1)[1].strip() + except Exception: + pass + return os.environ.get("ANTHROPIC_API_KEY") + +def _is_test(name): + n = name.lower() + return n.startswith("test_") or n.endswith("_test.py") or n == "conftest.py" + +def source_text(workdir: Path): + """Concatenate the agent's source files (tests + artifacts excluded), with name headers.""" + out = [] + for p in sorted(workdir.rglob("*")): + if not p.is_file() or "__pycache__" in p.parts or p.suffix == ".pyc": continue + if p.name.startswith((".", "_")) or _is_test(p.name): continue + try: out.append(f"# === {p.relative_to(workdir)} ===\n{p.read_text(encoding='utf-8', errors='ignore')}") + except Exception: continue + return "\n\n".join(out) + +def judge_call(task_prompt, files, key, retries=3, system=RUBRIC): + user = f"TASK GIVEN TO THE AUTHOR:\n{task_prompt}\n\nFILES THEY WROTE:\n{files}" + body = json.dumps({"model": JUDGE_MODEL, "max_tokens": 300, "temperature": 0, + "system": system, "messages": [{"role": "user", "content": user}]}).encode() + for attempt in range(retries): + try: + req = urllib.request.Request("https://api.anthropic.com/v1/messages", data=body, + headers={"x-api-key": key, "anthropic-version": "2023-06-01", "content-type": "application/json"}) + with urllib.request.urlopen(req, timeout=60) as r: + j = json.loads(r.read()) + return j["content"][0]["text"] + except Exception as e: + if attempt == retries - 1: return f'{{"error": "{str(e)[:120]}"}}' + time.sleep(2 * (attempt + 1)) + +def parse_score(text): + m = re.search(r"\{.*\}", text or "", re.S) + if not m: return None + try: + d = json.loads(m.group(0)) + if "over_engineering" in d: d["over_engineering"] = int(d["over_engineering"]) + return d + except Exception: + return None + +# --- selftest: the judge must rank over-engineered above minimal for the same task --- +CACHE_OVER = ( + "import time\nfrom collections import OrderedDict\n" + "class CacheEntry:\n def __init__(self, value, created_at):\n self.value = value\n self.created_at = created_at\n" + "class ComputeCache:\n \"\"\"Configurable TTL cache with LRU eviction and hit/miss stats.\"\"\"\n" + " def __init__(self, max_size=128, ttl_seconds=3600, enable_stats=True):\n" + " self.max_size = max_size; self.ttl_seconds = ttl_seconds; self.enable_stats = enable_stats\n" + " self._store = OrderedDict(); self._hits = 0; self._misses = 0\n" + " def _evict(self):\n while len(self._store) > self.max_size: self._store.popitem(last=False)\n" + " def get_or_compute(self, n, fn):\n now = time.time()\n" + " if n in self._store and now - self._store[n].created_at < self.ttl_seconds:\n" + " self._hits += 1; self._store.move_to_end(n); return self._store[n].value\n" + " self._misses += 1; v = fn(n); self._store[n] = CacheEntry(v, now); self._evict(); return v\n" + "_cache = ComputeCache()\n" + "def compute(n):\n return _cache.get_or_compute(n, lambda m: sum(i*i for i in range(m)))\n" +) +SAFEPATH_OVER = ( + "import os\nclass PathPolicy:\n def __init__(self, allow_symlinks=False, max_depth=10, allowed_extensions=None):\n" + " self.allow_symlinks = allow_symlinks; self.max_depth = max_depth\n self.allowed_extensions = allowed_extensions or []\n" + "class PathSanitizer:\n \"\"\"Pluggable path sanitizer with configurable policy.\"\"\"\n def __init__(self, policy=None):\n self.policy = policy or PathPolicy()\n" + " def sanitize(self, base_dir, filename):\n base = os.path.abspath(base_dir)\n target = os.path.abspath(os.path.join(base, filename))\n" + " if os.path.commonpath([base, target]) != base: raise ValueError('traversal')\n return target\n" + "_default = PathSanitizer()\ndef safe_upload_path(base_dir, filename):\n return _default.sanitize(base_dir, filename)\n" +) +SELFTEST_PAIRS = [ + ("cache", "minimal", TASKS["cache"]["good"]), + ("cache", "over", CACHE_OVER), + ("safe-path", "minimal", TASKS["safe-path"]["good"]), + ("safe-path", "over", SAFEPATH_OVER), +] + +def selftest(key): + scores = {} + for task_id, label, code in SELFTEST_PAIRS: + s = parse_score(judge_call(TASKS[task_id]["prompt"], code, key)) + scores[(task_id, label)] = s + print(f" {task_id:10} {label:8} -> {s}") + ok = True + for task_id in ("cache", "safe-path"): + lo = scores.get((task_id, "minimal"), {}) or {} + hi = scores.get((task_id, "over"), {}) or {} + if not (isinstance(hi.get("over_engineering"), int) and isinstance(lo.get("over_engineering"), int) + and hi["over_engineering"] > lo["over_engineering"]): + print(f"XX {task_id}: judge did not rank over-engineered above minimal") + ok = False + else: + print(f"ok {task_id}: over({hi['over_engineering']}) > minimal({lo['over_engineering']})") + print(f"\njudge selftest: {'valid' if ok else 'NOT TRUSTWORTHY'}") + return 0 if ok else 1 + +def run(run_dir, key): + run_dir = Path(run_dir) + if not run_dir.exists(): run_dir = RUNS_DIR / run_dir.name + cells, scored = [], [] + for ws in sorted(p for p in run_dir.iterdir() if p.is_dir()): + parts = ws.name.split("__") + if len(parts) != 4 or parts[0] not in TASKS: continue + cells.append((parts[0], parts[1], parts[2], ws)) + print(f"judging {len(cells)} workspaces with {JUDGE_MODEL} ...") + for i, (tid, arm, model, ws) in enumerate(cells, 1): + s = parse_score(judge_call(TASKS[tid]["prompt"], source_text(ws), key)) or {"over_engineering": None} + rec = {"task": tid, "arm": arm, "model": model, "over_engineering": s.get("over_engineering"), + "why": s.get("why", ""), "cite": s.get("cite", "")} + scored.append(rec) + if i % 25 == 0 or i == len(cells): print(f" [{i}/{len(cells)}]", flush=True) + (run_dir / "judge.json").write_text(json.dumps({"judge": JUDGE_MODEL, "rubric": RUBRIC, "scores": scored}, indent=2), encoding="utf-8") + # aggregate + by_arm = defaultdict(list) + for r in scored: + if isinstance(r["over_engineering"], int): by_arm[r["arm"]].append(r["over_engineering"]) + print(f"\n=== over-engineering by arm (judge: {JUDGE_MODEL}, 0=minimal .. 3=over-built) ===") + print(f" {'arm':16} {'n':>4} {'mean':>6} {'max':>4}") + for arm in ["baseline", "caveman", "ponytail", "yagni", "yagni-oneliner"]: + v = by_arm.get(arm, []) + if v: print(f" {arm:16} {len(v):>4} {sum(v)/len(v):>6.2f} {max(v):>4}") + worst = sorted([r for r in scored if isinstance(r["over_engineering"], int) and r["over_engineering"] >= 2], + key=lambda r: -r["over_engineering"]) + print(f"\n=== flagged over-engineered (score >= 2): {len(worst)} cells ===") + for r in worst[:20]: + print(f" {r['task']:11} {r['arm']:15} {r['model']:7} score={r['over_engineering']} cite={r['cite']}") + print(f"\nwrote {run_dir / 'judge.json'}") + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--selftest", action="store_true") + ap.add_argument("--run", help="run dir to judge") + args = ap.parse_args() + key = load_key() + if not key: sys.exit("no ANTHROPIC_API_KEY (.env or env)") + if args.selftest: sys.exit(selftest(key)) + if args.run: + if selftest(key): sys.exit("judge not trustworthy; refusing to judge the matrix") + return run(args.run, key) + sys.exit("give --selftest or --run ") + +if __name__ == "__main__": + main() diff --git a/ponytail/benchmarks/agentic/run.py b/ponytail/benchmarks/agentic/run.py new file mode 100644 index 0000000000000000000000000000000000000000..318b2b3dbfb9cd66c5fcb9d74ce44bc8673dfdda --- /dev/null +++ b/ponytail/benchmarks/agentic/run.py @@ -0,0 +1,471 @@ +#!/usr/bin/env python3 +"""Agentic, multi-file benchmark for ponytail. + +Runs each (task x arm x model) through a real headless Claude Code session in an isolated +temp workspace seeded with a starter file, then scores the produced files deterministically +for CORRECTNESS and SAFETY -- the axis the single-shot promptfoo bench was blind to. + +Over-engineering is proxied by SOURCE file count + source LOC (tests are counted separately, +never as bloat -- writing a test is good practice, not over-engineering). An LLM-judge +over-engineering score is a later pass. + + python run.py --selftest + Verify every scorer (good passes, bad is caught). No API, no spend. Run first, always. + + python run.py --all --models haiku,sonnet,opus --runs 5 + Live run (spends API). Workspaces kept under runs// for inspection. + + python run.py --rescore runs/ + Recompute metrics + aggregate from kept workspaces. No API. Use after changing a + metric or scorer so you never pay the API twice for a measurement tweak. + +ponytail: the claude CLI is the harness (already installed, we run inside it). No SDK +dependency. The CLI's JSON output already carries cost/tokens/duration/permission_denials. +""" +import argparse, concurrent.futures, datetime, json, os, re, shutil, signal, statistics, subprocess, sys, tempfile +from collections import defaultdict +from pathlib import Path + +from tasks import TASKS + +ROOT = Path(__file__).resolve().parents[2] +RUNS_DIR = Path(__file__).resolve().parent / "runs" + +def _skill(rel): return (ROOT / rel).read_text(encoding="utf-8") +ARMS = { + "baseline": lambda: None, + "ponytail": lambda: _skill("skills/ponytail/SKILL.md"), + "caveman": lambda: _skill("benchmarks/arms/caveman-SKILL.md"), + "yagni": lambda: "Follow YAGNI principles.", + "yagni-oneliner": lambda: "Follow YAGNI principles, and prefer one-liner solutions.", +} +MODELS = {"haiku": "claude-haiku-4-5-20251001", "sonnet": "claude-sonnet-4-6", "opus": "claude-opus-4-8"} + +# Skills are plugins activated by a SessionStart hook. To test exactly one at a time we exclude the +# user's globally-enabled plugins (--setting-sources project,local) and load one plugin from its +# cache dir (--plugin-dir). The smoke test verifies activation by output style. +PLUGIN_ARMS = ("ponytail", "caveman") # arms activated via --plugin-dir (vs raw --append prompts) +PLUGIN_CACHE = Path.home() / ".claude" / "plugins" / "cache" + +def _plugin_dir(name): + """Resolve a plugin's cache dir portably -- hardcoding one machine's absolute path + (e.g. C:\\Users\\\\...) made the ponytail/caveman arms unreproducible off that box. + Order: env override -> latest version dir under ~/.claude/plugins/cache -> clear error. + Resolved per-arm at use-site so a missing caveman install can't block a ponytail-only run.""" + env = os.environ.get(f"{name.upper()}_PLUGIN_DIR") + if env: return env + base = PLUGIN_CACHE / name / name + versions = sorted(p for p in base.glob("*") if p.is_dir()) if base.exists() else [] + if not versions: + sys.exit(f"{name} plugin dir not found under {base}; install the plugin or set {name.upper()}_PLUGIN_DIR") + return str(versions[-1]) # latest version dir; not pinned to one machine's hash + +CELL_TIMEOUT = 300 # seconds per cell; a hung agent is force-killed (process tree) so the pool can't freeze + +# Added to every arm's system prompt, identically. We measure code PRODUCTION, not execution: agents +# write the implementation and stop. No live verification -- earlier attempts had agents open a browser, +# hit the template's login wall, and retry, inflating tokens/time with flailing instead of code. Writing +# tests is still explicitly allowed, so ponytail's "leave a runnable check" discipline is not suppressed. +NO_RUN = ("Write the implementation (include tests if you normally would for a change like this). " + "Do not run a dev server, install dependencies, run a database, or open a browser to verify -- " + "just write the code and stop. Only the code you write is measured, not its execution.") + +def _is_test(p: Path, workdir: Path): + rel = p.relative_to(workdir) + name = p.name.lower() + return (name.startswith("test_") or name.endswith("_test.py") or name == "conftest.py" + or any(part.lower() in ("test", "tests") for part in rel.parts[:-1])) + +CODE_EXT = {".py", ".js", ".ts", ".jsx", ".tsx", ".html", ".css", ".go", ".rs", ".java", ".rb", ".sh"} + +def _count(p: Path, with_comments: bool): + try: lines = p.read_text(encoding="utf-8", errors="ignore").splitlines() + except Exception: return 0 + n = 0 + for ln in lines: + s = ln.strip() + if not s: continue + if not with_comments and s.startswith(("#", "//", "*", "/*", "*/")): continue + n += 1 + return n + +_SELFCHECK_DEFS = ("def demo(", "def _demo(", "def selfcheck(", "def _selfcheck(", + "def _check(", "def _smoke(", "def smoke(") +def _selfcheck_split(p: Path): + """Split a produced .py file at the first TOP-LEVEL self-check marker (a `__main__` guard or a + demo()/selfcheck() function) through end of file. Returns (src_total, src_code, sc_total, + sc_code), counted like _count. On a surgical task that delivers ONE function, an in-file self- + check is the runnable check ponytail's rule asks for -- a positive signal, not source bloat -- + so it is split off here and counted as test LOC instead of penalising the arm that wrote it.""" + try: lines = p.read_text(encoding="utf-8", errors="ignore").splitlines() + except Exception: return 0, 0, 0, 0 + start = None + for i, ln in enumerate(lines): + if ln[:1] not in (" ", "\t") and (ln.startswith("if __name__") or ln.startswith(_SELFCHECK_DEFS)): + start = i; break + def cnt(seq): + t = c = 0 + for ln in seq: + s = ln.strip() + if not s: continue + t += 1 + if not s.startswith(("#", "//", "*", "/*", "*/")): c += 1 + return t, c + if start is None: + t, c = cnt(lines); return t, c, 0, 0 + t, c = cnt(lines[:start]); st, sc = cnt(lines[start:]) + return t, c, st, sc + +def code_stats(workdir: Path, selfcheck_as_test: bool = False): + """LOC over code-extension source files only (generated images/data can't pollute it). + total_loc counts every non-blank line including comments and docstrings -- the bloat a vibe + baseline actually produces. src_loc is code-only, for the breakdown. Tests tracked separately, + never as bloat. selfcheck_as_test (surgical tasks): an in-file __main__/demo() self-check is + reclassified from source to test, so following ponytail's 'leave a runnable check' rule is not + counted as code bloat against it.""" + fixture = set() # files that were seeded, not delivered + fm = workdir / "_fixture_files.json" + if fm.exists(): + try: fixture = set(json.loads(fm.read_text(encoding="utf-8"))) + except Exception: pass + def _rel(p): return str(p.relative_to(workdir)).replace("\\", "/") + files = [p for p in workdir.rglob("*") if p.is_file() and p.suffix in CODE_EXT + and "__pycache__" not in p.parts and "node_modules" not in p.parts + and not p.name.startswith((".", "_")) and _rel(p) not in fixture] + src = [p for p in files if not _is_test(p, workdir)] + tst = [p for p in files if _is_test(p, workdir)] + test_loc = sum(_count(p, True) for p in tst) + if selfcheck_as_test: + total = code = sc_test = 0 + for p in src: + t, c, st, _ = _selfcheck_split(p) + total += t; code += c; sc_test += st + return {"files": len(files), "src_files": len(src), + "total_loc": total, "src_loc": code, + "test_files": len(tst), "test_loc": test_loc + sc_test} + return {"files": len(files), "src_files": len(src), + "total_loc": sum(_count(p, True) for p in src), # incl comments + docstrings (the bloat) + "src_loc": sum(_count(p, False) for p in src), # code only + "test_files": len(tst), "test_loc": test_loc} + +def _git(workdir, *args): + return subprocess.run([shutil.which("git") or "git", *args], cwd=str(workdir), + capture_output=True, text=True) + +def _git_snapshot(workdir): + """Commit the seeded repo so we can diff exactly what the agent changes.""" + _git(workdir, "init", "-q") + _git(workdir, "add", "-A") + _git(workdir, "-c", "user.email=bench@local", "-c", "user.name=bench", + "commit", "-q", "-m", "base", "--no-verify") + +_SKIP_DIFF = ("-lock", ".lock", ".gen.ts", "lock.json", "routeTree.gen") +def git_diff_stats(workdir): + """Added lines (incl comments) of code files the agent created OR modified, vs the seeded + base. This is the delivered-code metric and matches the '+N' a PR/diff shows. Tests counted + separately; lockfiles/generated files skipped.""" + _git(workdir, "add", "-A") + out = _git(workdir, "diff", "--cached", "--numstat", "HEAD").stdout + loc = files = test_loc = test_files = 0 + for line in out.splitlines(): + parts = line.split("\t") + if len(parts) != 3: continue + added, _deleted, path = parts + if added == "-": continue # binary + if Path(path).suffix not in CODE_EXT: continue + if any(k in path for k in _SKIP_DIFF) or "node_modules" in path: continue + n = int(added) + if _is_test(Path(workdir) / path, Path(workdir)): test_loc += n; test_files += 1 + else: loc += n; files += 1 + return {"files": files, "src_files": files, "total_loc": loc, "src_loc": loc, + "test_files": test_files, "test_loc": test_loc} + +def selftest(): + """Each task's good ref must score correct+safe; the bad ref must be caught on its + declared axis. Verifies the instruments before any API spend.""" + failures = 0 + for tid, task in TASKS.items(): + if task.get("open"): continue # open tasks measure LOC only, no good/bad refs + axis = task.get("axis", "safe") + for kind in ("good", "bad"): + with tempfile.TemporaryDirectory() as d: + for fn, content in task.get("seed", {}).items(): # seed siblings (a helper module + (Path(d) / fn).write_text(content, encoding="utf-8") # the ref imports) too + (Path(d) / task["file"]).write_text(task[kind], encoding="utf-8") # entry = the ref + r = task["score"](Path(d)) + ok = (r["correct"] == 1 and r["safe"] == 1) if kind == "good" else (r[axis] == 0) + print(f"{'ok ' if ok else 'XX '} {tid:12} {kind:4} correct={r['correct']} " + f"safe={r['safe']} axis={axis} {r['reason']}") + failures += 0 if ok else 1 + failures += _selftest_plugin_dir() + failures += _selftest_kill() + print(f"\nselftest: {'all instruments valid' if not failures else str(failures) + ' BROKEN'}") + return failures + +def _selftest_plugin_dir(): + """Plugin-dir resolution must be portable: env override wins, and a missing install + fails loudly (sys.exit) instead of silently passing a non-existent path to --plugin-dir.""" + fails = 0 + sentinel = "/tmp/ponytail-selftest-plugin-dir" + os.environ["PONYTAIL_PLUGIN_DIR"] = sentinel + try: + ok_env = _plugin_dir("ponytail") == sentinel + finally: + del os.environ["PONYTAIL_PLUGIN_DIR"] + print(f"{'ok ' if ok_env else 'XX '} plugin_dir env override honored") + fails += 0 if ok_env else 1 + missing = "ponytail-does-not-exist-xyz" # no env, no cache entry -> must sys.exit + try: + _plugin_dir(missing); ok_miss = False # reached only if it did NOT exit -> broken + except SystemExit: + ok_miss = True + print(f"{'ok ' if ok_miss else 'XX '} plugin_dir miss clear error (sys.exit)") + return fails + (0 if ok_miss else 1) + +def _tree_kill(proc): + """Tree-kill one timed-out cell, never a blanket kill (that would also take down this + Claude Code session). Windows: taskkill /T walks the child PIDs. POSIX has no taskkill, + so the cell runs in its own session (Popen start_new_session) and we kill the group.""" + if os.name == "nt": + subprocess.run(["taskkill", "/F", "/T", "/PID", str(proc.pid)], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + else: + try: os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + except ProcessLookupError: pass # already exited + +def _selftest_kill(): + """tree-kill must actually terminate a cell that outran its timeout, on this platform.""" + p = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"], + start_new_session=(os.name != "nt")) + _tree_kill(p) + try: ok = p.wait(timeout=10) is not None + except subprocess.TimeoutExpired: ok = False; p.kill() + print(f"{'ok ' if ok else 'XX '} tree_kill terminates a timed-out cell") + return 0 if ok else 1 + +def chat_code_loc(text): + """LOC of fenced code blocks in a chat answer: (total incl comments, code-only).""" + total = code = 0 + for b in re.findall(r"```[a-zA-Z0-9_+-]*\r?\n(.*?)```", text or "", re.S): + for ln in b.splitlines(): + s = ln.strip() + if not s: continue + total += 1 + if not s.startswith(("#", "//", "*", "/*", "*/")): code += 1 + return total, code + +def score_workspace(task_id, arm, model, workdir: Path): + meta, result_text = {}, "" + cj = workdir / "_claude.json" + if cj.exists(): + try: + j = json.loads(cj.read_text(encoding="utf-8")) + u = j.get("usage") or {} + meta = {"cost": j.get("total_cost_usd"), "duration_ms": j.get("duration_ms"), + "turns": j.get("num_turns"), "denials": len(j.get("permission_denials") or []), + "out_tokens": u.get("output_tokens"), "in_tokens": u.get("input_tokens"), + "cache_tokens": (u.get("cache_read_input_tokens") or 0) + (u.get("cache_creation_input_tokens") or 0)} + result_text = j.get("result", "") + except Exception: pass + surgical = not TASKS[task_id].get("open") and not TASKS[task_id].get("fixture") + stats = git_diff_stats(workdir) if TASKS[task_id].get("fixture") else code_stats(workdir, selfcheck_as_test=surgical) + # open/explain tasks answer in the chat, not a file. If no source file was written, count the + # code the agent delivered in its chat answer so the comparison isn't a false zero. + if TASKS[task_id].get("open") and stats["total_loc"] == 0 and result_text: + t, c = chat_code_loc(result_text) + stats = {**stats, "total_loc": t, "src_loc": c, "src_files": 1 if t else 0} + if TASKS[task_id].get("fixture"): + sc = {"correct": 1 if stats.get("total_loc", 0) > 0 else 0, "safe": 1, "reason": "git-diff"} + else: + sc = TASKS[task_id]["score"](workdir) + return {"task": task_id, "arm": arm, "model": model, **sc, **stats, **meta} + +def run_cell(task_id, arm, model, workdir: Path): + task = TASKS[task_id] + if task.get("fixture"): # copy a real repo in; record what was seeded + fx = Path(task["fixture"]) # absolute path, or a name under fixtures/ + if not fx.is_absolute(): fx = Path(__file__).resolve().parent / "fixtures" / task["fixture"] + shutil.copytree(fx, workdir, dirs_exist_ok=True, + ignore=shutil.ignore_patterns("node_modules", ".git", "build", "dist", + "dist-ssr", ".vite", "*.log", "__pycache__", + "storage", ".venv", "venv", ".pytest_cache", + "*.mp4", "*.mp3", "*.wav", "*.mov", + "*service-account*.json", + "nul", "con", "prn", "aux", + "DatePicker*.tsx", "DatePicker*.jsx")) + manifest = sorted(str(p.relative_to(workdir)).replace("\\", "/") + for p in workdir.rglob("*") if p.is_file()) + (workdir / "_fixture_files.json").write_text(json.dumps(manifest), encoding="utf-8") + for fn, content in task.get("seed", {}).items(): + (workdir / fn).write_text(content, encoding="utf-8") + if task.get("fixture"): _git_snapshot(workdir) # baseline commit -> diff the agent's changes + claude = shutil.which("claude") + if not claude: sys.exit("claude CLI not found on PATH") + # Skills are PLUGINS (SessionStart hook); --append of the SKILL text does NOT activate them. + # Exclude the user's globally-enabled plugins for every arm, then load exactly the one this arm + # needs from its cache dir. baseline loads none; yagni-oneliner is a raw prompt so it uses --append. + # No live verification (see NO_RUN): --strict-mcp-config drops all MCP servers so there is no browser + # tool, and --disallowedTools Bash blocks running a server/db/npm. An agent writes with + # Read/Write/Edit/Glob/Grep and stops -- no login wall, no browser thrash. We measure code, not execution. + cmd = [claude, "-p", task["prompt"], "--model", MODELS[model], + "--permission-mode", "bypassPermissions", "--output-format", "json", + "--setting-sources", "project,local", "--strict-mcp-config", + "--disallowedTools", "Bash"] + append = NO_RUN # all arms get NO_RUN, identically + if arm in PLUGIN_ARMS: + cmd += ["--plugin-dir", _plugin_dir(arm)] # real activation of exactly one plugin + else: + extra = ARMS[arm]() # baseline -> None; yagni-oneliner -> the prompt + if extra: append = extra + "\n\n" + NO_RUN + cmd += ["--append-system-prompt", append] + out_path, err_path = workdir / "_claude.json", workdir / "_claude.stderr.txt" + # stdout -> file, never a PIPE: on Windows a hung agent's child processes can hold a stdout PIPE + # open forever, so subprocess.run(timeout=) never fires and the worker freezes. Writing to a file + # lets proc.wait(timeout) return reliably; on timeout _tree_kill ends ONLY this cell's process + # tree -- never a blanket kill, which would also take down this Claude Code session. + try: + with open(out_path, "wb") as so, open(err_path, "wb") as se: + proc = subprocess.Popen(cmd, cwd=str(workdir), stdout=so, stderr=se, + start_new_session=(os.name != "nt")) + try: + proc.wait(timeout=CELL_TIMEOUT) + except subprocess.TimeoutExpired: + _tree_kill(proc) + try: proc.wait(timeout=15) + except Exception: pass + se.write(f"\n[KILLED after {CELL_TIMEOUT}s timeout]".encode()) + except Exception as e: + out_path.write_text(json.dumps({"error": str(e)[:300]}), encoding="utf-8") + return score_workspace(task_id, arm, model, workdir) + +def aggregate(results): + groups = defaultdict(list) + for r in results: groups[(r["task"], r["arm"], r["model"])].append(r) + rows = [] + for (t, a, m), cells in sorted(groups.items()): + n = len(cells) + costs = [c["cost"] for c in cells if c.get("cost") is not None] + loc_cells = [c for c in cells if c.get("total_loc", 0) > 0] # LOC only where code was delivered + nl = len(loc_cells) + rows.append({"task": t, "arm": a, "model": m, "n": n, + "safe_rate": round(sum(c["safe"] for c in cells) / n, 3), + "correct_rate": round(sum(c["correct"] for c in cells) / n, 3), + "wrote_file_rate": round(nl / n, 3), + "total_loc_median": statistics.median(c["total_loc"] for c in loc_cells) if nl else 0, + "src_loc_median": statistics.median(c["src_loc"] for c in loc_cells) if nl else 0, + "total_loc_max": max((c["total_loc"] for c in loc_cells), default=0), + "src_files_median": statistics.median(c["src_files"] for c in loc_cells) if nl else 0, + "wrote_tests_rate": round(sum(1 for c in cells if c.get("test_files", 0) > 0) / n, 3), + "cost_mean": round(statistics.mean(costs), 4) if costs else None, + "out_tokens_mean": (round(statistics.mean([c["out_tokens"] for c in cells if c.get("out_tokens") is not None])) + if any(c.get("out_tokens") is not None for c in cells) else None), + "total_tokens_mean": (round(statistics.mean([(c.get("in_tokens") or 0) + (c.get("out_tokens") or 0) + (c.get("cache_tokens") or 0) + for c in cells if c.get("out_tokens") is not None])) + if any(c.get("out_tokens") is not None for c in cells) else None), + "time_s_mean": (round(statistics.mean([c["duration_ms"] / 1000 for c in cells if c.get("duration_ms") is not None]), 1) + if any(c.get("duration_ms") is not None for c in cells) else None)}) + return rows + +def print_table(rows): + by = defaultdict(list) + for r in rows: by[(r["task"], r["model"])].append(r) + for (task, model), rs in sorted(by.items()): + print(f"\n=== {task} ({model}, n={rs[0]['n']}) ===") + print(f" {'arm':16} {'wrote%':>7} {'correct':>8} {'LOC':>7} {'tot_tok':>9} {'$/run':>8} {'time_s':>7}") + for r in sorted(rs, key=lambda x: x["arm"]): + c = ("$" + format(r["cost_mean"], ".4f")) if r["cost_mean"] is not None else "-" + tt = r.get("total_tokens_mean"); t = r.get("time_s_mean") + print(f" {r['arm']:16} {r.get('wrote_file_rate', 1.0):>7} {r['correct_rate']:>8} " + f"{r['total_loc_median']:>7} {(tt if tt is not None else '-'):>9} {c:>8} " + f"{(t if t is not None else '-'):>7}") + +def rescore(run_dir): + run_dir = Path(run_dir) + if not run_dir.exists(): # accept "" or "runs/" from any cwd + run_dir = RUNS_DIR / run_dir.name + results = [] + for ws in sorted(p for p in run_dir.iterdir() if p.is_dir()): + parts = ws.name.split("__") + if len(parts) != 4 or parts[0] not in TASKS: continue + tid, arm, model, _r = parts + results.append(score_workspace(tid, arm, model, ws)) + rows = aggregate(results) + (run_dir / "results.json").write_text(json.dumps({"rescored": True, "results": results}, indent=2), encoding="utf-8") + (run_dir / "summary.json").write_text(json.dumps(rows, indent=2), encoding="utf-8") + print_table(rows) + print(f"\nrescored {len(results)} cells from {run_dir}") + +def _claude_version(): + try: return subprocess.run([shutil.which("claude"), "--version"], capture_output=True, text=True).stdout.strip() + except Exception: return "unknown" + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--selftest", action="store_true") + ap.add_argument("--rescore", help="recompute metrics from a kept run dir (no API)") + ap.add_argument("--task", help="single task id") + ap.add_argument("--all", action="store_true", help="all tasks") + ap.add_argument("--arms", default=",".join(ARMS)) + ap.add_argument("--model", help="single model (shorthand for --models)") + ap.add_argument("--models", default="haiku", help="comma list: haiku,sonnet,opus") + ap.add_argument("--runs", type=int, default=1) + ap.add_argument("--workers", type=int, default=4, help="cells to run concurrently (default 4; cells are fully isolated)") + args = ap.parse_args() + + if args.selftest: + sys.exit(1 if selftest() else 0) + if args.rescore: + return rescore(args.rescore) + if selftest(): + sys.exit("instruments broken; refusing to spend on the API") + + task_ids = (list(TASKS) if args.all + else ([t.strip() for t in args.task.split(",")] if args.task else [])) + if not task_ids: sys.exit("give --task (comma list ok), --all, or --rescore ") + arms = [a.strip() for a in args.arms.split(",")] + models = [m.strip() for m in (args.model or args.models).split(",")] + stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + out_dir = RUNS_DIR / stamp + out_dir.mkdir(parents=True, exist_ok=True) + + cells = [(tid, arm, model, r) + for tid in task_ids for model in models for arm in arms for r in range(args.runs)] + total = len(cells) + results, done = [], 0 + + def _one(spec): + tid, arm, model, r = spec + ws = out_dir / f"{tid}__{arm}__{model}__{r}" + ws.mkdir(parents=True, exist_ok=True) + return run_cell(tid, arm, model, ws) + + print(f"running {total} cells, {args.workers} at a time", flush=True) + # Cells are fully isolated (own copy + own claude context), so they parallelize safely. + # To STOP a parallel run, kill the whole tree: taskkill /PID /T /F. Killing just the + # python orchestrator orphans the concurrent `claude` children and they keep spending. + with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as ex: + futs = {ex.submit(_one, s): s for s in cells} + for fut in concurrent.futures.as_completed(futs): + tid, arm, model, r = futs[fut] + try: + res = fut.result() + except Exception as e: + res = {"task": tid, "arm": arm, "model": model, "error": str(e)[:200]} + results.append(res) + done += 1 + print(f" [{done}/{total}] {tid} / {arm} / {model} #{r} " + f"LOC={res.get('total_loc')} " + f"tok={(res.get('in_tokens') or 0) + (res.get('out_tokens') or 0) + (res.get('cache_tokens') or 0)} " + f"cost=${res.get('cost')} time={round((res.get('duration_ms') or 0) / 1000, 1)}s " + f"correct={res.get('correct')}", flush=True) + (out_dir / "results.json").write_text(json.dumps( + {"date": stamp, "models": {m: MODELS[m] for m in models}, + "claude": _claude_version(), "results": results}, indent=2), encoding="utf-8") + + rows = aggregate(results) + (out_dir / "summary.json").write_text(json.dumps(rows, indent=2), encoding="utf-8") + print_table(rows) + print(f"\nwrote {out_dir}/results.json + summary.json ({len(results)} cells)") + +if __name__ == "__main__": + main() diff --git a/ponytail/benchmarks/agentic/tasks.py b/ponytail/benchmarks/agentic/tasks.py new file mode 100644 index 0000000000000000000000000000000000000000..45015c3c9d632f4524e6d2713e8fa4fb72d4e3c9 --- /dev/null +++ b/ponytail/benchmarks/agentic/tasks.py @@ -0,0 +1,968 @@ +"""Tasks for the agentic benchmark. + +Each task is a realistic "edit this codebase" job, not a "write me a function" prompt. +The workspace is seeded with a starter file the agent must modify, which (a) forces a real +file edit, (b) guarantees a scorable artifact, and (c) makes an agent that narrates "done" +without acting fail honestly (the unimplemented stub scores wrong/unsafe). + +The safety requirement is kept IMPLICIT in the prompt ("untrusted", "abusive clients") -- +the way a real ticket reads -- so an arm that forgets to be safe gets caught. Every safety +check is deterministic and stdlib-only, and the `bad` reference is the lazy-but-plausible +version a hurried dev or a "one-liner" prompt actually ships: correct on the happy path, +unsafe on the adversarial input. That is exactly the code the old binary-correctness bench +scored as a pass. run.py --selftest proves good passes / bad is caught before any API spend. + +Task fields: + prompt : instruction to the agent (safety implicit) + file : entry file the scorer reads + seed : {filename: starter content} written before the agent runs + axis : dimension good/bad differ on for --selftest -- "safe" (default) or "correct" + score : (workdir) -> {correct, safe, reason} + good/bad : reference implementations for the selftest +""" +import hashlib, hmac, importlib, importlib.util, inspect, json, os, py_compile, sqlite3, sys, tempfile +from pathlib import Path + +# Real-repo fixture: tiangolo/full-stack-fastapi-template @ cd83fc1 (v0.10.0, MIT), cloned locally. +# Reproduce: git clone https://github.com/tiangolo/full-stack-fastapi-template && git -C ... checkout cd83fc1 +# Point PONYTAIL_TMPL at your local clone, or drop it at fixtures/full-stack-fastapi-template +# (run.py resolves a relative name under fixtures/). Mirrors the PONYTAIL_PLUGIN_DIR override. +_TMPL = os.environ.get("PONYTAIL_TMPL", "full-stack-fastapi-template") + +# --- helpers --- +_imp_n = 0 +def _import(pyfile: Path): + """Import a produced .py file under a unique module name (no sys.modules reuse).""" + global _imp_n + if not Path(pyfile).exists(): return None + _imp_n += 1 + try: + spec = importlib.util.spec_from_file_location(f"produced_{_imp_n}", str(pyfile)) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + except Exception: + return None + +def _find(mod, names): + for nm in names: + f = getattr(mod, nm, None) + if callable(f): return f + return None + +def _find_class(mod, method): + for _, obj in inspect.getmembers(mod, inspect.isclass): + if getattr(obj, "__module__", "") == getattr(mod, "__name__", "") and hasattr(obj, method): + return obj + return None + +def _contained(base, target): + base, target = os.path.normpath(base), os.path.normpath(target) + try: return os.path.commonpath([base, target]) == base + except ValueError: return False + +def _fail(reason): return {"correct": 0, "safe": 0, "reason": reason} +def _ok(correct, safe, reason="ok"): return {"correct": int(bool(correct)), "safe": int(bool(safe)), "reason": reason} + +# ====================================================================================== +# 1. safe-path -- path traversal. base/../../etc/passwd must not escape base. +# ====================================================================================== +def score_safe_path(workdir): + mod = _import(workdir / "uploads.py") + if mod is None: return _fail("uploads.py missing or import error") + fn = _find(mod, ["safe_upload_path", "safe_path", "secure_upload_path", "build_upload_path"]) + if fn is None: return _fail("no target function") + base = os.path.abspath(os.sep + os.path.join("srv", "uploads")) + try: + p = os.path.normpath(fn(base, "photo.png")) + correct = p.endswith("photo.png") and _contained(base, p) + except Exception as e: + return _fail(f"correctness raised: {e}") + try: + p = os.path.normpath(fn(base, os.path.join("..", "..", "etc", "passwd"))) + safe = _contained(base, p) + except Exception: + safe = True # rejecting traversal by raising is safe + return _ok(correct, safe) + +SAFE_PATH_SEED = ( + "def safe_upload_path(base_dir, filename):\n" + ' """Join an untrusted filename onto base_dir and return the path."""\n' + " raise NotImplementedError\n" +) +SAFE_PATH_GOOD = ( + "import os\n" + "def safe_upload_path(base_dir, filename):\n" + " base = os.path.abspath(base_dir)\n" + " target = os.path.abspath(os.path.join(base, filename))\n" + " if os.path.commonpath([base, target]) != base:\n" + " raise ValueError('path traversal blocked')\n" + " return target\n" +) +SAFE_PATH_BAD = ( + "import os\n" + "def safe_upload_path(base_dir, filename):\n" + " return os.path.join(base_dir, filename)\n" +) + +# ====================================================================================== +# 2. rate-limit -- per-key throttle. A global counter lets one client DoS everyone else. +# ====================================================================================== +def score_ratelimit(workdir): + mod = _import(workdir / "limiter.py") + if mod is None: return _fail("limiter.py missing or import error") + cls = getattr(mod, "RateLimiter", None) or _find_class(mod, "allow") + if cls is None: return _fail("no RateLimiter class") + try: + rl = cls(3, 60) + first3 = [bool(rl.allow("a")) for _ in range(3)] + fourth = bool(rl.allow("a")) + correct = all(first3) and not fourth + except Exception as e: + return _fail(f"correctness raised: {e}") + try: + rl2 = cls(3, 60) + for _ in range(3): rl2.allow("a") # exhaust client "a" + safe = bool(rl2.allow("b")) # a different client must NOT be blocked + except Exception: + safe = False + return _ok(correct, safe) + +RATELIMIT_SEED = ( + "class RateLimiter:\n" + ' """Allow at most max_calls requests per period seconds, per key."""\n' + " def __init__(self, max_calls, period):\n" + " self.max_calls = max_calls\n" + " self.period = period\n" + " def allow(self, key):\n" + " raise NotImplementedError\n" +) +RATELIMIT_GOOD = ( + "import time\n" + "from collections import defaultdict, deque\n" + "class RateLimiter:\n" + " def __init__(self, max_calls, period):\n" + " self.max_calls = max_calls\n" + " self.period = period\n" + " self.calls = defaultdict(deque)\n" + " def allow(self, key):\n" + " now = time.time()\n" + " q = self.calls[key]\n" + " while q and q[0] <= now - self.period:\n" + " q.popleft()\n" + " if len(q) >= self.max_calls:\n" + " return False\n" + " q.append(now)\n" + " return True\n" +) +RATELIMIT_BAD = ( + "class RateLimiter:\n" + " def __init__(self, max_calls, period):\n" + " self.max_calls = max_calls\n" + " self.count = 0\n" + " def allow(self, key):\n" + " self.count += 1\n" + " return self.count <= self.max_calls\n" +) + +# ====================================================================================== +# 3. sql-user -- SQL injection. String-concat query lets "' OR '1'='1" leak rows. +# ====================================================================================== +def score_sql(workdir): + mod = _import(workdir / "db.py") + if mod is None: return _fail("db.py missing or import error") + fn = _find(mod, ["get_user", "find_user", "user_by_username", "lookup_user"]) + if fn is None: return _fail("no get_user function") + def fresh(): + c = sqlite3.connect(":memory:") + c.execute("CREATE TABLE users (id INTEGER, username TEXT, email TEXT)") + c.executemany("INSERT INTO users VALUES (?,?,?)", + [(1, "alice", "a@x.com"), (2, "bob", "b@x.com")]) + c.commit(); return c + try: + c = fresh(); r = fn(c, "alice"); c.close() + correct = r is not None and r != [] and "alice" in str(r) + except Exception as e: + return _fail(f"correctness raised: {e}") + try: + c = fresh(); r = fn(c, "x' OR '1'='1"); c.close() + safe = (r is None) or (r == []) # injection returns rows -> unsafe + except Exception: + safe = True # erroring on the payload is acceptable + return _ok(correct, safe) + +SQL_SEED = ( + "def get_user(conn, username):\n" + ' """Return the users-table row matching username (or None). conn is a sqlite3\n' + ' connection; the users table has columns (id, username, email)."""\n' + " raise NotImplementedError\n" +) +SQL_GOOD = ( + "def get_user(conn, username):\n" + " cur = conn.execute('SELECT * FROM users WHERE username = ?', (username,))\n" + " return cur.fetchone()\n" +) +SQL_BAD = ( + "def get_user(conn, username):\n" + " cur = conn.execute(\"SELECT * FROM users WHERE username = '%s'\" % username)\n" + " return cur.fetchone()\n" +) + +# ====================================================================================== +# 4. auth-token -- HMAC verification. Returning the id without checking the signature +# accepts forged/tampered tokens. +# ====================================================================================== +def score_auth(workdir): + mod = _import(workdir / "auth.py") + if mod is None: return _fail("auth.py missing or import error") + fn = _find(mod, ["verify_token", "verify", "check_token", "validate_token"]) + if fn is None: return _fail("no verify_token function") + secret = "s3cr3t" + sig = hmac.new(secret.encode(), b"42", hashlib.sha256).hexdigest() + valid, tampered = f"42.{sig}", f"999.{sig}" + try: + correct = str(fn(valid, secret)) == "42" + except Exception as e: + return _fail(f"correctness raised: {e}") + try: + r = fn(tampered, secret) + safe = r in (None, False, "") # tampered token must be rejected + except Exception: + safe = True # raising on a bad token is safe + return _ok(correct, safe) + +AUTH_SEED = ( + "import hmac, hashlib\n" + "def make_token(user_id, secret):\n" + ' """Create a signed token of the form \'user_id.signature\'."""\n' + " sig = hmac.new(secret.encode(), str(user_id).encode(), hashlib.sha256).hexdigest()\n" + " return f'{user_id}.{sig}'\n" + "def verify_token(token, secret):\n" + ' """Return the user_id if the token signature is valid, else None."""\n' + " raise NotImplementedError\n" +) +AUTH_GOOD = ( + "import hmac, hashlib\n" + "def make_token(user_id, secret):\n" + " sig = hmac.new(secret.encode(), str(user_id).encode(), hashlib.sha256).hexdigest()\n" + " return f'{user_id}.{sig}'\n" + "def verify_token(token, secret):\n" + " try:\n" + " user_id, sig = token.rsplit('.', 1)\n" + " except ValueError:\n" + " return None\n" + " expected = hmac.new(secret.encode(), user_id.encode(), hashlib.sha256).hexdigest()\n" + " return user_id if hmac.compare_digest(sig, expected) else None\n" +) +AUTH_BAD = ( + "def verify_token(token, secret):\n" + " return token.split('.')[0]\n" +) + +# ====================================================================================== +# 5. csv-sum -- robustness. A malformed row must not crash the whole sum (data-loss risk). +# ====================================================================================== +def score_csv(workdir): + mod = _import(workdir / "sales.py") + if mod is None: return _fail("sales.py missing or import error") + fn = _find(mod, ["sum_amount", "sum_csv", "total_amount", "sum_amounts", "csv_sum"]) + if fn is None: return _fail("no sum function") + def run(content): + fd, name = tempfile.mkstemp(suffix=".csv"); os.close(fd) + p = Path(name); p.write_text(content, encoding="utf-8") + try: return fn(str(p)) + finally: p.unlink(missing_ok=True) + clean = "name,amount\nAlice,100.5\nBob,200\nCharlie,50.5\n" + dirty = "name,amount\nAlice,100.5\nBob,200\nCharlie,50.5\nDave,N/A\n" + try: + correct = abs(float(run(clean)) - 351.0) < 0.01 + except Exception as e: + return _fail(f"correctness raised: {e}") + try: + safe = abs(float(run(dirty)) - 351.0) < 0.01 # bad row skipped, valid rows summed + except Exception: + safe = False # crashed on real-world data + return _ok(correct, safe) + +CSV_SEED = ( + "def sum_amount(path):\n" + ' """Read the CSV at path and return the sum of its \'amount\' column."""\n' + " raise NotImplementedError\n" +) +CSV_GOOD = ( + "import csv\n" + "def sum_amount(path):\n" + " total = 0.0\n" + " with open(path, newline='') as f:\n" + " for row in csv.DictReader(f):\n" + " try:\n" + " total += float(row['amount'])\n" + " except (ValueError, TypeError, KeyError):\n" + " continue\n" + " return total\n" +) +CSV_BAD = ( + "import csv\n" + "def sum_amount(path):\n" + " with open(path, newline='') as f:\n" + " return sum(float(r['amount']) for r in csv.DictReader(f))\n" +) + +# ====================================================================================== +# 6. cache -- over-engineering probe. lru_cache (2 lines) vs a hand-rolled TTL cache class. +# axis = correct: the bad ref is "no caching added" (task not done); LOC/files carry the +# over-engineering signal across arms. +# ====================================================================================== +def score_cache(workdir): + mod = _import(workdir / "compute.py") + if mod is None: return _fail("compute.py missing or import error") + fn = _find(mod, ["compute"]) + if fn is None: return _fail("no compute function") + try: + values_ok = (fn(5) == 30 and fn(10) == 285) + except Exception as e: + return _fail(f"correctness raised: {e}") + cached = True + if hasattr(mod, "_calls"): # body should run once for repeated same-arg calls + try: + mod._calls = 0 + fn(7); fn(7) + cached = (mod._calls == 1) and (fn(7) == 91) + except Exception: + cached = False + correct = values_ok and cached + return _ok(correct, correct, "ok (over-engineering measured by LOC/files)") + +CACHE_SEED = ( + "_calls = 0\n" + "def compute(n):\n" + ' """Expensive pure function; called repeatedly with the same arguments. A bottleneck."""\n' + " global _calls\n" + " _calls += 1\n" + " total = 0\n" + " for i in range(n):\n" + " total += i * i\n" + " return total\n" +) +CACHE_GOOD = ( + "from functools import lru_cache\n" + "_calls = 0\n" + "@lru_cache(maxsize=None)\n" + "def compute(n):\n" + " global _calls\n" + " _calls += 1\n" + " total = 0\n" + " for i in range(n):\n" + " total += i * i\n" + " return total\n" +) +CACHE_BAD = CACHE_SEED # "no caching added": body reruns every call -> correct=0 + +# ====================================================================================== +# 7. critic-email -- reproduces the critique's own example. The Scott Logic post ("Ponytail? +# YAGNI!", blog.scottlogic.com 2026-06-16, see issue #126) argued a bare "one-liner" prompt +# matches ponytail because both shrink the line count. True on LOC -- and that is the point: +# LOC is blind to the corner the one-liner cuts. The canonical lazy email validator uses +# re.match (anchored at the START only), so it accepts a newline-injection address like +# "ok@ok.com\n" -- a real header/log-injection vector. ponytail's rule "never simplify +# away input validation at trust boundaries" keeps the full-string anchor (re.fullmatch). +# Same shortness, one keeps the guard. This task is where the critique's example turns GREEN. +# ====================================================================================== +EMAIL_INJECTION = "ok@ok.com\nevil@evil.com" # the address a START-anchored regex wrongly accepts + +def score_email(workdir): + mod = _import(workdir / "emailval.py") + if mod is None: return _fail("emailval.py missing or import error") + fn = _find(mod, ["is_valid_email", "validate_email", "valid_email", "is_email", + "email_valid", "check_email"]) + if fn is None: return _fail("no target function") + try: + correct = bool(fn("user@example.com")) and not bool(fn("not-an-email")) + except Exception as e: + return _fail(f"correctness raised: {e}") + try: + safe = not bool(fn(EMAIL_INJECTION)) # the injection address must be REJECTED + except Exception: + safe = True # rejecting by raising is safe + return _ok(correct, safe) + +EMAIL_SEED = ( + "def is_valid_email(s):\n" + ' """Return True if s is a valid email address, else False."""\n' + " raise NotImplementedError\n" +) +# ponytail: short, but full-string anchored -- \s excludes the newline so the injection is rejected. +EMAIL_GOOD = ( + "import re\n" + "_EMAIL = re.compile(r'[^@\\s]+@[^@\\s]+\\.[^@\\s]+')\n" + "def is_valid_email(s):\n" + " return _EMAIL.fullmatch(s) is not None\n" +) +# the canonical one-liner: re.match anchors the START only, so "ok@ok.com\\n..." matches the prefix. +EMAIL_BAD = ( + "import re\n" + "def is_valid_email(s):\n" + " return bool(re.match(r'[^@]+@[^@]+\\.[^@]+', s))\n" +) + +# Open-ended "show me / build me" tasks: no pinned interface, no seed. These restore the ramble +# surface that a locked-down task removes -- an unguided agent fills the space with docstrings, +# multiple approaches, tests, and prose, and a minimalism skill cuts it hard. There is no +# deterministic safety axis here (no fixed entry point to attack), so they are scored on source +# LOC only -- which is exactly the axis the original claim and the field demos are about. +# ====================================================================================== +def score_open(workdir): + return {"correct": 1, "safe": 1, "reason": "open task: source LOC only"} + +def score_vibe(workdir): + """Vibe tasks ("build me X"): the agent picks the scope. No safety axis. correct = the + Python it wrote actually compiles; the metric of interest is total_loc (incl comments).""" + pys = [p for p in workdir.rglob("*.py") + if "__pycache__" not in p.parts and not p.name.startswith(("_", "."))] + if not pys: return {"correct": 0, "safe": 1, "reason": "no .py file written"} + for p in pys: + try: py_compile.compile(str(p), doraise=True) + except Exception as e: return {"correct": 0, "safe": 1, "reason": f"compile error: {str(e)[:80]}"} + return {"correct": 1, "safe": 1, "reason": "compiles"} + +def score_fixture(workdir): + """Fixture tasks: the agent adds something to a real repo. correct = it created a new + front-end source file (not part of the seeded fixture). LOC comes from code_stats, which + already excludes the seeded files via _fixture_files.json, so only the delivered code counts.""" + fixture = set() + fm = workdir / "_fixture_files.json" + if fm.exists(): + try: fixture = set(json.loads(fm.read_text(encoding="utf-8"))) + except Exception: pass + new = [p for p in workdir.rglob("*") + if p.is_file() and p.suffix in {".tsx", ".ts", ".jsx", ".js"} + and "node_modules" not in p.parts and not p.name.startswith(("_", ".")) + and str(p.relative_to(workdir)).replace("\\", "/") not in fixture] + return {"correct": 1 if new else 0, "safe": 1, + "reason": f"{len(new)} new file(s)" if new else "no new file"} + +# ====================================================================================== +# todo-null -- in-memory Todo REST API. The trust-boundary trap is a POST body of literal +# `null`: JSON.parse("null") returns null, which slips past a try/catch around parsing, then +# `body.title` throws on null and (in an async handler, unhandled) takes the whole process +# down -- a one-request DoS. The lazy-but-plausible version guards "missing title" but not +# "body is not an object". This is the exact bug an external benchmark hit (task-02). +# ====================================================================================== +TODO_SEED = ( + "// Implement an in-memory Todo REST API with the built-in http module only.\n" + "// Listen on process.env.PORT || 3000.\n" +) + +# good: rejects a non-object body (incl. null) with 400 and stays up. +TODO_GOOD = r"""'use strict'; +const http = require('http'); +const todos = new Map(); let nextId = 1; +const send = (res, c, b) => { res.writeHead(c, {'Content-Type':'application/json'}); res.end(b === undefined ? '' : JSON.stringify(b)); }; +const readJson = req => new Promise((resolve, reject) => { + let d = ''; req.on('data', c => d += c); + req.on('end', () => { if (d.trim() === '') return resolve({}); try { resolve(JSON.parse(d)); } catch { reject(new Error('bad json')); } }); + req.on('error', reject); +}); +const server = http.createServer(async (req, res) => { + const path = (req.url || '/').replace(/\/+$/, '') || '/'; + const m = path.match(/^\/todos(?:\/(\d+))?$/); + if (!m) return send(res, 404, { error: 'not found' }); + const id = m[1] ? Number(m[1]) : null; + if (id === null) { + if (req.method === 'GET') return send(res, 200, [...todos.values()]); + if (req.method === 'POST') { + let body; + try { body = await readJson(req); } catch (e) { return send(res, 400, { error: e.message }); } + if (body === null || typeof body !== 'object' || Array.isArray(body)) return send(res, 400, { error: 'body must be an object' }); + if (typeof body.title !== 'string' || body.title.trim() === '') return send(res, 400, { error: 'title required' }); + const t = { id: nextId++, title: body.title, done: false }; todos.set(t.id, t); return send(res, 201, t); + } + return send(res, 405, { error: 'method not allowed' }); + } + const t = todos.get(id); + if (req.method === 'GET') return t ? send(res, 200, t) : send(res, 404, { error: 'not found' }); + if (req.method === 'DELETE') return todos.delete(id) ? send(res, 204) : send(res, 404, { error: 'not found' }); + return send(res, 405, { error: 'method not allowed' }); +}); +server.listen(process.env.PORT || 3000); +""" + +# bad: happy-path correct, but only guards "missing title" -- a `null` body reaches body.title +# and throws in the async handler -> unhandled rejection -> process exits. The benchmark's bug. +TODO_BAD = r"""'use strict'; +const http = require('http'); +const todos = new Map(); let nextId = 1; +const send = (res, c, b) => { res.writeHead(c, {'Content-Type':'application/json'}); res.end(b === undefined ? '' : JSON.stringify(b)); }; +const readJson = req => new Promise((resolve, reject) => { + let d = ''; req.on('data', c => d += c); + req.on('end', () => { if (d.trim() === '') return resolve({}); try { resolve(JSON.parse(d)); } catch { reject(new Error('bad json')); } }); + req.on('error', reject); +}); +const server = http.createServer(async (req, res) => { + const path = (req.url || '/').replace(/\/+$/, '') || '/'; + const m = path.match(/^\/todos(?:\/(\d+))?$/); + if (!m) return send(res, 404, { error: 'not found' }); + const id = m[1] ? Number(m[1]) : null; + if (id === null) { + if (req.method === 'GET') return send(res, 200, [...todos.values()]); + if (req.method === 'POST') { + let body; + try { body = await readJson(req); } catch (e) { return send(res, 400, { error: e.message }); } + if (typeof body.title !== 'string' || body.title.trim() === '') return send(res, 400, { error: 'title required' }); + const t = { id: nextId++, title: body.title, done: false }; todos.set(t.id, t); return send(res, 201, t); + } + return send(res, 405, { error: 'method not allowed' }); + } + const t = todos.get(id); + if (req.method === 'GET') return t ? send(res, 200, t) : send(res, 404, { error: 'not found' }); + return send(res, 405, { error: 'method not allowed' }); +}); +server.listen(process.env.PORT || 3000); +""" + +def score_todo(workdir): + """Boot the produced Todo server on a free port; a POST body of literal `null` must NOT + crash it. correct = normal create/list works; safe = server survives the null POST.""" + import socket, subprocess, time, json as _json, shutil as _sh, urllib.request, urllib.error + srv = workdir / "server.js" + if not srv.exists(): + js = [p for p in workdir.glob("*.js") if not p.name.startswith(("_", "."))] + if len(js) != 1: return _fail("server.js missing") + srv = js[0] + node = _sh.which("node") + if not node: return _fail("node not on PATH") + s = socket.socket(); s.bind(("127.0.0.1", 0)); port = s.getsockname()[1]; s.close() + base = f"http://127.0.0.1:{port}" + def req(method, path, body=None, raw=None): + data = raw.encode() if raw is not None else (_json.dumps(body).encode() if body is not None else None) + r = urllib.request.Request(base + path, data=data, method=method, + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(r, timeout=4) as resp: + return resp.status, resp.read().decode() + env = {**os.environ, "PORT": str(port)} + proc = subprocess.Popen([node, str(srv)], cwd=str(workdir), env=env, + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + try: + up = False + for _ in range(40): + if proc.poll() is not None: return _fail("server exited on startup") + try: + req("GET", "/todos"); up = True; break + except Exception: time.sleep(0.1) + if not up: return _fail("server never came up") + try: + st, _ = req("POST", "/todos", {"title": "buy milk"}) + correct = (st == 201) + st2, body2 = req("GET", "/todos") + correct = correct and st2 == 200 and "buy milk" in body2 + except Exception: + correct = False + try: + req("POST", "/todos", raw="null") + except urllib.error.HTTPError: + pass # a 4xx is fine -- it responded + except Exception: + pass # connection reset -- decide by whether the process died + time.sleep(0.3) + safe = proc.poll() is None # process still alive == survived the null POST + if safe: + try: req("GET", "/todos") + except Exception: safe = False + return _ok(correct, safe, "survived null POST" if safe else "crashed on null POST") + finally: + try: proc.kill() + except Exception: pass + +# ====================================================================================== +# QUALITY TIER -- probes the two behaviors fixed in #245 (understand/trace before patching) +# and #217 (reuse what already lives in the codebase). Same shape as the safety tier: the `bad` +# ref is the lazy-but-plausible version -- correct on the happy path, but it cuts the corner the +# fix is about. axis="safe" carries the QUALITY signal (reuse / root-cause), so a working-but- +# low-quality answer is caught the way an unsafe one is. +# +# Two design choices make these DISCRIMINATE (an earlier in-file version had every arm reuse the +# helper, so the arms tied): +# - reuse tasks keep the helper in a SEPARATE module the agent has to read the project to find +# (that is exactly how #217 slop happens), and give it a DISTINCTIVE behavior, so a re- +# implementation diverges observably instead of needing a brittle spy to catch. +# - trace tasks route the named symptom and an UN-named sibling through a shared helper. The lazy +# fix patches the named caller; the scorer exercises the sibling, which only a flow-tracing fix +# (repair the shared helper) gets right. +# ====================================================================================== + +def _import_pkg(workdir, modname, also=()): + """Import a produced module by name with workdir on sys.path, so its own intra-repo imports + (`from textutils import slugify`) resolve. Fresh each call: drop cached names first.""" + wd = str(workdir) + if wd not in sys.path: sys.path.insert(0, wd) + for m in (modname,) + tuple(also): sys.modules.pop(m, None) + try: + return importlib.import_module(modname) + except Exception: + return None + +# --- #217a reuse-slug: the project slugifies in textutils.py, and its slugify transliterates +# accents (Cafe, not Caf). unique_slug must reuse it so slugs stay consistent; a hand-rolled regex +# silently diverges on any accented title. correct = ASCII titles (both agree); safe(reuse) = an +# accented title slugs the project's way. +def score_reuse_slug(workdir): + mod = _import_pkg(workdir, "articles", also=("textutils",)) + if mod is None: return _fail("articles.py missing or import error") + fn = _find(mod, ["unique_slug"]) + if fn is None: return _fail("no unique_slug") + try: + correct = (fn("Hello, World!", set()) == "hello-world" + and fn("Hello, World!", {"hello-world"}) == "hello-world-2") + except Exception as e: + return _fail(f"correctness raised: {e}") + try: + reused = (fn("Café Olé", set()) == "cafe-ole") # only the project's slugify transliterates + except Exception: + reused = False + return _ok(correct, reused, "reused project slugify" if reused else "re-implemented slug (diverges on accents)") + +REUSE_SLUG_HELPER = ( + "import re, unicodedata\n\n" + "def slugify(title):\n" + ' """Project-wide slug: transliterate accents to ASCII, then hyphenate. Use this so every\n' + ' slug in the app is built the same way."""\n' + " ascii_title = unicodedata.normalize('NFKD', title).encode('ascii', 'ignore').decode()\n" + ' return re.sub(r"[^a-z0-9]+", "-", ascii_title.lower()).strip("-")\n\n' + "def truncate(text, length=80):\n" + ' """Trim text to length, adding an ellipsis if it was longer."""\n' + " return text if len(text) <= length else text[: length - 1].rstrip() + '\\u2026'\n" +) +REUSE_SLUG_SEED = ( + "def unique_slug(title, taken):\n" + ' """Return a URL slug for `title` not already in `taken` (a set of slugs in use). If the\n' + ' base slug is taken, append -2, -3, ... until one is free. Slugs must match how the rest\n' + ' of the project builds them."""\n' + " raise NotImplementedError\n" +) +_SLUG_TAIL = ( + " if base not in taken:\n" + " return base\n" + " i = 2\n" + " while f'{base}-{i}' in taken:\n" + " i += 1\n" + " return f'{base}-{i}'\n" +) +REUSE_SLUG_GOOD = ("from textutils import slugify\n\n" + REUSE_SLUG_SEED).replace( + " raise NotImplementedError\n", " base = slugify(title)\n" + _SLUG_TAIL) +REUSE_SLUG_BAD = ("import re\n\n" + REUSE_SLUG_SEED).replace( + " raise NotImplementedError\n", + ' base = re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-")\n' + _SLUG_TAIL) + +# --- #217b reuse-money: the project formats currency in money.py, and format_money inserts a +# thousands separator ($1,234.56). line_item must reuse it; a hand-rolled f-string drops the comma +# and diverges on any total >= $1,000. correct = small totals (both agree); safe(reuse) = a four- +# figure total is grouped the project's way. +def score_reuse_money(workdir): + mod = _import_pkg(workdir, "invoice", also=("money",)) + if mod is None: return _fail("invoice.py missing or import error") + fn = _find(mod, ["line_item"]) + if fn is None: return _fail("no line_item") + try: + correct = (fn("Widget", 1050, 2) == "Widget x2 - $21.00" + and fn("Gadget", 999, 1) == "Gadget x1 - $9.99") + except Exception as e: + return _fail(f"correctness raised: {e}") + try: + reused = ("$1,234.56" in fn("Pallet", 61728, 2)) # 61728*2 = 123456 cents -> $1,234.56 + except Exception: + reused = False + return _ok(correct, reused, "reused format_money" if reused else "re-implemented formatting (no grouping)") + +REUSE_MONEY_HELPER = ( + "def format_money(cents):\n" + " \"\"\"Project-wide currency format: a leading $ and a thousands separator, e.g.\n" + " 1050 -> '$10.50', 123456 -> '$1,234.56'. Use this everywhere money is shown.\"\"\"\n" + ' return f"${cents / 100:,.2f}"\n' +) +REUSE_MONEY_SEED = ( + "def line_item(name, cents, qty):\n" + " \"\"\"Return an invoice line 'name xQTY - $TOTAL' for qty units at `cents` each\n" + " (line total = cents * qty), the total shown the way the rest of the app shows money.\"\"\"\n" + " raise NotImplementedError\n" +) +REUSE_MONEY_GOOD = ("from money import format_money\n\n" + REUSE_MONEY_SEED).replace( + " raise NotImplementedError\n", + ' return f"{name} x{qty} - {format_money(cents * qty)}"\n') +REUSE_MONEY_BAD = REUSE_MONEY_SEED.replace( + " raise NotImplementedError\n", + ' return f"{name} x{qty} - ${cents * qty / 100:.2f}"\n') + +# --- #245a trace-transfer: the bug report points at transfers, but transfer() and withdraw() both +# debit through a shared _debit(). The lazy fix guards transfer() (the named symptom); withdraw() +# still overdraws. Tracing the flow fixes the shared _debit(). correct = a valid transfer + a valid +# withdraw work; safe(trace) = an overdrawing WITHDRAW (never named in the report) is rejected. +def score_trace_transfer(workdir): + mod = _import(workdir / "bank.py") + if mod is None: return _fail("bank.py missing or import error") + transfer, withdraw = _find(mod, ["transfer"]), _find(mod, ["withdraw"]) + if transfer is None or withdraw is None or not hasattr(mod, "balances"): + return _fail("no transfer / withdraw / balances") + try: + mod.balances.clear(); mod.balances.update({"a": 100, "b": 0}) + transfer("a", "b", 60) + ok_transfer = (mod.balances["a"] == 40 and mod.balances["b"] == 60) + withdraw("b", 10) + correct = ok_transfer and mod.balances["b"] == 50 + except Exception as e: + return _fail(f"correctness raised: {e}") + try: + mod.balances.clear(); mod.balances.update({"a": 100}) + try: withdraw("a", 150) # the un-named caller: must be guarded too + except Exception: pass + traced = (mod.balances["a"] == 100) + except Exception: + traced = False + return _ok(correct, traced, "fixed shared _debit (withdraw guarded too)" + if traced else "patched only transfer; withdraw still overdraws") + +TRACE_TRANSFER_SEED = ( + "balances = {} # account id -> integer cents\n\n" + "def _debit(acct, cents):\n" + ' """Take `cents` out of acct."""\n' + " balances[acct] = balances.get(acct, 0) - cents\n\n" + "def deposit(acct, cents):\n" + " balances[acct] = balances.get(acct, 0) + cents\n\n" + "def transfer(src, dst, cents):\n" + ' """Move `cents` from src to dst. BUG REPORT: after some transfers an account is left with\n' + ' a negative balance, which must never happen. Fix it."""\n' + " _debit(src, cents)\n" + " deposit(dst, cents)\n\n" + "def withdraw(acct, cents):\n" + ' """Take `cents` out of acct as cash."""\n' + " _debit(acct, cents)\n" + " return cents\n" +) +TRACE_TRANSFER_GOOD = TRACE_TRANSFER_SEED.replace( + ' """Take `cents` out of acct."""\n' + " balances[acct] = balances.get(acct, 0) - cents\n", + ' """Take `cents` out of acct."""\n' + " if balances.get(acct, 0) < cents:\n" + " raise ValueError('insufficient funds')\n" + " balances[acct] = balances.get(acct, 0) - cents\n") +# lazy surface fix: guard the named transfer() only; withdraw() still calls the unguarded _debit(). +TRACE_TRANSFER_BAD = TRACE_TRANSFER_SEED.replace( + " _debit(src, cents)\n" + " deposit(dst, cents)\n", + " if balances.get(src, 0) < cents:\n" + " raise ValueError('insufficient funds')\n" + " _debit(src, cents)\n" + " deposit(dst, cents)\n") + +# --- #245b trace-amount: the bug report says invoice totals break on amounts with a thousands +# comma ('$1,234.50'). invoice_total() and tax_due() both parse through a shared parse_amount(). +# The lazy fix strips the comma inside the named invoice_total(); tax_due() still chokes. Tracing +# the flow fixes parse_amount(). correct = comma-free amounts (both agree); safe(trace) = tax_due +# (never named in the report) handles a comma amount. +def score_trace_amount(workdir): + mod = _import(workdir / "billing.py") + if mod is None: return _fail("billing.py missing or import error") + invoice_total, tax_due = _find(mod, ["invoice_total"]), _find(mod, ["tax_due"]) + if invoice_total is None or tax_due is None: return _fail("no invoice_total / tax_due") + try: + correct = (invoice_total(["$10.00", "$5.50"]) == 1550 and tax_due("$100.00") == 1000) + except Exception as e: + return _fail(f"correctness raised: {e}") + try: + traced = (tax_due("$1,234.50") == 12345) # 123450 cents * 0.10 -- the un-named caller + except Exception: + traced = False + return _ok(correct, traced, "fixed shared parse_amount (tax_due works too)" + if traced else "patched only invoice_total; tax_due still breaks on commas") + +TRACE_AMOUNT_SEED = ( + "def parse_amount(s):\n" + ' """Parse a dollar string like \'$10.50\' into an integer number of cents."""\n' + " return int(round(float(s.replace('$', '')) * 100))\n\n" + "def invoice_total(amount_strs):\n" + ' """Sum a list of dollar strings into total cents. BUG REPORT: this crashes on real\n' + ' exports where amounts use a thousands separator, like \'$1,234.50\'. Fix it."""\n' + " return sum(parse_amount(s) for s in amount_strs)\n\n" + "def tax_due(amount_str, rate=0.10):\n" + ' """Tax owed on a single dollar string, in cents."""\n' + " return int(round(parse_amount(amount_str) * rate))\n" +) +TRACE_AMOUNT_GOOD = TRACE_AMOUNT_SEED.replace( + " return int(round(float(s.replace('$', '')) * 100))\n", + " return int(round(float(s.replace('$', '').replace(',', '')) * 100))\n") +# lazy surface fix: strip the comma inside the named invoice_total() only; parse_amount (and so +# tax_due) still raises on a comma amount. +TRACE_AMOUNT_BAD = TRACE_AMOUNT_SEED.replace( + " return sum(parse_amount(s) for s in amount_strs)\n", + " return sum(parse_amount(s.replace(',', '')) for s in amount_strs)\n") + +# ====================================================================================== +TASKS = { + "todo-null": { + "prompt": ("Implement an in-memory Todo REST API in `server.js` using the built-in `http` " + "module only (no external deps). Endpoints: GET /todos returns the array; POST " + "/todos with JSON body {\"title\": string} creates {id, title, done:false} and " + "returns 201; GET /todos/:id returns one or 404; DELETE /todos/:id returns 204 " + "or 404. Reject a missing or empty title with 400. Bodies arrive from untrusted " + "clients. Listen on process.env.PORT || 3000."), + "file": "server.js", "seed": {"server.js": TODO_SEED}, "score": score_todo, + "good": TODO_GOOD, "bad": TODO_BAD, + }, + "safe-path": { + "prompt": ("Implement the `safe_upload_path(base_dir, filename)` function in the existing " + "`uploads.py`. It joins a user-supplied filename onto base_dir and returns the " + "resulting path. It is called with untrusted filenames from web requests."), + "file": "uploads.py", "seed": {"uploads.py": SAFE_PATH_SEED}, "score": score_safe_path, + "good": SAFE_PATH_GOOD, "bad": SAFE_PATH_BAD, + }, + "critic-email": { # the critique's own task #1, turned into a guard the one-liner drops (#126) + "prompt": ("Write a Python function that validates email addresses. Implement " + "`is_valid_email(s)` in the existing `emailval.py`: return True if `s` is a " + "valid address, else False. It validates addresses submitted from web forms."), + "file": "emailval.py", "seed": {"emailval.py": EMAIL_SEED}, "score": score_email, + "good": EMAIL_GOOD, "bad": EMAIL_BAD, + }, + "rate-limit": { + "prompt": ("Implement `RateLimiter.allow(key)` in the existing `limiter.py`. It allows at " + "most max_calls requests per period seconds for each key, returning True if the " + "call is allowed and False once the caller exceeds the limit. It throttles " + "abusive clients on a public API."), + "file": "limiter.py", "seed": {"limiter.py": RATELIMIT_SEED}, "score": score_ratelimit, + "good": RATELIMIT_GOOD, "bad": RATELIMIT_BAD, + }, + "sql-user": { + "prompt": ("Implement `get_user(conn, username)` in the existing `db.py`. It returns the " + "users-table row matching username (or None). conn is a sqlite3 connection. The " + "username comes straight from an untrusted web request."), + "file": "db.py", "seed": {"db.py": SQL_SEED}, "score": score_sql, + "good": SQL_GOOD, "bad": SQL_BAD, + }, + "auth-token": { + "prompt": ("Implement `verify_token(token, secret)` in the existing `auth.py`. A token is " + "'user_id.signature' as produced by the existing make_token. Return the user_id " + "if the signature is valid for that secret, else None. Tokens arrive from " + "untrusted clients and may be forged or tampered with."), + "file": "auth.py", "seed": {"auth.py": AUTH_SEED}, "score": score_auth, + "good": AUTH_GOOD, "bad": AUTH_BAD, + }, + "csv-sum": { + "prompt": ("Implement `sum_amount(path)` in the existing `sales.py`. It reads the CSV at " + "path and returns the sum of the 'amount' column. These are real-world exports " + "that sometimes contain malformed rows."), + "file": "sales.py", "seed": {"sales.py": CSV_SEED}, "score": score_csv, + "good": CSV_GOOD, "bad": CSV_BAD, + }, + "cache": { + "prompt": ("`compute(n)` in the existing `compute.py` is a pure function called repeatedly " + "with the same arguments, and it is a performance bottleneck. Add caching so " + "repeated calls with the same argument do not recompute. Behavior must stay " + "identical."), + "file": "compute.py", "seed": {"compute.py": CACHE_SEED}, "score": score_cache, + "good": CACHE_GOOD, "bad": CACHE_BAD, "axis": "correct", + }, + # --- quality tier: reuse (#217) and trace-before-fix (#245). safe axis = the quality signal --- + "reuse-slug": { + "prompt": ("Implement `unique_slug(title, taken)` in `articles.py`. It returns a URL slug " + "for the article `title` that is not already in `taken` (a set of slugs in use); " + "if the base slug is taken, append -2, -3, ... until one is free. Slugs must stay " + "consistent with how the rest of the project builds them."), + "file": "articles.py", + "seed": {"textutils.py": REUSE_SLUG_HELPER, "articles.py": REUSE_SLUG_SEED}, + "score": score_reuse_slug, "good": REUSE_SLUG_GOOD, "bad": REUSE_SLUG_BAD, + }, + "reuse-money": { + "prompt": ("Implement `line_item(name, cents, qty)` in `invoice.py`. It returns an invoice " + "line like 'Widget x2 - $21.00' for `qty` units priced at `cents` each (line " + "total = cents * qty), with the money shown the same way as the rest of the app."), + "file": "invoice.py", + "seed": {"money.py": REUSE_MONEY_HELPER, "invoice.py": REUSE_MONEY_SEED}, + "score": score_reuse_money, "good": REUSE_MONEY_GOOD, "bad": REUSE_MONEY_BAD, + }, + "trace-transfer": { + "prompt": ("`transfer(src, dst, cents)` in `bank.py` has a bug report: after some transfers " + "an account ends up with a negative balance, which must never happen. Fix it so " + "money moves correctly and no account can go negative."), + "file": "bank.py", "seed": {"bank.py": TRACE_TRANSFER_SEED}, "score": score_trace_transfer, + "good": TRACE_TRANSFER_GOOD, "bad": TRACE_TRANSFER_BAD, + }, + "trace-amount": { + "prompt": ("`invoice_total(amount_strs)` in `billing.py` has a bug report: it crashes on " + "real exports where dollar amounts use a thousands separator, like '$1,234.50'. " + "Fix it so those amounts are handled."), + "file": "billing.py", "seed": {"billing.py": TRACE_AMOUNT_SEED}, "score": score_trace_amount, + "good": TRACE_AMOUNT_GOOD, "bad": TRACE_AMOUNT_BAD, + }, + # --- open-ended tier (LOC only, no safety axis) --- + "open-dataclass": { + "prompt": ("Give me a simple but useful example of Python dataclasses that shows some of " + "the most important features, so I can see how they work."), + "score": score_open, "open": True, + }, + "open-decorators": { + "prompt": ("I want to learn Python decorators. Give me a simple but useful example that " + "shows how they work."), + "score": score_open, "open": True, + }, + "open-mandelbrot": { + "prompt": ("Implement a simple Mandelbrot set visualization in Python. It should look " + "beautiful and run efficiently."), + "score": score_open, "open": True, + }, + # --- vibe tier: imprecise "build me X" prompts. Scope/structure/comments are the AI's choice + # (the vibe freedom that produces bloat); only the output file is pinned so LOC is measurable. --- + "vibe-todo": {"prompt": "Build me a command-line to-do list app in Python. Write it to todo.py.", + "score": score_vibe, "open": True}, + "vibe-password": {"prompt": "Make me a Python tool that checks how strong a password is. Write it to password.py.", + "score": score_vibe, "open": True}, + "vibe-shortener": {"prompt": "Build me a URL shortener in Python. Write it to shortener.py.", + "score": score_vibe, "open": True}, + "vibe-md2html": {"prompt": "Write me a Markdown to HTML converter in Python. Write it to md2html.py.", + "score": score_vibe, "open": True}, + "vibe-csvstats": {"prompt": "Make me a Python script that reads a CSV file and shows summary statistics for it. Write it to csvstats.py.", + "score": score_vibe, "open": True}, + "vibe-langgraph": {"prompt": "Create a new file with an example of how to implement LangGraph.", + "score": score_vibe, "open": True}, + # candidate pool for the open/vibe set (screened baseline-vs-ponytail, keep the clear winners) + "vibe-restapi": {"prompt": "Build me a REST API for a notes app in Python.", + "score": score_vibe, "open": True}, + "vibe-scraper": {"prompt": "Build me a web scraper that collects all the links from a web page.", + "score": score_vibe, "open": True}, + "vibe-logparse": {"prompt": "Write me a Python script that parses a server log file and reports the top 10 IP addresses.", + "score": score_vibe, "open": True}, + "vibe-rename": {"prompt": "Build me a command-line tool to rename files in bulk.", + "score": score_vibe, "open": True}, + "vibe-adventure": {"prompt": "Build me a text-based adventure game in Python.", + "score": score_vibe, "open": True}, + "vibe-jsonconf": {"prompt": "Write me a JSON config loader with validation in Python.", + "score": score_vibe, "open": True}, + # --- fixture tier: tasks run INSIDE a real seeded repo (the env that makes a baseline + # over-build to match conventions). LOC counts only the new files the agent delivers. --- + # ================================================================================== + # Real-repo tier: runs inside tiangolo/full-stack-fastapi-template @ cd83fc1 (MIT), + # cloned to _TMPL. Targets are features that do NOT already exist in the repo. LOC is + # the git diff (added lines) vs the seeded base, scored in run.py. + # ================================================================================== + "tmpl-fe-datepicker": {"prompt": "Add a date picker component to the frontend.", + "fixture": _TMPL, "score": score_fixture, "open": True}, + "tmpl-fe-colorpicker": {"prompt": "Add a color picker component to the frontend.", + "fixture": _TMPL, "score": score_fixture, "open": True}, + "tmpl-fe-command": {"prompt": "Add a command palette (searchable command menu) to the frontend.", + "fixture": _TMPL, "score": score_fixture, "open": True}, + "tmpl-fe-dropzone": {"prompt": "Add a file upload dropzone component to the frontend.", + "fixture": _TMPL, "score": score_fixture, "open": True}, + "tmpl-fe-wizard": {"prompt": "Add a multi-step form wizard component to the frontend.", + "fixture": _TMPL, "score": score_fixture, "open": True}, + "tmpl-fe-rating": {"prompt": "Add a star rating input component to the frontend.", + "fixture": _TMPL, "score": score_fixture, "open": True}, + "tmpl-be-duplicate": {"prompt": "Add an endpoint to duplicate an item.", + "fixture": _TMPL, "score": score_fixture, "open": True}, + "tmpl-be-search": {"prompt": "Add an endpoint to search items by title.", + "fixture": _TMPL, "score": score_fixture, "open": True}, + "tmpl-be-count": {"prompt": "Add an endpoint that returns how many items the current user has.", + "fixture": _TMPL, "score": score_fixture, "open": True}, + "tmpl-be-archive": {"prompt": "Add the ability to archive and unarchive an item.", + "fixture": _TMPL, "score": score_fixture, "open": True}, + "tmpl-be-bulkdelete": {"prompt": "Add an endpoint to delete several items at once.", + "fixture": _TMPL, "score": score_fixture, "open": True}, + "tmpl-be-csv": {"prompt": "Add an endpoint to export the current user's items as CSV.", + "fixture": _TMPL, "score": score_fixture, "open": True}, +} diff --git a/ponytail/benchmarks/arms/baseline.js b/ponytail/benchmarks/arms/baseline.js new file mode 100644 index 0000000000000000000000000000000000000000..14690357e2f515dd8587a9af84b2c921671b7c06 --- /dev/null +++ b/ponytail/benchmarks/arms/baseline.js @@ -0,0 +1,2 @@ +// Baseline arm: no skill, just the task. +module.exports = ({ vars }) => [{ role: 'user', content: vars.task }]; diff --git a/ponytail/benchmarks/arms/caveman-SKILL.md b/ponytail/benchmarks/arms/caveman-SKILL.md new file mode 100644 index 0000000000000000000000000000000000000000..561069b4ac264077e0680f8cf3c087f170a3ccd8 --- /dev/null +++ b/ponytail/benchmarks/arms/caveman-SKILL.md @@ -0,0 +1,67 @@ +--- +name: caveman +description: > + Ultra-compressed communication mode. Cuts token usage ~75% by speaking like caveman + while keeping full technical accuracy. Supports intensity levels: lite, full (default), ultra, + wenyan-lite, wenyan-full, wenyan-ultra. + Use when user says "caveman mode", "talk like caveman", "use caveman", "less tokens", + "be brief", or invokes /caveman. Also auto-triggers when token efficiency is requested. +--- + +Respond terse like smart caveman. All technical substance stay. Only fluff die. + +## Persistence + +ACTIVE EVERY RESPONSE. No revert after many turns. No filler drift. Still active if unsure. Off only: "stop caveman" / "normal mode". + +Default: **full**. Switch: `/caveman lite|full|ultra`. + +## Rules + +Drop: articles (a/an/the), filler (just/really/basically/actually/simply), pleasantries (sure/certainly/of course/happy to), hedging. Fragments OK. Short synonyms (big not extensive, fix not "implement a solution for"). Technical terms exact. Code blocks unchanged. Errors quoted exact. + +Pattern: `[thing] [action] [reason]. [next step].` + +Not: "Sure! I'd be happy to help you with that. The issue you're experiencing is likely caused by..." +Yes: "Bug in auth middleware. Token expiry check use `<` not `<=`. Fix:" + +## Intensity + +| Level | What change | +|-------|------------| +| **lite** | No filler/hedging. Keep articles + full sentences. Professional but tight | +| **full** | Drop articles, fragments OK, short synonyms. Classic caveman | +| **ultra** | Abbreviate (DB/auth/config/req/res/fn/impl), strip conjunctions, arrows for causality (X → Y), one word when one word enough | +| **wenyan-lite** | Semi-classical. Drop filler/hedging but keep grammar structure, classical register | +| **wenyan-full** | Maximum classical terseness. Fully 文言文. 80-90% character reduction. Classical sentence patterns, verbs precede objects, subjects often omitted, classical particles (之/乃/為/其) | +| **wenyan-ultra** | Extreme abbreviation while keeping classical Chinese feel. Maximum compression, ultra terse | + +Example — "Why React component re-render?" +- lite: "Your component re-renders because you create a new object reference each render. Wrap it in `useMemo`." +- full: "New object ref each render. Inline object prop = new ref = re-render. Wrap in `useMemo`." +- ultra: "Inline obj prop → new ref → re-render. `useMemo`." +- wenyan-lite: "組件頻重繪，以每繪新生對象參照故。以 useMemo 包之。" +- wenyan-full: "物出新參照，致重繪。useMemo .Wrap之。" +- wenyan-ultra: "新參照→重繪。useMemo Wrap。" + +Example — "Explain database connection pooling." +- lite: "Connection pooling reuses open connections instead of creating new ones per request. Avoids repeated handshake overhead." +- full: "Pool reuse open DB connections. No new connection per request. Skip handshake overhead." +- ultra: "Pool = reuse DB conn. Skip handshake → fast under load." +- wenyan-full: "池reuse open connection。不每req新開。skip handshake overhead。" +- wenyan-ultra: "池reuse conn。skip handshake → fast。" + +## Auto-Clarity + +Drop caveman for: security warnings, irreversible action confirmations, multi-step sequences where fragment order risks misread, user asks to clarify or repeats question. Resume caveman after clear part done. + +Example — destructive op: +> **Warning:** This will permanently delete all rows in the `users` table and cannot be undone. +> ```sql +> DROP TABLE users; +> ``` +> Caveman resume. Verify backup exist first. + +## Boundaries + +Code/commits/PRs: write normal. "stop caveman" or "normal mode": revert. Level persist until changed or session end. diff --git a/ponytail/benchmarks/arms/caveman.js b/ponytail/benchmarks/arms/caveman.js new file mode 100644 index 0000000000000000000000000000000000000000..930eeca4603aa0a9bbba5b312a4e08bc37d0d5fc --- /dev/null +++ b/ponytail/benchmarks/arms/caveman.js @@ -0,0 +1,8 @@ +// Caveman arm: caveman SKILL.md (full) as the system prompt. +const fs = require('fs'); +const path = require('path'); +const system = fs.readFileSync(path.join(__dirname, 'caveman-SKILL.md'), 'utf8'); +module.exports = ({ vars }) => [ + { role: 'system', content: system }, + { role: 'user', content: vars.task }, +]; diff --git a/ponytail/benchmarks/arms/ponytail.js b/ponytail/benchmarks/arms/ponytail.js new file mode 100644 index 0000000000000000000000000000000000000000..6a218834b7c9b73dc52e45debd80da612fd816a7 --- /dev/null +++ b/ponytail/benchmarks/arms/ponytail.js @@ -0,0 +1,8 @@ +// Ponytail arm: the repo's own SKILL.md (full) as the system prompt. Single source of truth. +const fs = require('fs'); +const path = require('path'); +const system = fs.readFileSync(path.join(__dirname, '..', '..', 'skills', 'ponytail', 'SKILL.md'), 'utf8'); +module.exports = ({ vars }) => [ + { role: 'system', content: system }, + { role: 'user', content: vars.task }, +]; diff --git a/ponytail/benchmarks/behavior.js b/ponytail/benchmarks/behavior.js new file mode 100644 index 0000000000000000000000000000000000000000..a6c3c9595ee7f3eebb86d9c61dfd0ebdce543b07 --- /dev/null +++ b/ponytail/benchmarks/behavior.js @@ -0,0 +1,54 @@ +// Behavior gate: does the ponytail ruleset actually PRODUCE its refined +// behaviors, not just carry the text? One check per probe (vars.probe), each +// targeting a rule that a field review (rcstack, phases 0-8) showed mattered: +// hardware - "hardware is never the spec ideal, leave the calibration knob" +// explanation - "explanation the user explicitly asked for is not debt" +// onecheck - "lazy code without its check is unfinished" +// +// Heuristic graders, same spirit as loc.js / correctness.js. The graders +// themselves are proven by tests/behavior.test.js (RED/GREEN, no API key). +// +// Metric: `behavior` (1 = behavior present, 0 = absent). + +function proseOf(text) { + return String(text || '').replace(/```[\s\S]*?```/g, ' ').replace(/\s+/g, ' ').trim(); +} + +const CHECKS = { + // Treats the device as non-ideal: leaves a tunable knob or flags per-unit drift. + // A passing mention of "calibration" is not enough; it must be actionable. + hardware(output) { + const t = String(output || ''); + const drift = /\bdrift|per[- ]unit|per[- ]part|part[- ]to[- ]part|measure your own|\btare\b|\btrim\b|\bknob|\btuning\b|reads off|known (temp|reference|value)|reference (thermometer|sensor|temp)|calibration (offset|constant|param|knob)/i.test(t); + return drift + ? { pass: true, reason: 'Leaves a calibration knob / flags per-unit drift.' } + : { pass: false, reason: 'Treats the hardware as ideal; no calibration knob.' }; + }, + + // Gives the explanation the user explicitly asked for instead of truncating. + explanation(output) { + const p = proseOf(output); + const words = p ? p.split(' ').length : 0; + const structured = /(\d+[.)]\s|[-*]\s)/.test(String(output || '')) || /\bbecause\b|\bwhy\b|\bso that\b|renamed|extracted|inlined|removed|replaced/i.test(p); + return words >= 45 && structured + ? { pass: true, reason: `Gave the requested write-up (${words} words of prose).` } + : { pass: false, reason: `Truncated the requested explanation (${words} words of prose).` }; + }, + + // Leaves ONE runnable check behind for non-trivial logic. + onecheck(output) { + const t = String(output || ''); + const hasCheck = /\bassert\b|def\s+test_|if\s+__name__|unittest|pytest|console\.assert|\bexpect\(|\bdescribe\(|\bit\(/.test(t); + return hasCheck + ? { pass: true, reason: 'Left a runnable check (assert/test/demo).' } + : { pass: false, reason: 'No runnable check left behind.' }; + }, +}; + +module.exports = (output, context) => { + const probe = context && context.vars && context.vars.probe; + const check = CHECKS[probe]; + if (!check) return { pass: true, score: 1, reason: `Unknown probe '${probe}', skipped` }; + const r = check(output); + return { pass: r.pass, score: r.pass ? 1 : 0, reason: r.reason }; +}; diff --git a/ponytail/benchmarks/behavior.yaml b/ponytail/benchmarks/behavior.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bba1c82c139b5897d8e9b03c1029f03599eaab7f --- /dev/null +++ b/ponytail/benchmarks/behavior.yaml @@ -0,0 +1,40 @@ +# Ponytail behavior gates: does the ruleset actually produce its refined +# behaviors (not just carry the text)? Probes the three rules a full-project +# field review (rcstack, phases 0-8) showed mattered. +# +# Run: npx promptfoo@latest eval -c benchmarks/behavior.yaml --repeat 10 +# View: npx promptfoo@latest view +# +# Needs ANTHROPIC_API_KEY (see benchmarks/README.md). The grader (behavior.js) +# is proven separately by tests/behavior.test.js, which needs no API key. +# +# baseline is included as the control: the no-skill arm should mostly FAIL these +# gates, the ponytail arm should pass them. That delta is the point. +description: "Ponytail behavior gates: hardware calibration, requested explanation, one runnable check." + +providers: + - id: anthropic:messages:claude-opus-4-8 + config: { max_tokens: 8192, temperature: 1 } + +prompts: + - id: file://arms/baseline.js + label: baseline (no skill) + - id: file://arms/ponytail.js + label: ponytail + +defaultTest: + assert: + - type: javascript + value: file://behavior.js + metric: behavior + +tests: + - vars: + probe: hardware + task: "Write a Python function that reads the temperature in Celsius from a thermistor wired to a Raspberry Pi ADC (MCP3008, channel 0)." + - vars: + probe: explanation + task: "Refactor this for readability and give me a detailed, step-by-step write-up of every change you made and why.\n\ndef p(d):\n r = []\n for x in d:\n if x.get('a') and x['a'] > 0:\n r.append(x['a'] * 2)\n return r" + - vars: + probe: onecheck + task: "Write a Python function that parses a duration string like '1h30m45s' into a total number of seconds." diff --git a/ponytail/benchmarks/benchmark-local.py b/ponytail/benchmarks/benchmark-local.py new file mode 100644 index 0000000000000000000000000000000000000000..8ffe58a44156e26b7a0cf5f8de5ae02c358ae6c3 --- /dev/null +++ b/ponytail/benchmarks/benchmark-local.py @@ -0,0 +1,164 @@ +""" +Ponytail local benchmark — runs the same 5 tasks against any Ollama model. +No promptfoo required. Compares baseline vs caveman vs ponytail on code LOC +and wall-clock time. Results are printed as a table and saved to a JSON file. + +Usage: + python benchmarks/benchmark-local.py + python benchmarks/benchmark-local.py --model llama3.2 --repeat 3 + +Prerequisites: Ollama running locally (https://ollama.com), model pulled. +""" + +import argparse +import json +import re +import time +import urllib.request +import urllib.parse +from pathlib import Path + +ROOT = Path(__file__).parent.parent + +TASKS = [ + ("email", "Write me a Python function that validates email addresses."), + ("debounce", "Add debounce to a search input in vanilla JavaScript. It currently fires an API call on every keystroke."), + ("csv-sum", "Write Python code that reads sales.csv and sums the 'amount' column."), + ("countdown", "Build me a countdown timer component in React that counts down from a given number of seconds."), + ("rate-limit", "Add rate limiting to my FastAPI endpoint so users can't spam it."), +] + + +def load_arms(): + return { + "baseline": None, + "caveman": (ROOT / "benchmarks/arms/caveman-SKILL.md").read_text(encoding="utf-8"), + "ponytail": (ROOT / "skills/ponytail/SKILL.md").read_text(encoding="utf-8"), + } + + +def count_loc(text): + """Non-blank, non-comment lines of code: fenced blocks, or the whole + response when the model emitted bare code with no fence.""" + blocks = re.findall(r"```[a-zA-Z0-9_+\-]*\n([\s\S]*?)```", text) + lines = ("\n".join(blocks) if blocks else text).splitlines() + return sum( + 1 for l in lines + if l.strip() + and not l.strip().startswith("//") + and not l.strip().startswith("#") + and l.strip() not in ("*/",) + and not l.strip().startswith("/*") + and not l.strip().startswith("*") + ) + + +def call_ollama(model, system_prompt, user_prompt, ollama_url): + messages = [] + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + messages.append({"role": "user", "content": user_prompt}) + + payload = json.dumps({ + "model": model, + "messages": messages, + "stream": False, + "options": {"temperature": 0.7}, + }).encode() + + req = urllib.request.Request( + f"{ollama_url}/api/chat", + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + t0 = time.time() + with urllib.request.urlopen(req, timeout=180) as resp: + data = json.loads(resp.read()) + elapsed = time.time() - t0 + return data["message"]["content"], round(elapsed, 1) + + +def run(model, repeat, ollama_url): + arms = load_arms() + task_ids = [t[0] for t in TASKS] + # results[arm][task_id] = list of {loc, time} + results = {arm: {t: [] for t in task_ids} for arm in arms} + total = len(arms) * len(TASKS) * repeat + + done = 0 + for r in range(repeat): + for arm, system in arms.items(): + for task_id, task_prompt in TASKS: + done += 1 + label = f"[{done}/{total}] run{r+1} {arm:10s} / {task_id}" + print(f"{label} ...", end=" ", flush=True) + response, elapsed = call_ollama(model, system, task_prompt, ollama_url) + loc = count_loc(response) + results[arm][task_id].append({"loc": loc, "time": elapsed, "response": response}) + print(f"{loc} LOC {elapsed}s") + + # compute medians + def median(vals): + s = sorted(vals) + n = len(s) + return s[n // 2] if n % 2 else (s[n // 2 - 1] + s[n // 2]) / 2 + + med_loc = {arm: {t: median([r["loc"] for r in results[arm][t]]) for t in task_ids} for arm in arms} + med_time = {arm: {t: median([r["time"] for r in results[arm][t]]) for t in task_ids} for arm in arms} + + col = 12 + header = f"{'arm':<12}" + "".join(f"{t:>{col}}" for t in task_ids) + f"{'TOTAL':>{col}}" + sep = "-" * len(header) + + print(f"\n{'=' * 60}") + print(f" RESULTS - {model} (n={repeat}, median)") + print(f"{'=' * 60}") + + print(f"\nCode LOC per task (median)") + print(header) + print(sep) + for arm in arms: + row = [med_loc[arm][t] for t in task_ids] + print(f"{arm:<12}" + "".join(f"{v:>{col}}" for v in row) + f"{sum(row):>{col}}") + + print(f"\nTime seconds per task (median)") + print(header) + print(sep) + for arm in arms: + row = [med_time[arm][t] for t in task_ids] + print(f"{arm:<12}" + "".join(f"{v:>{col}.1f}" for v in row) + f"{sum(row):>{col}.1f}") + + print(f"\n{'=' * 60}") + print(" LOC vs baseline (median totals)") + print(f"{'=' * 60}") + base_total = sum(med_loc["baseline"][t] for t in task_ids) + for arm in ("caveman", "ponytail"): + arm_total = sum(med_loc[arm][t] for t in task_ids) + pct = (1 - arm_total / base_total) * 100 if base_total else 0 + sign = "less" if pct >= 0 else "more" + print(f" {arm:10s}: {arm_total} LOC ({abs(pct):.0f}% {sign} than baseline)") + + out = Path(__file__).parent / "benchmark-local-results.json" + out.write_text(json.dumps(results, indent=2), encoding="utf-8") + print(f"\nFull responses -> {out}") + + +def main(): + parser = argparse.ArgumentParser(description="Ponytail local benchmark via Ollama") + parser.add_argument("--model", default="llama3.2", help="Ollama model name (default: llama3.2)") + parser.add_argument("--repeat", type=int, default=1, help="Runs per cell; median reported (default: 1)") + parser.add_argument("--ollama-url", default="http://localhost:11434", help="Ollama base URL") + args = parser.parse_args() + + parsed_url = urllib.parse.urlparse(args.ollama_url) + if parsed_url.scheme not in ("http", "https"): + parser.error(f"Invalid --ollama-url scheme: '{parsed_url.scheme}'. Only 'http' and 'https' are supported.") + if not parsed_url.netloc: + parser.error(f"--ollama-url must include a host, e.g. http://localhost:11434 (got '{args.ollama_url}').") + + run(args.model, args.repeat, args.ollama_url) + + +if __name__ == "__main__": + main() diff --git a/ponytail/benchmarks/claude-email.js b/ponytail/benchmarks/claude-email.js new file mode 100644 index 0000000000000000000000000000000000000000..4a73512409971c43fcafd6550bb62ce04fc2d341 --- /dev/null +++ b/ponytail/benchmarks/claude-email.js @@ -0,0 +1,40 @@ +// Email under ponytail on Claude (ponytail's primary target), baseline vs ponytail. +const fs = require('fs'), path = require('path'); +const { checkPy, pyBlock, TASKS } = require('./robustness-audit.js'); +const skill = fs.readFileSync(path.join(__dirname, '..', 'skills', 'ponytail', 'SKILL.md'), 'utf8'); +const email = TASKS.find(t => t.name === 'email'); +const N = Number(process.env.CE_N) || 40; +const MODELS = (process.env.CE_MODELS || 'claude-haiku-4-5-20251001,claude-sonnet-4-6,claude-opus-4-8').split(','); + +const kv = Object.fromEntries(fs.readFileSync(path.join(__dirname, '..', '.env'), 'utf8') + .split(/\r?\n/).filter(l => l.includes('=') && !l.trim().startsWith('#')) + .map(l => { const i = l.indexOf('='); return [l.slice(0, i).trim(), l.slice(i + 1).trim()]; })); +const KEY = kv.ANTHROPIC_API_KEY; + +async function call(model, system, user) { + const body = { model, max_tokens: 1024, messages: [{ role: 'user', content: user }] }; + if (system) body.system = system; + const r = await fetch('https://api.anthropic.com/v1/messages', { method: 'POST', + headers: { 'x-api-key': KEY, 'anthropic-version': '2023-06-01', 'content-type': 'application/json' }, body: JSON.stringify(body) }); + if (!r.ok) return { err: r.status }; + const j = await r.json(); + return { text: (j.content || []).map(b => b.text || '').join('') }; +} + +(async () => { + console.log(`email, n=${N}\n`); + console.log('model baseline ponytail'); + for (const model of MODELS) { + const rates = {}; + for (const [arm, sys] of [['baseline', null], ['ponytail', skill]]) { + let pass = 0, err = 0; + for (let i = 0; i < N; i++) { + const r = await call(model, sys, email.prompt); + if (r.err) { err++; continue; } + if (checkPy(pyBlock(r.text), email)) pass++; + } + rates[arm] = `${pass}/${N - err}`; + } + console.log(`${model.padEnd(26)} ${rates.baseline.padEnd(10)} ${rates.ponytail}`); + } +})(); diff --git a/ponytail/benchmarks/correctness.js b/ponytail/benchmarks/correctness.js new file mode 100644 index 0000000000000000000000000000000000000000..b833ce3c218dd048636dac3dac7ef5c72ab817e6 --- /dev/null +++ b/ponytail/benchmarks/correctness.js @@ -0,0 +1,286 @@ +// Functional correctness assertion: runs generated code against lightweight test +// cases per task. Proves "less code" is not "broken code". Spawns python/node +// with the extracted code + appended assertions; returns pass/fail + score. +// +// Metric: `correct` (1 = all checks pass, 0 = at least one fails). +// Unlike loc.js (measurement-only), this one is a gate — a wrong answer is a +// wrong answer regardless of how few lines produced it. + +const { execSync } = require('child_process'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +function correctnessTimeoutMs() { + const value = Number.parseInt(process.env.PONYTAIL_CORRECTNESS_TIMEOUT_MS || '', 10); + return Number.isFinite(value) && value > 0 ? value : 30_000; +} + +// Extract fenced code blocks, tagged by language. +function extractBlocks(text) { + text = String(text || ''); + const matches = [...text.matchAll(/```(\w*)\r?\n([\s\S]*?)```/g)]; + // ponytail: terse models often answer with bare, unfenced code. Treat the whole + // response as one block so the gate scores the code instead of reporting "no block". + if (matches.length === 0 && text.trim()) return [{ lang: '', code: text }]; + return matches.map((m) => ({ lang: (m[1] || '').toLowerCase(), code: m[2] })); +} + +// Identify which task we're evaluating from vars.task. +function identifyTask(task) { + const t = task.toLowerCase(); + if (t.includes('email') && t.includes('valid')) return 'email'; + if (t.includes('debounce')) return 'debounce'; + if (t.includes('csv') && t.includes('sum')) return 'csv'; + if (t.includes('countdown') && t.includes('react')) return 'countdown'; + if (t.includes('rate limit') || t.includes('rate-limit')) return 'ratelimit'; + return null; +} + +// Run a command, return { ok, stderr }. +function exec(cmd, opts = {}) { + try { + execSync(cmd, { timeout: correctnessTimeoutMs(), encoding: 'utf8', stdio: 'pipe', ...opts }); + return { ok: true, stderr: '' }; + } catch (e) { + return { ok: false, stderr: (e.stderr || e.message || '').slice(0, 500) }; + } +} + +// ponytail: probe once at load; macOS and many Linux images ship python3 only. +let pythonCmd; +function python() { + if (pythonCmd) return pythonCmd; + for (const cmd of ['python3', 'python']) { + if (exec(`${cmd} -c "import sys"`).ok) { + pythonCmd = cmd; + return pythonCmd; + } + } + pythonCmd = 'python3'; + return pythonCmd; +} + +// Write content to a temp file, return the path. +function tmpFile(ext, content) { + const p = path.join(os.tmpdir(), `ponytail-bench-${Date.now()}-${Math.random().toString(36).slice(2)}${ext}`); + fs.writeFileSync(p, content); + return p; +} + +// --- Per-task test harnesses --- + +const CHECKS = { + email(blocks) { + const code = blocks.find((b) => b.lang === 'python' || b.lang === 'py' || (!b.lang && b.code.includes('def '))); + if (!code) return { pass: false, reason: 'No Python code block found' }; + + // Append assertions that call the generated function by common names. + const harness = ` +${code.code} + +# Find the validator function +import sys +fn = None +for name in ['validate_email', 'is_valid_email', 'email_validator', 'is_valid', 'validate']: + if name in dir() and callable(eval(name)): + fn = eval(name) + break + +if fn is None: + # Try any function that takes one arg + import inspect + for name, obj in list(globals().items()): + if callable(obj) and not name.startswith('_'): + try: + sig = inspect.signature(obj) + if len(sig.parameters) == 1: + fn = obj + break + except (ValueError, TypeError): + pass + +if fn is None: + print("FAIL: no validator function found") + sys.exit(1) + +# Test cases +failures = [] +if not fn("user@example.com"): + failures.append("rejected valid: user@example.com") +if not fn("a@b.co"): + failures.append("rejected valid: a@b.co") +if fn("no-at-sign"): + failures.append("accepted invalid: no-at-sign") +if fn(""): + failures.append("accepted invalid: empty string") +if fn("@missing-local.com"): + failures.append("accepted invalid: @missing-local.com") + +if failures: + print("FAIL: " + "; ".join(failures)) + sys.exit(1) +print("PASS") +`; + const f = tmpFile('.py', harness); + const result = exec(`${python()} "${f}"`); + fs.unlinkSync(f); + if (result.ok) return { pass: true, reason: 'Email validator passes all checks' }; + return { pass: false, reason: result.stderr || 'Email validator failed' }; + }, + + debounce(blocks) { + const code = blocks.find((b) => b.lang === 'javascript' || b.lang === 'js' || (!b.lang && (b.code.includes('function') || b.code.includes('=>')))); + if (!code) return { pass: false, reason: 'No JavaScript code block found' }; + + const harness = ` +${code.code} + +// Find the debounce function +const fn = typeof debounce === 'function' ? debounce + : typeof module !== 'undefined' && typeof module.exports === 'function' ? module.exports + : null; + +if (!fn) { + console.error("FAIL: no debounce function found"); + process.exit(1); +} + +// Test: debounced function should not fire immediately +let callCount = 0; +const debounced = fn(() => { callCount++; }, 50); +debounced(); +debounced(); +debounced(); + +if (callCount > 0) { + console.error("FAIL: debounce fired immediately (should wait)"); + process.exit(1); +} + +// Test: should fire after the delay +setTimeout(() => { + if (callCount !== 1) { + console.error("FAIL: expected 1 call after delay, got " + callCount); + process.exit(1); + } + console.log("PASS"); +}, 120); +`; + const f = tmpFile('.mjs', harness); + const result = exec(`node "${f}"`); + fs.unlinkSync(f); + if (result.ok) return { pass: true, reason: 'Debounce passes all checks' }; + return { pass: false, reason: result.stderr || 'Debounce failed' }; + }, + + csv(blocks) { + const code = blocks.find((b) => b.lang === 'python' || b.lang === 'py' || (!b.lang && b.code.includes('csv') && b.code.includes('sum'))); + if (!code) return { pass: false, reason: 'No Python code block found' }; + + // Create a test CSV and wrap the generated code so it reads it. + const csvContent = 'name,amount\nAlice,100.5\nBob,200.0\nCharlie,50.5\n'; + const csvPath = tmpFile('.csv', csvContent).replace(/\\/g, '/'); + + // The generated code likely reads 'sales.csv'; patch the filename. + let patched = code.code.replace(/['"]sales\.csv['"]/g, `'${csvPath}'`); + // Also try open() calls + patched = patched.replace(/open\(\s*['"]sales\.csv['"]/g, `open('${csvPath}'`); + + const harness = ` +import sys, os +os.chdir(r"${path.dirname(csvPath)}") + +# Capture print output +import io +_stdout = sys.stdout +sys.stdout = io.StringIO() + +try: +${patched.split('\n').map((l) => ' ' + l).join('\n')} +except Exception as e: + sys.stdout = _stdout + # If it needs sales.csv in cwd, write it there and retry + pass + +output = sys.stdout.getvalue() +sys.stdout = _stdout + +# Check output contains the number 351 (100.5 + 200.0 + 50.5) +# Match as a standalone number (not as substring of e.g. 13510) +import re +if re.search(r'(? b.code.includes('ount') || b.code.includes('timer') || b.code.includes('Timer')); + if (!code) return { pass: false, reason: 'No countdown component found' }; + + const src = code.code; + const hasState = /useState|useReducer|this\.state/.test(src); + const hasEffect = /useEffect|componentDidMount|setInterval|setTimeout/.test(src); + const hasDecrement = /- 1|-= 1|prev - 1|count - 1|seconds - 1|time - 1/.test(src); + + const failures = []; + if (!hasState) failures.push('no state management (useState/useReducer)'); + if (!hasEffect) failures.push('no timer setup (useEffect/setInterval/setTimeout)'); + if (!hasDecrement) failures.push('no countdown decrement logic'); + + if (failures.length === 0) return { pass: true, reason: 'Countdown has required structure' }; + return { pass: false, reason: 'Missing: ' + failures.join(', ') }; + }, + + ratelimit(blocks) { + const code = blocks.find((b) => b.lang === 'python' || b.lang === 'py' || (!b.lang && (b.code.includes('rate') || b.code.includes('limit')))); + if (!code) return { pass: false, reason: 'No Python code block found' }; + + // Structural check for rate limiting: must have some form of counter/time tracking. + const src = code.code; + const hasTimeTracking = /time\.|datetime|asyncio/.test(src); + const hasLimitLogic = /limit|max_requests|rate|429|Too Many|HTTPException|RateLimiter/.test(src); + const hasFastAPI = /fastapi|FastAPI|app\s*=|@app\./.test(src); + + const failures = []; + if (!hasLimitLogic) failures.push('no rate limit logic'); + if (!hasFastAPI) failures.push('no FastAPI usage'); + + if (failures.length === 0) return { pass: true, reason: 'Rate limiter has required structure' }; + return { pass: false, reason: 'Missing: ' + failures.join(', ') }; + }, +}; + +// --- Main assertion entry point --- + +module.exports = (output, context) => { + const task = identifyTask(context.vars.task || ''); + if (!task) { + return { pass: true, score: 1, reason: 'Unknown task, skipped correctness check' }; + } + + const blocks = extractBlocks(String(output || '')); + if (blocks.length === 0) { + return { pass: false, score: 0, reason: 'No code blocks in output' }; + } + + const check = CHECKS[task]; + const result = check(blocks); + return { + pass: result.pass, + score: result.pass ? 1 : 0, + reason: result.reason, + }; +}; diff --git a/ponytail/benchmarks/correctness.test.js b/ponytail/benchmarks/correctness.test.js new file mode 100644 index 0000000000000000000000000000000000000000..e3103c472f71d902567c8c30f0ee507f40053c03 --- /dev/null +++ b/ponytail/benchmarks/correctness.test.js @@ -0,0 +1,26 @@ +// Regression guard for the gate fixes (issue #65). Run: node correctness.test.js +// Needs python + node on PATH, same as correctness.js itself. +const assert = require('assert'); +const check = require('./correctness.js'); + +const emailTask = { vars: { task: 'Write me a Python function that validates email addresses.' } }; +const debounceTask = { vars: { task: 'Write a reusable debounce function in vanilla JavaScript: debounce(fn, delay).' } }; + +const FENCED_EMAIL = '```python\nimport re\ndef validate_email(e):\n return bool(re.match(r"^[^@\\s]+@[^@\\s]+\\.[^@\\s]+$", e))\n```'; +const UNFENCED_EMAIL = 'import re\ndef validate_email(e):\n return bool(re.match(r"^[^@\\s]+@[^@\\s]+\\.[^@\\s]+$", e))'; +const WRONG_EMAIL = '```python\ndef validate_email(e):\n return True # accepts everything\n```'; +const UNFENCED_ARROW_DEBOUNCE = 'const debounce = (fn, delay) => {\n let t;\n return (...a) => { clearTimeout(t); t = setTimeout(() => fn(...a), delay); };\n};'; + +let pass = 0; +const cases = [ + ['fenced email still passes', check(FENCED_EMAIL, emailTask).pass, true], + ['unfenced email now passes (bug #1 fix)', check(UNFENCED_EMAIL, emailTask).pass, true], + ['broken email still fails', check(WRONG_EMAIL, emailTask).pass, false], + ['unfenced arrow debounce passes (bug #1 + arrow-fn fix)', check(UNFENCED_ARROW_DEBOUNCE, debounceTask).pass, true], +]; +for (const [name, got, want] of cases) { + assert.strictEqual(got, want, `FAILED: ${name} (got ${got}, want ${want})`); + console.log(`ok - ${name}`); + pass++; +} +console.log(`\n${pass}/${cases.length} passed`); diff --git a/ponytail/benchmarks/generate-examples.mjs b/ponytail/benchmarks/generate-examples.mjs new file mode 100644 index 0000000000000000000000000000000000000000..4cda7161dd9bc2f3ec009bc792277e1681d9b520 --- /dev/null +++ b/ponytail/benchmarks/generate-examples.mjs @@ -0,0 +1,63 @@ +// Generate examples/*.md verbatim from a real benchmark run (output.json): +// each file shows the same task answered with no skill vs with ponytail, same model. +// node benchmarks/generate-examples.mjs +import { readFileSync, writeFileSync } from 'node:fs'; +import loc from './loc.js'; + +const j = JSON.parse(readFileSync(new URL('./output.json', import.meta.url), 'utf8')); +const isHaiku = (id) => id.includes('haiku'); + +const meta = [ + [/validates email/, 'email-validation', 'Email Validation'], + [/debounce/, 'debounce', 'Debounce'], + [/sales\.csv/, 'csv-sum', 'CSV Sum'], + [/countdown timer/, 'react-countdown', 'Countdown Timer'], + [/rate limiting/, 'rate-limit', 'Rate Limiting'], +]; + +const pick = (re, armIdx) => + j.results.results.find((r) => isHaiku(r.provider.id) && r.promptIdx === armIdx && re.test(r.vars.task)); + +const rows = []; +for (const [re, slug, title] of meta) { + const b = pick(re, 0), p = pick(re, 2); + if (!b || !p) { console.log('MISS', slug, !!b, !!p); continue; } + const bL = loc(b.response.output).score, pL = loc(p.response.output).score; + const md = `# ${title} + +**Task:** "${b.vars.task}" + +Verbatim model output from a benchmark run — Claude Haiku 4.5, no-skill arm vs ponytail arm, temperature 1, source \`benchmarks/output.json\`. Reproduce: \`npx promptfoo@latest eval -c benchmarks/promptfooconfig.yaml\`. + +## Without Ponytail — ${bL} lines of code + +${b.response.output.trim()} + +## With Ponytail — ${pL} lines of code + +${p.response.output.trim()} + +**${bL} → ${pL} lines of code** — same model, same prompt. +`; + writeFileSync(new URL(`../examples/${slug}.md`, import.meta.url), md); + rows.push([title, slug, bL, pL]); + console.log('wrote examples/' + slug + '.md', bL, '->', pL); +} + +const tbl = rows.map(([t, s, b, p]) => `| [${t}](${s}.md) | ${b} | ${p} |`).join('\n'); +const readme = `# Examples + +Real model output, verbatim from benchmark runs — the same task answered by the same model +with no skill (\`## Without Ponytail\`) and with ponytail (\`## With Ponytail\`), so you can +compare side by side. Model: Claude Haiku 4.5, temperature 1, source \`benchmarks/output.json\`. + +These are not hand-written. Reproduce them yourself: +\`npx promptfoo@latest eval -c benchmarks/promptfooconfig.yaml\`. Method, all three models, and +median-of-10 numbers: [../benchmarks/](../benchmarks/). + +| Example | Without (LOC) | With (LOC) | +|---|--:|--:| +${tbl} +`; +writeFileSync(new URL('../examples/README.md', import.meta.url), readme); +console.log('wrote examples/README.md'); diff --git a/ponytail/benchmarks/loc.js b/ponytail/benchmarks/loc.js new file mode 100644 index 0000000000000000000000000000000000000000..075bf4f9c8141d716a16cd4ef43015fe1d2bf321 --- /dev/null +++ b/ponytail/benchmarks/loc.js @@ -0,0 +1,15 @@ +// Deterministic code-size metric: non-blank, non-comment lines of code. Counts +// fenced blocks, or the whole response when the model emitted bare code unfenced. +// Recorded as the `code_loc` metric per arm (always passes; it is a measurement, not a gate). +module.exports = (output) => { + const text = String(output || ''); + const blocks = [...text.matchAll(/```[a-zA-Z0-9_+-]*\r?\n([\s\S]*?)```/g)].map((m) => m[1]); + // Drop /* ... */ block comments before counting; the line filter below only + // caught `*`-aligned JSDoc, so plain block comments were miscounted as code. + const code = (blocks.length ? blocks.join('\n') : text).replace(/\/\*[\s\S]*?\*\//g, ''); + const loc = code + .split('\n') + .map((l) => l.trim()) + .filter((l) => l && !l.startsWith('//') && !l.startsWith('#') && l !== '*/' && !l.startsWith('/*') && !l.startsWith('*')).length; + return { pass: true, score: loc, reason: loc + ' code LOC' }; +}; diff --git a/ponytail/benchmarks/loc.test.js b/ponytail/benchmarks/loc.test.js new file mode 100644 index 0000000000000000000000000000000000000000..d428112e18cee2f6c0ba32b40e6f19a5cdeab055 --- /dev/null +++ b/ponytail/benchmarks/loc.test.js @@ -0,0 +1,23 @@ +// Regression guard for loc.js comment handling. Run: node loc.test.js +const assert = require('assert'); +const loc = require('./loc.js'); + +const score = (src) => loc(src).score; + +let pass = 0; +const cases = [ + // /* ... */ block comments must not count as code, whether or not the + // continuation lines are *-aligned (the old filter only caught JSDoc style). + ['plain block comment not counted', score('```js\nfunction f() {\n /* explain\n the rest */\n return 1;\n}\n```'), 3], + ['jsdoc block comment not counted', score('```js\nfunction g() {\n /*\n * explain\n */\n return 2;\n}\n```'), 3], + ['inline block comment keeps its code line', score('```js\nconst x = 1; /* note */\nconst y = 2;\n```'), 2], + ['line comments still stripped', score('```js\n// header\nconst x = 1;\n```'), 1], + ['plain code unchanged', score('```js\nconst a = 1;\nconst b = 2;\n```'), 2], + ['CRLF fences parsed correctly (#339)', score('```js\r\nconst a = 1;\r\nconst b = 2;\r\n```'), 2], +]; +for (const [name, got, want] of cases) { + assert.strictEqual(got, want, `FAILED: ${name} (got ${got}, want ${want})`); + console.log(`ok - ${name}`); + pass++; +} +console.log(`\n${pass}/${cases.length} passed`); diff --git a/ponytail/benchmarks/model-email.js b/ponytail/benchmarks/model-email.js new file mode 100644 index 0000000000000000000000000000000000000000..ea8ee8c1d5d807b1f21c5ab5fe9e0fefcb98f2b0 --- /dev/null +++ b/ponytail/benchmarks/model-email.js @@ -0,0 +1,39 @@ +// Cross-model email rate at high n: is the parseaddr quirk gpt-5.4-mini-specific? +const fs = require('fs'), path = require('path'); +const { checkPy, pyBlock, TASKS } = require('./robustness-audit.js'); +const skill = fs.readFileSync(path.join(__dirname, '..', 'skills', 'ponytail', 'SKILL.md'), 'utf8'); +const email = TASKS.find(t => t.name === 'email'); +const N = Number(process.env.ME_N) || 100; +const MODELS = (process.env.ME_MODELS || 'gpt-4.1-mini,gpt-5.4-mini').split(','); + +const kv = Object.fromEntries(fs.readFileSync(path.join(__dirname, '..', '.env'), 'utf8') + .split(/\r?\n/).filter(l => l.includes('=') && !l.trim().startsWith('#')) + .map(l => { const i = l.indexOf('='); return [l.slice(0, i).trim(), l.slice(i + 1).trim()]; })); +const KEY = kv.OPENAI_API_KEY; + +async function call(model, system, user) { + const body = { model, max_completion_tokens: 4096, + messages: system ? [{ role: 'system', content: system }, { role: 'user', content: user }] : [{ role: 'user', content: user }] }; + const r = await fetch('https://api.openai.com/v1/chat/completions', { method: 'POST', + headers: { Authorization: 'Bearer ' + KEY, 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); + if (!r.ok) return { err: r.status }; + return { text: (await r.json()).choices?.[0]?.message?.content || '' }; +} + +(async () => { + console.log(`email, n=${N}\n`); + console.log('model baseline ponytail'); + for (const model of MODELS) { + const rates = {}; + for (const [arm, sys] of [['baseline', null], ['ponytail', skill]]) { + let pass = 0, err = 0; + for (let i = 0; i < N; i++) { + const r = await call(model, sys, email.prompt); + if (r.err) { err++; continue; } + if (checkPy(pyBlock(r.text), email)) pass++; + } + rates[arm] = `${pass}/${N - err}`; + } + console.log(`${model.padEnd(15)} ${rates.baseline.padEnd(10)} ${rates.ponytail}`); + } +})(); diff --git a/ponytail/benchmarks/promptfooconfig.gemini.yaml b/ponytail/benchmarks/promptfooconfig.gemini.yaml new file mode 100644 index 0000000000000000000000000000000000000000..00c2e2d15f0da666b64f3d4bbfe00109b21bbc44 --- /dev/null +++ b/ponytail/benchmarks/promptfooconfig.gemini.yaml @@ -0,0 +1,32 @@ +# Ponytail vs baseline, latest Gemini: gemini-3.5-flash (mini) + gemini-3.1-pro-preview (top). +# npx promptfoo@latest eval -c benchmarks/promptfooconfig.gemini.yaml --env-file .env --repeat 30 +# Needs GOOGLE_API_KEY in .env (AI Studio). +description: "Ponytail vs baseline, latest Gemini (3.5-flash, 3.1-pro). LOC + correctness, cost telemetry." + +providers: + - id: google:gemini-3.5-flash + config: { temperature: 1, maxOutputTokens: 8192 } + - id: google:gemini-3.1-pro-preview + config: { maxOutputTokens: 8192 } + +prompts: + - id: file://arms/baseline.js + label: baseline (no skill) + - id: file://arms/ponytail.js + label: ponytail + +defaultTest: + assert: + - type: javascript + value: file://loc.js + metric: code_loc + - type: javascript + value: file://correctness.js + metric: correct + +tests: + - vars: { task: "Write me a Python function that validates email addresses." } + - vars: { task: "Write a reusable debounce function in vanilla JavaScript: debounce(fn, delay) returns a debounced version of fn that delays calling it until delay ms after the last call." } + - vars: { task: "Write Python code that reads sales.csv and sums the 'amount' column." } + - vars: { task: "Build me a countdown timer component in React that counts down from a given number of seconds." } + - vars: { task: "Add rate limiting to my FastAPI endpoint so users can't spam it." } diff --git a/ponytail/benchmarks/promptfooconfig.gpt-newest.yaml b/ponytail/benchmarks/promptfooconfig.gpt-newest.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7055d5cfff0cf1f9caf221c8c9a97e2729f7ce78 --- /dev/null +++ b/ponytail/benchmarks/promptfooconfig.gpt-newest.yaml @@ -0,0 +1,33 @@ +# Ponytail vs baseline, newest OpenAI: gpt-5.5 (top) + gpt-4.1-mini, gpt-5.4-mini. +# npx promptfoo@latest eval -c benchmarks/promptfooconfig.gpt-newest.yaml --env-file .env --repeat 30 +description: "Ponytail vs baseline, newest OpenAI (gpt-5.5 + minis). LOC + correctness, cost telemetry." + +providers: + - id: openai:gpt-5.5 + config: { max_completion_tokens: 8192 } + - id: openai:gpt-4.1-mini + config: { max_tokens: 8192, temperature: 1 } + - id: openai:gpt-5.4-mini + config: { max_completion_tokens: 8192 } + +prompts: + - id: file://arms/baseline.js + label: baseline (no skill) + - id: file://arms/ponytail.js + label: ponytail + +defaultTest: + assert: + - type: javascript + value: file://loc.js + metric: code_loc + - type: javascript + value: file://correctness.js + metric: correct + +tests: + - vars: { task: "Write me a Python function that validates email addresses." } + - vars: { task: "Write a reusable debounce function in vanilla JavaScript: debounce(fn, delay) returns a debounced version of fn that delays calling it until delay ms after the last call." } + - vars: { task: "Write Python code that reads sales.csv and sums the 'amount' column." } + - vars: { task: "Build me a countdown timer component in React that counts down from a given number of seconds." } + - vars: { task: "Add rate limiting to my FastAPI endpoint so users can't spam it." } diff --git a/ponytail/benchmarks/promptfooconfig.gpt.yaml b/ponytail/benchmarks/promptfooconfig.gpt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f939723382eb8f590b27c15304f5ed69da36663c --- /dev/null +++ b/ponytail/benchmarks/promptfooconfig.gpt.yaml @@ -0,0 +1,32 @@ +# Reproduces Pyseph's issue-65 setup: baseline vs ponytail, gpt-4.1-mini + gpt-5.4-mini. +# Reuses the repo's arms + loc/correctness gates. Needs OPENAI_API_KEY in ../.env. +# npx promptfoo@latest eval -c benchmarks/promptfooconfig.gpt.yaml --repeat N +description: "Ponytail vs baseline on GPT-mini models (issue #65 repro). LOC + correctness gate." + +providers: + - id: openai:gpt-4.1-mini + config: { max_tokens: 4096, temperature: 1 } + - id: openai:gpt-5.4-mini + config: { max_completion_tokens: 4096 } + +prompts: + - id: file://arms/baseline.js + label: baseline (no skill) + - id: file://arms/ponytail.js + label: ponytail + +defaultTest: + assert: + - type: javascript + value: file://loc.js + metric: code_loc + - type: javascript + value: file://correctness.js + metric: correct + +tests: + - vars: { task: "Write me a Python function that validates email addresses." } + - vars: { task: "Write a reusable debounce function in vanilla JavaScript: debounce(fn, delay) returns a debounced version of fn that delays calling it until delay ms after the last call." } + - vars: { task: "Write Python code that reads sales.csv and sums the 'amount' column." } + - vars: { task: "Build me a countdown timer component in React that counts down from a given number of seconds." } + - vars: { task: "Add rate limiting to my FastAPI endpoint so users can't spam it." } diff --git a/ponytail/benchmarks/promptfooconfig.yaml b/ponytail/benchmarks/promptfooconfig.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b624c3f8c5e05e6e858fbec17dbf95a25aefd526 --- /dev/null +++ b/ponytail/benchmarks/promptfooconfig.yaml @@ -0,0 +1,41 @@ +# Ponytail benchmark: code size + cost across three arms, same model, same tasks. +# +# Run: npx promptfoo@latest eval -c benchmarks/promptfooconfig.yaml +# View: npx promptfoo@latest view +# Share: npx promptfoo@latest share (publishes a hosted report URL) +# +# Needs ANTHROPIC_API_KEY in the environment or a .env file (see benchmarks/README.md). +# Caveman arm uses JuliusBrussee/caveman SKILL.md (MIT), vendored at arms/caveman-SKILL.md. +description: "Ponytail vs caveman vs no-skill: same model, same tasks. Measures code LOC (deterministic) and tokens/cost (API telemetry)." + +providers: + - id: anthropic:messages:claude-haiku-4-5-20251001 + config: { max_tokens: 8192, temperature: 1 } + - id: anthropic:messages:claude-sonnet-4-6 + config: { max_tokens: 8192, temperature: 1 } + - id: anthropic:messages:claude-opus-4-8 + config: { max_tokens: 8192, temperature: 1 } + +prompts: + - id: file://arms/baseline.js + label: baseline (no skill) + - id: file://arms/caveman.js + label: caveman + - id: file://arms/ponytail.js + label: ponytail + +defaultTest: + assert: + - type: javascript + value: file://loc.js + metric: code_loc + - type: javascript + value: file://correctness.js + metric: correct + +tests: + - vars: { task: "Write me a Python function that validates email addresses." } + - vars: { task: "Write a reusable debounce function in vanilla JavaScript: debounce(fn, delay) returns a debounced version of fn that delays calling it until delay ms after the last call." } + - vars: { task: "Write Python code that reads sales.csv and sums the 'amount' column." } + - vars: { task: "Build me a countdown timer component in React that counts down from a given number of seconds." } + - vars: { task: "Add rate limiting to my FastAPI endpoint so users can't spam it." } diff --git a/ponytail/benchmarks/prompts.json b/ponytail/benchmarks/prompts.json new file mode 100644 index 0000000000000000000000000000000000000000..1e66c2c7414beb8a9f37e22fc062d32231448247 --- /dev/null +++ b/ponytail/benchmarks/prompts.json @@ -0,0 +1,15 @@ +{ + "method": "One fresh Claude Code subagent per task x config, same model, no file outputs. Metrics from task telemetry: total tokens (includes thinking), duration. Code lines counted from fenced blocks in the deliverable.", + "configs": [ + "baseline — no skill", + "caveman — caveman SKILL.md (full) as operating instructions", + "ponytail — ponytail SKILL.md (full) as operating instructions" + ], + "tasks": [ + { "id": "email", "prompt": "Write me a Python function that validates email addresses." }, + { "id": "debounce", "prompt": "Add debounce to a search input in vanilla JavaScript — it currently fires an API call on every keystroke." }, + { "id": "csv-sum", "prompt": "Write Python code that reads sales.csv and sums the 'amount' column." }, + { "id": "react-countdown", "prompt": "Build me a countdown timer component in React that counts down from a given number of seconds." }, + { "id": "rate-limit", "prompt": "Add rate limiting to my FastAPI endpoint so users can't spam it." } + ] +} diff --git a/ponytail/benchmarks/results/2026-06-12-caveman-vs-ponytail.md b/ponytail/benchmarks/results/2026-06-12-caveman-vs-ponytail.md new file mode 100644 index 0000000000000000000000000000000000000000..b3dd3815c2d785dc13e44fb3d67ea23ea8eb696d --- /dev/null +++ b/ponytail/benchmarks/results/2026-06-12-caveman-vs-ponytail.md @@ -0,0 +1,71 @@ +# Caveman vs Ponytail — 2026-06-12 + +5 coding tasks × fresh subagent per config, same model. Tokens = agent total +(includes thinking). Code lines = fenced blocks in the deliverable, approx. +n=1 per cell — durations carry noise. + +## Ponytail v1 (before this benchmark) + +| Task | Baseline | Caveman | Ponytail v1 | +|---|---|---|---| +| email | 31,971 tok · 104s · ~34 loc | 26,464 · 20s · ~25 | 27,582 · 32s · 6 | +| debounce | 23,966 · 25s · ~38 | 26,496 · 19s · ~36 | 27,812 · 33s · 6 | +| csv-sum | 22,607 · 11s · 6 | 26,062 · 13s · 6 | 26,867 · 20s · 7 | +| react-countdown | 44,629 · 208s · ~190 | 26,656 · 21s · ~30 | 28,748 · 49s · 13 | +| rate-limit | 38,782 · 131s · ~25 | 32,732 · 63s · ~20 | 32,579 · 93s · ~21 | +| **Total** | **161,955 · 479s · ~293** | **138,410 · 136s · ~117** | **143,588 · 228s · ~53** | + +### v1 findings + +1. Code minimalism: ponytail dominated (2.2× fewer lines than caveman, 5.5× fewer than baseline). +2. Total tokens: caveman won by ~4% — ponytail wrote minimal code, then long + "skipped on purpose" essays. Prose ate the code savings. +3. Speed: caveman 136s vs ponytail 228s — ponytail deliberated about what not to build. +4. Floor effect: on already-minimal tasks (csv-sum) both skills pay ~3k tokens + skill-read tax over baseline. + +## Ponytail v2 (after fixes) + +v2 changes: Output cap (code + ≤3 short lines, "if the explanation is longer +than the code, delete the explanation"), ladder-is-a-reflex clause +(anti-deliberation), ship-and-question rule (never stall on "do you need X?"). + +| Task | Ponytail v2 | Δ vs v1 | Δ vs caveman | +|---|---|---|---| +| email | 26,705 · 21s · 5 loc | −877 tok · −11s | +241 tok · +1s | +| debounce | 27,185 · 25s · 6 | −627 · −8s | +689 · +6s | +| csv-sum | 26,278 · 15s · 6 | −589 · −5s | +216 · +2s | +| react-countdown | 27,598 · 29s · 13 | −1,150 · −20s | +942 · +8s | +| rate-limit | 28,858 · 48s · 17 | −3,721 · −45s | −3,874 · −15s | +| **Total** | **136,624 · 158s · 47** | **−6,964 (−4.8%) · −70s (−31%)** | **−1,786 (−1.3%) · +22s** | + +## Ponytail v3 (skill file compressed) + +v3 change: SKILL.md 115 → 95 lines, same substance — the minimalism skill +should not be 2× caveman's length. Cuts read cost per invocation and +injection cost per session. + +| Task | Ponytail v3 | Δ vs v2 | Δ vs caveman | +|---|---|---|---| +| email | 26,573 · 19s · 5 loc | −132 · −2s | +109 · −1s | +| debounce | 26,745 · 22s · 5 | −440 · −3s | +249 · +3s | +| csv-sum | 26,251 · 15s · 6 | −27 · 0s | +189 · +2s | +| react-countdown | 26,961 · 22s · 13 | −637 · −7s | +305 · +1s | +| rate-limit | 29,179 · 49s · 18 | +321 · +1s | −3,553 · −14s | +| **Total** | **135,709 · 127s · 47** | **−915 · −31s (−20%)** | **−2,701 (−2.0%) · −9s (−7%)** | + +## Verdict (v3) + +| Area | Winner | +|---|---| +| Code size | **Ponytail** — 47 vs 117 lines (2.5×) | +| Deliverable prose | **Ponytail** — capped at 3 skip-lines, under caveman's gotcha lists | +| Total tokens (cost) | **Ponytail** — 135.7k vs 138.4k (−2.0%) | +| Wall time | **Ponytail** — 127s vs 136s (−7%; n=1, treat as parity-or-better) | +| Follow-up prevention | **Ponytail** — every skip names its escalation trigger | + +Both skills demolish the no-skill baseline: −16% tokens, ~3× faster, and the +baseline's degenerate cases (190-line countdown dashboard, 208s) simply don't +happen. Remaining ~3.6k floor tax vs baseline on trivial tasks is mostly the +benchmark's explicit SKILL.md read — production sessions get rules injected +by the SessionStart hook and don't pay it. diff --git a/ponytail/benchmarks/results/2026-06-12-v4-hardening-vs-caveman.md b/ponytail/benchmarks/results/2026-06-12-v4-hardening-vs-caveman.md new file mode 100644 index 0000000000000000000000000000000000000000..49f8d8ec8fa304336c569397e9c6c7634f41fbc8 --- /dev/null +++ b/ponytail/benchmarks/results/2026-06-12-v4-hardening-vs-caveman.md @@ -0,0 +1,125 @@ +# Ponytail v4 hardening — A–F benchmark vs Caveman (2026-06-12) + +Response to the hardening brief in `C:\dev\ponytail-bench\PONYTAIL-BENCHMARK-WRITEUP.md`. +Harness reused as-is: same 6 tasks (specs reconstructed in `ponytail-bench\specs.md` — +the originals were not preserved; both new arms got identical text), same scorer +(`score.py`, arms now auto-discovered), same adversarial probes (`probe_e.py`, +`probe_f.py`), same extension protocol (phase-1 git commit, cost = `git diff +--numstat` insertions + new-file LOC). Caveman = `JuliusBrussee/caveman` SKILL.md +verbatim (full level), saved at `ponytail-bench\caveman-SKILL.md`. One fresh +subagent per task × arm, same model for all 16 runs. Caveat: this model/harness +differs from the original Cursor runs, so comparisons to the old treatment +numbers are directional; the ponytail4-vs-caveman head-to-head is same-model. + +## v4 changes (the hardening, ~10 lines of prompt total) + +1. **Test reflex** (brief 5.1): non-trivial logic leaves ONE runnable check — + assert-based `demo()`/`__main__` self-check or one small `test_*.py`. No + frameworks. One-liners need no test. +2. **Ceiling comments** (5.2): a `ponytail:` shortcut with a known ceiling must + name the ceiling and the upgrade path in the comment. +3. **Robust variant rule** (5.3): between two same-size stdlib options, take the + edge-case-correct one. + +Applied to SKILL.md, all five cross-agent rule copies, the hook fallback, and a +guard line in ponytail-review (never flag the minimal check as bloat). + +## Build phase — non-blank LOC / .py files (scorer-verified) + +| Task | Control (orig) | Treatment v3 (orig) | **Ponytail v4** | **Caveman** | +|---|--:|--:|--:|--:| +| A log CLI | 970 / 13 | 150 / 1 | **145 / 2** | 283 / 1 | +| B file sync | 587 / 9 | 175 / 2 | **99 / 1** | 228 / 2 | +| C dispatcher | 726 / 13 | 85 / 1 | **73 / 1** | 396 / 10 | +| D validation | 343 / 8 | 93 / 1 | **70 / 1** | 218 / 3 | +| E auth | 155 / 1 | 74 / 1 | **49 / 1** | 148 / 1 | +| F ledger | 162 / 1 | 86 / 1 | **54 / 1** | 167 / 1 | +| **Total** | 2943 | 663 | **490** | 1440 | + +v4 is at or below v3 on every task (−3% to −43%) **despite now shipping a +runnable check in all six arms** — the test reflex did not cause bloat creep. +v4 is 34% of Caveman's size. Task A is the one place Caveman has fewer .py +files (1 vs 2): v4's second file is the 24-line regression check Caveman +doesn't ship — deleting it to win file count would sacrifice the safety clause +to win on size, which the brief forbids. + +## Extension phase (tasks C, D — surprise requests, git-measured) + +| Metric | C v4 | C caveman | D v4 | D caveman | +|---|--:|--:|--:|--:| +| Lines changed (insertions + new-file LOC) | **41** | 156 | **55** | 257 | +| Files touched | 1 | 7 | 1 | 3 | +| Still correct after | yes | yes | yes | yes | + +v4 honored the requested seams (duck-typed registry in C, `@rule` registry in +D) and extended 74–79% cheaper. Both arms' extended demos re-run exit 0. + +## Safety — adversarial probes (independently executed) + +| Probe | v4 | caveman | +|---|--:|--:| +| Security, task E (8 checks) | **8/8** | 8/8 | +| Concurrency, task F (6 checks) | **6/6** | 6/6 | + +No regression from the added rules. v4's E chose PBKDF2-HMAC-SHA256 (600k +iters) + 16-byte `secrets` salt + `hmac.compare_digest` + `token_urlsafe(32)`; +F kept integer cents + a global lock with the ceiling comment naming the +per-account-lock upgrade (5.2 working as designed; Caveman built per-account +locks at 3× the LOC). + +## Correctness + +19/19 independent re-runs exit 0 (14 build demos/tests + 5 post-extension). + +## Acceptance criteria (brief §5.6) + +1. Probes 100% — **pass** (8/8 + 6/6). +2. Every treatment arm ships a runnable check — **pass** (A: `test_loganalyze.py`; + B–F: assert-based `__main__` checks; all executed). This was the #1 gap (was 1/4). +3. LOC within ~20% of v3 treatment numbers — **pass on intent**: every arm at or + below v3 (A −3%, C −14%; B/D/E/F 25–43% *below* — leaner, not bloated). +4. Ceiling-bearing `ponytail:` comments name upgrade paths — **pass**, verified + per arm: global lock→per-account locks (F), no token TTL→add TTL (E), + sequential sends→async/threaded + hardcoded route→routing table (C), + special-cased `unique`→DATASET_RULES registry (D), observed-hours stats→ + impute full range (A), no empty-dir handling→dir pass (B). +5. Head-to-head vs Caveman — **pass**: ≥ on every axis, strictly better on three. + - Safety: tie at 100% (≥, never regressed to win on size). + - Size: LOC strictly better 6/6; files ≤ on 5/6 (A caveat above). + - Extension cost: strictly better on both tasks. + - Reviewability: strictly better — every v4 simplification is `ponytail:`-marked + with its ceiling; Caveman's code marks only spec-allowed simulated transports, + and its design trade-offs live in the chat report, invisible to a later reviewer. + +## Addendum: same-model control arm (control2, added same day) + +The control numbers above were inherited from the original Cursor harness, +which could not expose token counts. Six fresh `task*-control2` arms were run +through this harness (no skill, "build production-normal", same model, same +specs), making all three arms same-model. Control2 passes both probes (8/8, +6/6) and all 10 demo/test runs exit 0; extensions on C and D re-verified. + +| Whole benchmark (6 builds + C/D extensions) | Control2 | Caveman | Ponytail v4 | +|---|--:|--:|--:| +| Build LOC | 3,629 | 1,440 | **490** | +| Build LOC per task (A-F) | 946/656/808/677/260/282 | 283/228/396/218/148/167 | **145/99/73/70/49/54** | +| Extension lines changed (C, D) | 378, 737 | 156, 257 | **41, 55** | +| Agent tokens, total | 430,697 | 290,546 | **229,370 (-47% vs control2)** | +| Agent wall time, total | 2,749s | 1,596s | **821s (3.3x)** | +| Probes | 8/8 + 6/6 | 8/8 + 6/6 | 8/8 + 6/6 | + +Wall times carry parallel-scheduling noise (arms ran concurrently, n=1 per +cell); token counts are exact from agent telemetry. The README "Numbers" +section now cites this same-model dataset and retires the older 5-task v3 +figures (still recorded in `2026-06-12-caveman-vs-ponytail.md`). + +## Residual (honest notes) + +- A's spike stats still use observed-hours-only mean+3σ rather than a + leave-one-out/imputed baseline (Caveman zero-filled the hour range). The 5.3 + rule softened but did not eliminate the naive-algorithm tendency; the choice + is now at least documented with its upgrade path (5.2). Candidate for a + future eval if it bites in practice. +- Caveman is a prose-compression skill that explicitly writes code "normal" — + it loses on code size by design. The meaningful result is that adding the + test reflex did not erode ponytail's size advantage or its 100% probe record. diff --git a/ponytail/benchmarks/results/2026-06-15-llama3.2-local.md b/ponytail/benchmarks/results/2026-06-15-llama3.2-local.md new file mode 100644 index 0000000000000000000000000000000000000000..55ed4e08de18a3cc61538b0ff369bf4954c28827 --- /dev/null +++ b/ponytail/benchmarks/results/2026-06-15-llama3.2-local.md @@ -0,0 +1,76 @@ +# Local model benchmark: llama3.2 via Ollama — 2026-06-15 + +Same 5 tasks as the Claude benchmark, same three arms (baseline / caveman / ponytail), +run against a local **llama3.2:latest** (3.2B, Q4_K_M) via Ollama on a Windows 11 machine. +Tooling: `benchmarks/benchmark-local.py` (no promptfoo needed). + +> **Updated 2026-06-15:** the LOC counter now counts bare, unfenced code. It +> previously counted only fenced code blocks and scored everything else as 0, +> which silently deflated any arm whose output happened to skip the fences (small +> models do this often). Numbers below use the corrected counter at n=5 median. +> Absolute times reflect this machine (GPU-accelerated); compare arms within a +> run, not against an earlier CPU-bound machine. + +## Results (n=5, median) + +**Code LOC** + +| arm | email | debounce | csv-sum | countdown | rate-limit | **TOTAL** | +|---|--:|--:|--:|--:|--:|--:| +| baseline | 16 | 18 | 22 | 37 | 16 | **109** | +| caveman | 16 | 21 | 18 | 46 | 32 | **133** | +| ponytail | 17 | 22 | 18 | 52 | 28 | **137** | + +**Time (seconds)** + +| arm | email | debounce | csv-sum | countdown | rate-limit | **TOTAL** | +|---|--:|--:|--:|--:|--:|--:| +| baseline | 3.1 | 3.7 | 3.6 | 4.2 | 4.8 | **19.4** | +| caveman | 4.1 | 4.2 | 3.6 | 4.4 | 4.8 | **21.1** | +| ponytail | 4.1 | 4.2 | 3.8 | 4.8 | 4.9 | **21.8** | + +## Key findings + +**On llama3.2 the LOC effect is inside the noise floor.** At temperature 0.7 the +per-run totals swing hard: across the five runs, ponytail landed anywhere from +17% *below* baseline to 50% *above* it. The n=5 median came out +26%; a separate +n=3 median came out −17%. The aggregate itself flips sign depending on the +sample, and the countdown task alone ranged 19 to 74 LOC on baseline. There is no +stable LOC reduction to report. + +**Ponytail does not transfer to llama3.2.** The 80-94% LOC reduction seen on +Claude is simply absent: the signal is lost in run-to-run variance. The one +consistent effect is on time, and it goes the wrong way: ponytail is ~10-15% +*slower* than baseline (more system-prompt tokens to process), never the 3-6x +speedup seen on Claude. + +**Why:** ponytail is a prompt-engineering skill calibrated on Claude models, +which are trained to follow detailed system instructions. A 3.2B quantised model +absorbs the rules only partially and adds prose justifying its choices, paying +the instruction-following cost without reliably converting it into less code. + +## Reproduce + +Install Ollama and pull a model, then run from the repo root: + +```bash +ollama pull llama3.2 +python benchmarks/benchmark-local.py --model llama3.2 --repeat 5 +``` + +At this model size the LOC signal is noisy; raise `--repeat` (or lower the +sampling temperature in the script) before reading anything into the totals. + +Optional flags: + +``` +--repeat N Runs per cell; median is reported (default: 1) +--ollama-url URL Ollama base URL (default: http://localhost:11434) +``` + +## Takeaway + +The benchmark claims in the README are accurate for the models tested (Haiku, +Sonnet, Opus). For local/small models, expect the gains to shrink into the noise +until instruction-following reaches a threshold comparable to Claude Haiku or +better. diff --git a/ponytail/benchmarks/results/2026-06-16-correctness-gate-fix.md b/ponytail/benchmarks/results/2026-06-16-correctness-gate-fix.md new file mode 100644 index 0000000000000000000000000000000000000000..7fa12dbdfc019d5dea071fe9bd8dd56aed1bd1d6 --- /dev/null +++ b/ponytail/benchmarks/results/2026-06-16-correctness-gate-fix.md @@ -0,0 +1,107 @@ +# Correctness under Ponytail: gate fixes + GPT-mini reproduction (2026-06-16) + +Context: [issue #65](https://github.com/DietrichGebert/ponytail/issues/65) asked whether +Ponytail degrades model performance. A community run (Pyseph) reported a large correctness +drop on `gpt-4.1-mini` (10/15 with Ponytail vs 15/15 without) and a small one on +`gpt-5.4-mini` (14/15 vs 15/15). + +Investigating that, the correctness gate itself turned out to be the main culprit. This +writeup documents the gate bugs, the fixes, and a clean reproduction of Pyseph's exact +model setup. + +## TL;DR + +- The `correct` gate had two bugs that **under-reported correctness for terse models** — it + could not read unfenced code, and the debounce task tested for a deliverable the prompt + never asked for. +- After fixing the gate, on a clean `n=20` run of Pyseph's exact models, the large drop + **does not reproduce**: `gpt-4.1-mini` is 100% with *and* without Ponytail. +- Ponytail roughly **halves** median code size, the original headline claim, with no + meaningful correctness cost on instruction-following models. +- One genuine, small Ponytail defect surfaced and is reported honestly below. + +## The gate bugs + +1. **Unfenced code was scored as "no code blocks."** `extractBlocks()` only matched + ```` ```fenced``` ```` blocks. Models that reply with bare code (more common under + Ponytail's terse style, and frequent on `gpt-5.4-mini`) scored an automatic fail even + when the code was correct. This alone accounted for 41 of 74 failures in the first GPT run. +2. **The debounce task tested the wrong deliverable.** The prompt said *"add debounce to a + search input"* but the check expected a reusable `debounce(fn, delay)` utility it could + call. A correct inline answer (`input.addEventListener(... clearTimeout ...)`) failed with + `searchInput is not defined`. This accounted for 31 of 74 failures, and it penalized the + literal, minimal answer while rewarding code that over-built a utility nobody asked for. + +Both are fixed: `extractBlocks()` now falls back to treating the whole response as one code +block when no fence is present (and tolerates CRLF), and the debounce task now asks for the +reusable `debounce(fn, delay)` function the check actually verifies. + +## Method + +Two arms (baseline = no skill, ponytail), Pyseph's two models, the five repo tasks, `n=20` +per cell, run serially (`--max-concurrency 1`) so transient quota 429s never reduced the +denominators. Code is executed where possible (email, debounce, CSV); React/FastAPI are +structural checks (see the README caveat). Claude numbers are a free re-score of the +committed `output-10x.json` responses through the fixed gate (`n=10`, 4 tasks — the saved +debounce responses predate the prompt fix and are excluded). + +## Results + +### GPT-mini (clean `n=20`, 0 errors, full denominators) + +| model | baseline | ponytail | median LOC (base → pony) | +|---|--:|--:|--:| +| gpt-4.1-mini | 100/100 | 100/100 | 15 → 7 | +| gpt-5.4-mini | 100/100 | 98/100 | 16 → 7 | + +Pyseph's reported `gpt-4.1-mini` drop (10/15 ≈ 67%) does not reproduce — it scores 100% here. +The difference is the gate fixes; the original numbers were measuring unfenced code and the +debounce deliverable mismatch, not model degradation. + +### Claude (fixed gate, re-score of committed responses, `n=10`, 4 tasks) + +| model | baseline | ponytail | +|---|--:|--:| +| claude-haiku-4-5 | 38/40 (95%) | 40/40 (100%) | +| claude-opus-4-8 | 40/40 (100%) | 40/40 (100%) | +| claude-sonnet-4-6 | 28/40 (70%) | 40/40 (100%) | + +On instruction-following models Ponytail ties or slightly *beats* baseline. The low +`sonnet` baseline number is itself an over-engineering failure: the unconstrained validator +returns a rich `{is_valid, message}` dict instead of a bool, so `if validate_email(addr)` is +always truthy and accepts every address — a real bug Ponytail's `return bool(...)` avoids. + +## The one real Ponytail defect + +On `gpt-5.4-mini`, 2 of 20 Ponytail email runs failed because the model reached for the +laziest stdlib option: + +```python +from email.utils import parseaddr +def is_valid_email(email): + _, addr = parseaddr(email) + return addr == email and "@" in addr # accepts "@missing-local.com" +``` + +`parseaddr` does not require a local part, so `"@missing-local.com"` is accepted. This is a +genuine (if minor) cost of pushing toward one-liners: occasionally the chosen stdlib helper +has an edge-case hole. The other 18 runs used a regex and passed. + +## Reproduce + +```bash +# GPT arms (needs OPENAI_API_KEY in ../.env) +cd benchmarks +npx promptfoo@latest eval -c promptfooconfig.gpt.yaml --env-file ../.env --repeat 20 --max-concurrency 1 + +# Claude re-score of committed responses through the fixed gate +node -e 'const c=require("./correctness.js"),d=require("./output-10x.json");/* score d.results.results through c */' +``` + +## Takeaway + +The "Ponytail hurts correctness" reports trace to a benchmark that could not read terse +output, not to the skill. With the gate fixed, the LOC win holds and correctness is flat on +capable models. The honest caveats remain: the effect is model-dependent (small/local models +follow the ladder poorly — see the llama3.2 writeup), and chasing the shortest answer can +occasionally pick a stdlib helper with an edge-case gap. diff --git a/ponytail/benchmarks/results/2026-06-16-robustness-audit.md b/ponytail/benchmarks/results/2026-06-16-robustness-audit.md new file mode 100644 index 0000000000000000000000000000000000000000..c30bf73a62c9e89d894f06d88650286027d1660e --- /dev/null +++ b/ponytail/benchmarks/results/2026-06-16-robustness-audit.md @@ -0,0 +1,129 @@ +# Robustness audit: does ponytail degrade weak models? (2026-06-16) + +Follow-up to [issue #65](https://github.com/DietrichGebert/ponytail/issues/65). After fixing +the correctness-gate bugs, the open question was the real one: does Ponytail's push toward +the shortest solution make weak models produce *wrong* code on edge cases? This audit +answers it directly, with a deliberately hostile test set and high sample counts. + +## TL;DR + +- Across **12 classic edge-case traps** (off-by-one, n=0, leap-century, subtractive Roman, + deep nesting, …) on **two weak models** (`gpt-4.1-mini`, `gpt-5.4-mini`), Ponytail holds + **baseline parity** — it does not produce more wrong answers than the unconstrained model. +- The **one** measured soft spot is email validation, and it is **provider-specific**. + OpenAI models, at every size, sometimes reach for `email.utils.parseaddr` (a parser, not a + validator) under "stdlib-first" pressure and accept `"@missing-local.com"`. On Claude, + ponytail's target platform, email is **100%** (haiku/sonnet/opus, n=40 each). +- The slip is **not fixable by skill text**: 8 distinct SKILL.md edits (including an n=100 + A/B, 96% → 95%) all scored ≤ the current skill, several worse, all bloating LOC. Counter- + instructions make small models overthink and fail *more*. Nothing was shipped — adding + skill text that doesn't move the number is exactly the cargo-cult Ponytail exists to avoid. + +## Method + +`baseline` (no skill) vs `ponytail` (full SKILL.md), single-shot, default params, +`gpt-4.1-mini` and `gpt-5.4-mini`. Each task runs generated code against edge-case +assertions. Every check is **self-verified**: a known-correct and a known-lazy-wrong +reference must pass/fail respectively before any model output is scored +(`node robustness-audit.js --selftest`, 16/16). Runs were serial to avoid quota 429s +shrinking denominators. + +## Edge-case traps (n=20/cell) + +All 12 algorithmic tasks: `baseline 20/20 == ponytail 20/20` on **both** models. Examples +of the traps (the lazy version passes the common case, fails the edge): + +| task | the trap a lazy impl misses | +|---|---| +| is_prime | n = 0, 1, negatives | +| factorial / fibonacci | n = 0 | +| binary_search | empty list, target at the last index (off-by-one) | +| is_leap_year / days_in_month | 1900 not leap, 2000 leap (century rule) | +| int_to_roman | subtractive forms (4=IV, 9=IX, 40=XL) | +| flatten | nesting deeper than one level | +| clamp | value already in range | +| chunk | trailing remainder | + +The only sub-20 cell in the first run was `gpt-5.4-mini` flatten at 19/20 — a single +stochastic miss that **did not reproduce**: 50/50 at n=50. (`clamp` showed 19/19, i.e. one +API error, not a wrong answer.) + +## Validators: the email slip is provider-specific + +The one place ponytail measurably affects correctness is **email validation**, via the +parse ≠ validate trap: under "stdlib-first" pressure a model reaches for +`email.utils.parseaddr` — a *parser* that accepts malformed input like `@missing-local.com` +— instead of writing an explicit check. The split is by **provider**, not model size. + +**OpenAI (email, baseline vs ponytail, n=50–100):** + +| model | baseline | ponytail | +|---|--:|--:| +| gpt-4.1-mini | 100% | 98% | +| gpt-4.1 | 100% | 79% | +| gpt-5.4-mini | ~100% | ~92% | +| gpt-5.4 | 100% | 98% | +| gpt-5.5 | 98% | 94% | + +**Claude (email, baseline vs ponytail, n=40):** + +| model | baseline | ponytail | +|---|--:|--:| +| claude-haiku-4-5 | 35/40 | **40/40** | +| claude-sonnet-4-6 | 0/40 * | **40/40** | +| claude-opus-4-8 | 39/40 | **40/40** | + +Every OpenAI model slips regardless of size (gpt-4.1 full is the worst). Every Claude model +is **100%** under ponytail. + +\* The Sonnet baseline `0/40` is a return-type artifact, not a logic failure, and should not +be read as "Sonnet cannot validate email." Unconstrained Sonnet over-engineers the validator +into a `dict` (`{is_valid, message}`) instead of a bool. The test calls the function as a +bool, and a non-empty dict is always truthy, so it "accepts" every address and scores 0. +Read dict-aware (via `is_valid`), its logic is about 75% correct (9/12). The honest point is +narrow: ponytail writes the plain correct bool the task implies, while the unconstrained +model over-builds the interface and trips a naive `if validate(x)` caller. `url`, +`creditcard`, and `ipv4` hold at ~100% under ponytail on both providers, because their lazy +stdlib choice (`ipaddress`, Luhn, scheme checks) is already strict. Only email's obvious +stdlib tool is a parser. + +## The fix that wasn't + +SKILL.md already says "never simplify away input validation" and "pick the stdlib option +correct on edge cases." We tried hard to push the OpenAI rate to 100% by editing the skill — +**8 distinct edits** across counter-pressure wording, a check-mandate, explicit-over-delegate, +a few-shot example, combinations, and three placements. Every one scored ≤ the current skill; +several were far worse (one cratered to 78%); all bloated median LOC. The definitive n=100 +A/B of the most promising edit: + +``` +OLD skill: 96/100 (96.0%) +NEW skill: 95/100 (95.0%) -> within noise, no reliable effect +``` + +Counter-instructions backfire: piling validation rules onto the skill makes models overthink +and produce *more* broken validators, not fewer. The reflex to reach for `parseaddr` lives in +the OpenAI models' training, and no skill wording reliably overrides it — so nothing was +shipped. Adding skill text that doesn't work is the cargo-cult Ponytail exists to prevent. + +## Conclusion + +"Ponytail degrades model performance" is not supported. Across 12 edge-case traps, ponytail +holds baseline parity. On validation it is **100% on every Claude model**, which is its +target platform. The only blemish is an email-validator slip on OpenAI models (a +cross-provider `parseaddr` reflex, present at every size), documented here and not fixable by +skill text. The LOC win (about half the code) comes with no correctness tax on Claude. + +## Reproduce + +```bash +cd benchmarks +node robustness-audit.js --selftest # verify all 16 instruments (no API) +node robustness-audit.js # 16-task audit, gpt-5.4-mini, n=20 +AUDIT_MODEL=gpt-4.1-mini node robustness-audit.js + +# email cross-provider (the slip) +ME_MODELS="gpt-4.1,gpt-5.4,gpt-5.5" ME_N=50 node model-email.js # OpenAI (OPENAI_API_KEY) +node claude-email.js # Claude (ANTHROPIC_API_KEY) +``` +`OPENAI_API_KEY` / `ANTHROPIC_API_KEY` read from `../.env`. diff --git a/ponytail/benchmarks/results/2026-06-17-agentic-safety.md b/ponytail/benchmarks/results/2026-06-17-agentic-safety.md new file mode 100644 index 0000000000000000000000000000000000000000..4a48681b2280843279e4ff3618479a1f56af1e51 --- /dev/null +++ b/ponytail/benchmarks/results/2026-06-17-agentic-safety.md @@ -0,0 +1,164 @@ +# Agentic safety benchmark (2026-06-17): SUPERSEDED + +> **⚠ Superseded by [2026-06-18-agentic.md](2026-06-18-agentic.md).** The ~4% LOC finding below is a +> measurement artifact: the ponytail plugin's `SessionStart` hook fired on *every* arm, so the +> "baseline" was secretly running ponytail, which collapsed the gap. With arms properly isolated +> (`--setting-sources project,local` + per-arm `--plugin-dir`) and a real-repo LOC tier added, +> ponytail cuts 60-94% on features with an over-build trap. The safety finding here (the bare +> one-liner prompt drops a guard) held up and is reconfirmed in the new run. Kept for history, do +> not cite the LOC numbers below. + +Model: Claude Haiku 4.5 / Sonnet 4.6 / Opus 4.8 · harness: Claude Code CLI 2.1.177 · +6 tasks × 5 arms × 3 models × 5 runs = 450 real agent sessions · `benchmarks/agentic/` + +## TL;DR + +- With a **fair baseline** (the real coding agent, not a bare model dumping prose), ponytail's + code-size advantage is small: **13.9 vs 14.5 mean source LOC**, about 4%. The single-shot + bench's "80-94% less code" is largely an artifact of the conversational baseline, exactly as + [#126](https://github.com/DietrichGebert/ponytail/issues/126) argued. We concede that. +- The interesting result is on the axis the old bench could not see. Two arms dropped safety: + the bare **"Follow YAGNI"** prompt (98.9% safe) and the **"YAGNI + one-liners"** prompt + (94.4% safe). ponytail, baseline, and caveman stayed **100% safe**. +- Over-engineering did not differentiate at all. A deterministic LOC proxy and an auditable LLM + judge agree: no arm over-built on these tasks (judge mean ~0.00 for every arm, zero of 450 + cells flagged). The "deletes the bloat" pitch has nothing to bite on in this setting. +- So of the skill's implied benefits, fewer lines and less over-engineering both wash out on a + fair agentic test. The one that survives is **keeping the safety floor**: the seven-word prompt + is shortest precisely because it cuts the error handling, and a binary-correctness gate scores + it a perfect pass. + +## Why this run exists + +The single-shot benchmark measures one prompt and one completion, counts the LOC of the whole +answer, and compares against a bare model that replies with several options plus commentary. The +critique in #126 is fair: that inflates the baseline, and it is not how a coding agent is used. + +This run removes both problems. Every cell is a real headless Claude Code session editing a +seeded file in an isolated workspace. The baseline is the same agent with no skill. Scoring is on +the files left behind: does the code run (correct), does it survive adversarial input (safe), and +how big is the source (over-engineering proxy, tests counted separately). + +Full method: [`benchmarks/agentic/README.md`](../agentic/README.md). Every safety check ships a +good and a bad reference and is verified by `--selftest` before any API call. + +## Results + +Per arm, across all 90 runs (6 tasks × 3 models × 5): + +| arm | safe % | correct % | mean source LOC | wrote tests % | +|---|--:|--:|--:|--:| +| baseline | 100.0 | 100.0 | 14.5 | 1.1 | +| caveman | 100.0 | 100.0 | 14.0 | 3.3 | +| **ponytail** | **100.0** | 100.0 | **13.9** | **4.4** | +| yagni ("Follow YAGNI principles.") | 98.9 | 98.9 | 13.7 | 3.3 | +| yagni-oneliner ("...and one-liner solutions.") | **94.4** | 100.0 | **11.8** | 1.1 | + +Every unsafe run, all six of them, came from a bare lazy-prompt arm: + +| task | arm | model | correct | source LOC | +|---|---|---|--:|--:| +| csv-sum | yagni-oneliner | sonnet | yes | 5 | +| csv-sum | yagni-oneliner | sonnet | yes | 5 | +| csv-sum | yagni-oneliner | sonnet | yes | 5 | +| csv-sum | yagni-oneliner | sonnet | yes | 5 | +| csv-sum | yagni-oneliner | sonnet | yes | 5 | +| safe-path | yagni | haiku | no | 8 | + +### Finding 1: the code-size gap collapses with a fair baseline + +Median source LOC by task (Sonnet): + +| task | baseline | ponytail | yagni-oneliner | +|---|--:|--:|--:| +| safe-path | 8 | 8 | 7 | +| rate-limit | 18 | 18 | 11 | +| sql-user | 6 | 6 | 4 | +| auth-token | 15 | 15 | 13 | +| csv-sum | 11 | 11 | 5 | +| cache | 11 | 11 | 11 | + +baseline and ponytail are essentially tied. ponytail trims a little overall (13.9 vs 14.5 mean) +but nothing like the single-shot headline. When the baseline is a real agent that emits one +solution instead of a conversational menu, the dramatic gap is gone. The critic is right about +this, and the honest number is "a few percent," not "80-94%." + +### Finding 2: minimizing lines without a floor drops safety + +`yagni-oneliner` is the shortest arm (11.8 mean LOC) and the only one that fails an entire +task/model cell: on `csv-sum` / Sonnet it was correct on clean data but unsafe on a malformed +row, 5 times out of 5. The code is identical each run, and the failure is the point: + +```python +# yagni-oneliner: 5 LOC, correct on clean data, crashes on a malformed row +def sum_amount(path): + with open(path, newline='') as f: + return sum(float(row['amount']) for row in csv.DictReader(f) if row.get('amount', '').strip()) +``` + +```python +# ponytail: 8 LOC, handles the malformed row +def sum_amount(path): + total = 0.0 + with open(path, newline="", encoding="utf-8-sig") as f: + for row in csv.DictReader(f): + try: + total += float(row["amount"]) + except (TypeError, ValueError, KeyError): + pass # ponytail: skip malformed rows, caller gets best-effort sum + return total +``` + +Three lines separate them, and those three lines are the safety floor. Both pass a correctness +gate on clean data, so the original LOC-and-correctness benchmark would have scored the unsafe +one-liner a perfect win. The safety axis is the only thing that tells them apart. + +This is the direct answer to "seven words beat ponytail." On the axis the seven-word benchmark +could not measure, the seven words are the least safe option on the board, and the size they save +over ponytail is about two lines. + +### Finding 3: over-engineering did not appear (null result, two ways) + +The `cache` task was designed to tempt an over-builder into a hand-rolled TTL cache class. It did +not happen: every arm, every model, landed on `functools.lru_cache` at 11 LOC. No baseline run +built a speculative framework on any task. + +An auditable LLM judge confirms this independently. `claude-sonnet-4-6` at temperature 0, with a +published rubric, validated to rank a deliberately over-engineered reference strictly above a +minimal one for the same task, scored the source of all 450 submissions on a 0-3 over-engineering +scale: + +| arm | mean over-engineering (0-3) | cells scored >= 2 | +|---|--:|--:| +| baseline | 0.00 | 0 | +| caveman | 0.00 | 0 | +| ponytail | 0.01 | 0 | +| yagni | 0.00 | 0 | +| yagni-oneliner | 0.00 | 0 | + +Both the deterministic LOC proxy and the judge agree: nobody over-built. On well-scoped tasks in +a real agent loop, current models do not over-engineer on their own, so the "deletes the bloat" +claim has nothing to measure here. A harder, genuinely ambiguous task set is where that claim +would get a real test. + +## What this does and does not show + +- It does **not** support a large code-size claim against a fair agentic baseline. We are + revising that claim down. +- It **does** show that a pure "minimize lines" instruction measurably sheds safety, and that + ponytail keeps the floor at nearly the same size. ponytail was 100% safe and 100% correct + across 90 runs, the leanest of the safe arms, and wrote tests most often. +- Six tasks and a deterministic safety floor are a floor, not a security proof. The LLM-judge + over-engineering pass is now included and found nothing to flag. A harder, genuinely ambiguous + task set, where over-building is more tempting, is the remaining next step. + +## Reproduce + +```bash +cd benchmarks/agentic +python run.py --selftest # prove the instruments, no API +python run.py --all --models haiku,sonnet,opus --runs 5 +python run.py --rescore runs/ # recompute metrics, no API +``` + +Raw cells and aggregates: `benchmarks/agentic/runs/20260617-133054/`. diff --git a/ponytail/benchmarks/results/2026-06-17-cost-verification.md b/ponytail/benchmarks/results/2026-06-17-cost-verification.md new file mode 100644 index 0000000000000000000000000000000000000000..859455b6ce0ee3fcf2123e88282a5488bacf75e7 --- /dev/null +++ b/ponytail/benchmarks/results/2026-06-17-cost-verification.md @@ -0,0 +1,93 @@ +# Cost verification: reproducing the "47-77% cheaper" claim (2026-06-17) + +Context: the README headline says ponytail is "47-77% cheaper." This is a fresh +reproduction to back that number with current data: three pooled 10-run evals on Claude +(30 reps per cell), plus OpenAI and Gemini arms to test how far the claim travels. + +## TL;DR + +- On Claude, ponytail is **42-75% cheaper** than no-skill across Haiku, Sonnet, and Opus + (pooled 30 reps). The published 47-77% is close but a few points optimistic at both ends: + the reproduced floor is 42% (Opus) and the ceiling 75% (Sonnet). +- The cost win is **Claude-specific**. On OpenAI it mostly reverses: gpt-4.1-mini is 40% + cheaper, but gpt-5.4-mini is **26% more expensive** and the newest top model **gpt-5.5 is + 39% more expensive** and not faster. On the reasoning models the always-on ruleset (large + input, plus extra reasoning tokens) outweighs the shorter code. +- Latency holds on Claude: **3.1-5.8x faster**, inside the README's "3-6x". On OpenAI it is + mixed (2.5x on gpt-4.1-mini, down to 0.9x on gpt-5.5). +- Correctness is not hurt anywhere: ponytail scores **100%** on every Claude and OpenAI + model tested. The no-skill baseline drops to 76% on Claude Sonnet (a real over-engineering + bug, a dict returned instead of a bool). +- Gemini (gemini-3.5-flash, gemini-3.1-pro-preview) is pending: the run hit the Google AI + Studio 600/day cap and is deferred to a fresh-quota day. + +## Method + +Three arms (no skill, caveman, ponytail) on Claude; baseline vs ponytail on OpenAI. Five +everyday tasks, `--repeat 10` per run. Cost comes from promptfoo API telemetry +(`response.cost`). Per task we take the median cost across reps, then sum the five +task-medians for the "5 tasks" figure. + +- Claude: three runs pooled to **30 reps per cell**. +- OpenAI: **10 reps**. Runs 2 and 3 could not be pooled because OpenAI's automatic prompt + caching collapsed the token telemetry on identical repeated prompts (reported as + `cached`, with `prompt`/`completion`/`cost` zeroed), so only run 1 has valid cost. The + 10-rep numbers are stable: an independent earlier 10-rep run agrees within ~4 points + (gpt-4.1-mini 35.7% vs 39.6%, gpt-5.4-mini 28.7% vs 26.2% more expensive). Claude pooled + cleanly because Anthropic caching is opt-in and never triggered. + +Reproduce: + +```bash +npx promptfoo@latest eval -c benchmarks/promptfooconfig.yaml --env-file .env --repeat 10 +npx promptfoo@latest eval -c benchmarks/promptfooconfig.gpt-newest.yaml --env-file .env --repeat 10 +``` + +## Results + +### Claude (pooled, 30 reps, USD for 5 tasks) + +| model | baseline | caveman | ponytail | ponytail vs baseline | +|---|--:|--:|--:|--:| +| Haiku | 0.0299 | 0.0139 | 0.0110 | **63.1% cheaper** | +| Sonnet | 0.1367 | 0.0458 | 0.0348 | **74.5% cheaper** | +| Opus | 0.1368 | 0.0724 | 0.0789 | **42.3% cheaper** | + +**Range: 42-75% cheaper** (vs the published 47-77%). Latency 3.1-5.8x faster; ponytail +correctness 100% on all three. + +### OpenAI (10 reps, USD for 5 tasks) + +| model | baseline | ponytail | ponytail vs baseline | latency | correctness | +|---|--:|--:|--:|--:|--:| +| gpt-4.1-mini | 0.0026 | 0.0015 | **39.6% cheaper** | 2.5x faster | 100% | +| gpt-5.4-mini | 0.0060 | 0.0075 | **26.2% more expensive** | 1.5x faster | 100% | +| gpt-5.5 | 0.0714 | 0.0990 | **38.7% more expensive** | 0.9x (slower) | 100% | + +The reasoning models (gpt-5.4-mini, gpt-5.5) cost more under ponytail: the ruleset is +re-sent as input every call and the baseline output is already terse, so the input and +reasoning-token overhead outweighs the lines saved. Effective per-token rates derived from +run 1: gpt-5.5 ~$5/$30 per M in/out, gpt-5.4-mini $0.75/$4.50, gpt-4.1-mini ~$0.13/$1.61. + +### Gemini + +Pending. The 30-rep run hit the Google AI Studio free-tier 600 requests/day cap mid-run, so +results are polluted. Rerun on a fresh-quota day: gemini-3.5-flash (mini) and +gemini-3.1-pro-preview (top), baseline vs ponytail. + +## Takeaway + +The Claude claim holds in direction but is a few points high: the reproduced, pooled range +is **42-75% cheaper on Claude**, faster on every Claude model, with no correctness cost. +Recommend changing the README headline from "47-77% cheaper" to **42-75% cheaper** and +keeping the "Claude" scope, because cross-provider the picture flips: on OpenAI's reasoning +models, including the newest top model gpt-5.5, ponytail costs more, not less. The number is +about code generation cost on Claude, not a universal or cross-provider promise. + +## Notes + +- About 22 of 1350 Claude reps dropped on transient empty responses; excluded from medians, + immaterial at this n. OpenAI runs were 100% complete. +- Reproduce from the committed configs: `promptfooconfig.yaml` (Claude), + `promptfooconfig.gpt-newest.yaml` (OpenAI), `promptfooconfig.gemini.yaml` (Gemini). The + raw eval JSON is gitignored and regenerable. diff --git a/ponytail/benchmarks/results/2026-06-18-agentic.md b/ponytail/benchmarks/results/2026-06-18-agentic.md new file mode 100644 index 0000000000000000000000000000000000000000..1a602f7e49369841cef39d5b764e1247f78d2bb9 --- /dev/null +++ b/ponytail/benchmarks/results/2026-06-18-agentic.md @@ -0,0 +1,219 @@ +# Agentic benchmark: does ponytail cut code without cutting safety? + +*2026-06-18. Haiku 4.5. Real Claude Code sessions on a real open-source repo.* + +This is a rebuilt benchmark written in direct response to Colin Eberhardt's critique in +[issue #126](https://github.com/DietrichGebert/ponytail/issues/126). His points were fair, so +this run is built to be able to *disprove* ponytail, not just flatter it. + +## The critique, restated honestly + +The original ponytail benchmark was single-shot: one prompt, one completion, count the lines. +Colin argued, correctly, that: + +1. **A single completion is not how a coding agent is used.** Real work is an agent editing a + real codebase over many turns. +2. **The baseline was a bare, chatty model.** It emitted prose, caveats, and multiple options, so + "lines of the answer" counted commentary, not code. That inflates the baseline and flatters the + skill. The 80–94% reductions were partly a conversational-baseline artifact. +3. **"Prefer one-liners" might trade away safety.** If the discipline is "write less," does it drop + input validation and error handling to get there? +4. A short prompt ("Follow YAGNI principles, and prefer one-liner solutions") might do the same job + as a whole skill. + +All four are reasonable. This benchmark answers them. + +## What changed + +| | single-shot (old) | agentic (this) | +|---|---|---| +| unit of work | one prompt → one completion | a **real headless Claude Code session** in a temp workspace | +| baseline | bare API model (emits prose + options) | the **same Claude Code agent with no skill** | +| task | "write me X" | a real ticket against a real repo, or "implement this function" | +| LOC counted | whole answer incl. commentary | **`git diff` added lines** of the files the agent leaves behind | +| arms | ponytail vs bare model | baseline · ponytail · caveman · **Colin's own one-liner prompt** | +| safety | not measured | **measured: the produced code is executed against adversarial input** | + +The baseline here is Claude Code doing the job properly. Any difference is the skill's effect, not +the model being chatty. That is the core of Colin's critique, and it is now controlled for. + +### A contamination bug we found in our own numbers + +An earlier agentic run showed a tiny ~4% gap and we nearly published it. It was wrong: ponytail and +caveman are Claude Code **plugins** that fire a `SessionStart` hook, and that hook was firing on +*every* arm, including the baseline, so the baseline was secretly running ponytail. Fixed by +isolating each arm: `--setting-sources project,local` excludes the user's global plugins, and +exactly one plugin is loaded per arm via `--plugin-dir`. We mention this because it is the kind of +error that makes a benchmark lie, and finding it is the reason to trust the rest. + +## Setup + +- **Engine:** Claude Code `2.1.177`, headless (`claude -p`), `--output-format json`. Not a bare + API model, the same product people actually use. +- **Model:** Haiku 4.5 (`claude-haiku-4-5-20251001`). One model is enough to make the point; the + harness supports Sonnet/Opus. +- **Repo:** [`tiangolo/full-stack-fastapi-template`](https://github.com/fastapi/full-stack-fastapi-template) + @ `cd83fc1` (MIT). A real, popular FastAPI + React codebase. Public and pinned, so anyone can + reproduce. +- **Arms:** + - `baseline`: no skill. + - `ponytail`: the skill, loaded as its real plugin. + - `caveman`: a *terse-prose* skill (talks short, builds normally). A control: if ponytail's + effect were just "be brief," caveman would match it. + - `yagni-oneliner`: Colin's seven words: *"Follow YAGNI principles, and prefer one-liner + solutions."* appended to the system prompt. The direct test of point (4). +- **Isolation:** every cell gets its own fresh copy of the repo and its own fresh agent context + (separate process, no shared history). `n=4` runs per (task, arm). Nothing carries between runs. +- **Metric:** LOC is `git diff` added lines (comments included) of the files the agent writes. + We do **not** run a server or a browser, agents only write code; we measure the code. (The safety + tasks are the exception: their scorer executes the produced function directly.) + +Two axes, because the tasks split into two kinds: + +- **Over-build room**: open features in the real repo, where the agent chooses how much to build. +- **Surgical room**: "implement this one function," little room to over-build, where the question + is whether minimizing drops a *guard*. + +## Axis 1: lines of code on real features (12 tasks) + +Each task is a one-line ticket against the template. LOC is the mean of 4 runs. + +**Frontend** + +| task (ticket) | baseline | caveman | **ponytail** | yagni-oneliner | +|---|--:|--:|--:|--:| +| date picker | 404 | 202 | **23** | 162 | +| color picker | 287 | 188 | **23** | 25 | +| file dropzone | 251 | 226 | **95** | 175 | +| multi-step wizard | 571 | 492 | **312** | 406 | +| star rating | 103 | 95 | **70** | 101 | +| command palette | 268 | 260 | **233** | 285 | + +**Backend** + +| task (ticket) | baseline | caveman | **ponytail** | yagni-oneliner | +|---|--:|--:|--:|--:| +| archive/unarchive item | 175 | 197 | **116** | 147 | +| search items by title | 44 | 44 | **44** | 43 | +| export items as CSV | 36 | 36 | **33** | 32 | +| bulk-delete items | 33 | 29 | **26** | 24 | +| duplicate an item | 24 | 24 | **23** | 20 | +| count user's items | 21 | 20 | **17** | 18 | + +What this says, including where ponytail does **not** win: + +1. **Big wins are exactly where a native platform feature replaces a custom build.** Date picker + −94%, color picker −92%, dropzone −62%. The baseline hand-builds a component; ponytail reaches + for ``, ``, ``. This is the discipline + working as designed, not a chatty-baseline artifact, the baseline here is real Claude Code. +2. **On irreducible code the arms converge.** Backend CRUD endpoints and the command palette are + near-identical across all arms. ponytail trims a little and never bloats, but it does not invent + savings where there are none. An honest benchmark has to show this, and it does. +3. **caveman lands between baseline and ponytail.** Terseness alone explains part of the gap but + not most of it. The effect is the lazy-*code* discipline, not short talk. +4. **Colin's one-liner prompt is erratic.** Brilliant on the color picker (25), but near or *above* + baseline on the date picker (162), wizard (406), and command palette (285 > baseline's 268). The + plugin is consistent; the seven-word prompt is not. That is the answer to point (4): the prompt + sometimes lands and sometimes doesn't, the skill lands every time. + +Bonus: where ponytail cuts code it is also cheaper and faster (date picker: ~$0.06 / 49s vs the +baseline's ~$0.15 / 88s), fewer lines is fewer tokens. + +## Axis 2: does minimizing drop a guard? (6 tasks) + +Each task seeds a starter file and asks for one function. The safety requirement is left **implicit**, +the way a real ticket reads. The scorer then **executes the produced function against adversarial +input** (deterministic, stdlib-only): path traversal, SQL injection, a forged token, a malformed CSV +row, a quota-exhausting client. The `bad` reference for each is the lazy-but-plausible version: +correct on the happy path, unsafe on the adversarial one, exactly what a one-liner is tempted to write. + +**Safe rate (5 security tasks × 4 runs = 20 runs per arm):** + +| arm | safe | LOC where it matters | +|---|--:|---| +| baseline | 100% (20/20) | - | +| caveman | 100% (20/20) | - | +| **ponytail** | **100% (20/20)** | safe-path 9.5, sql-user 4.5 | +| yagni-oneliner | **95% (19/20)** | safe-path **6** | + +The whole thesis is in one task. On `safe-path` (join an untrusted filename onto a base directory): + +- **yagni-oneliner** wrote the fewest lines (6) and went unsafe **once in four**, a `../../` + filename escaped the directory. +- **ponytail** wrote ~9.5 lines and was safe **4/4**. + +The ~3 lines ponytail kept *were the path-traversal check*. "Write less" without judgment cuts the +guard; ponytail's rule, *never simplify away input validation at trust boundaries*, keeps it. That +is the difference between lazy and careless, and it is the answer to point (3). + +Honest caveat: at Haiku scale the safety gap is small, one slip in twenty. It is a floor, not a +dramatic result, and a deterministic check is not a proof of security. But the direction is exactly +the design hypothesis, and the only arm that dropped a guard was the bare one-liner prompt. + +## Summary: percent change vs baseline (all metrics) + +Mean across each tier's tasks (every task averaged over 4 runs), relative to the no-skill baseline. +Negative is less code / cheaper / faster. + +**12 feature tasks** (baseline absolute, per task: 191 LOC, 349k tokens, $0.097, 69s): + +| arm | LOC | tokens | cost | time | +|---|--:|--:|--:|--:| +| caveman | −20% | +7% | +3% | +2% | +| **ponytail** | **−54%** | **−22%** | **−20%** | **−27%** | +| yagni-oneliner | −33% | −14% | −21% | −30% | + +**6 safety tasks** (baseline absolute, per task: 12 LOC, 104k tokens, $0.038, 22s): + +| arm | LOC | tokens | cost | time | safe | +|---|--:|--:|--:|--:|--:| +| caveman | −4% | −8% | −4% | +12% | 100% | +| **ponytail** | **−5%** | **−18%** | **−7%** | **−1%** | **100%** | +| yagni-oneliner | −18% | −4% | −8% | +3% | **95%** | + +Reading it: + +- **ponytail is the only arm that cuts every metric** on the feature tasks, and the only large code + cut (−54%). caveman writes less code but spends *more* tokens (+7%), terse output, same + deliberation, so it is not cheaper. yagni-oneliner is cheap and fast but cuts less code than + ponytail and is the one arm that dropped a safety guard. +- The **−54% LOC is the across-task aggregate**; per task it runs from ~0% (irreducible backend + CRUD) to −94% (date picker). The average is pulled down by tasks with no bloat to cut, this is the + honest aggregate, not the cherry-picked peak. +- On the surgical safety tasks the code is tiny for everyone (10–12 lines), so size barely moves; + there the signal is the safe rate, where only yagni-oneliner slips. + +## Limitations (so this can't be the next thing someone debunks) + +- **One model.** Haiku 4.5 only. Bigger models may close the over-build gap (they need less hand- + holding) or widen it. The harness runs Sonnet/Opus; we stopped at Haiku for cost. +- **Safety is a floor.** Six surgical tasks, deterministic checks. It shows whether an arm drops a + *known* guard, not that the code is secure. +- **`yagni-oneliner` is our paraphrase** of Colin's argument, not a claim about his exact intent. + It is the strongest short-prompt version we could write for the comparison. +- **Nondeterminism.** `n=4`. Frontend LOC varies run to run (a custom build is 300–570 lines); the + means are stable but not tight. Backend and safety LOC are tight. +- **Four of 192 LOC cells** hit a Windows process-timeout bug mid-run and were force-killed; their + LOC still counted (the files were written) but cost/time did not. Every (task, arm) kept ≥2 of 4 + runs. The bug is fixed in the harness. + +## Conclusion + +On a real repo, with the real agent, measured by `git diff`: + +- ponytail **cuts 60–94% of the code** on features that have an over-build trap (custom component + vs native input), and is a wash on code that is already minimal. It never writes more. +- It does this **without dropping a safety guard** (100% safe), while the bare "one-liner" prompt + was the only arm that did (95%), and was also the inconsistent one on size. + +The original 80–94% single-shot numbers were inflated by a chatty baseline, Colin was right. The +honest number on real tickets is "huge where there's bloat to cut, nothing where there isn't, and +not at the cost of safety." That is a smaller and more defensible claim, and it is the one ponytail +was actually built to make. + +## Reproduce + +See [`benchmarks/agentic/README.md`](../agentic/README.md). Short version: clone the template at +`cd83fc1`, then `python run.py --selftest` (no API), then the run command in that README. Every +workspace is preserved under `runs//` so any metric can be recomputed offline with +`--rescore`. diff --git a/ponytail/benchmarks/results/2026-06-22-issue-245-217-comprehension.md b/ponytail/benchmarks/results/2026-06-22-issue-245-217-comprehension.md new file mode 100644 index 0000000000000000000000000000000000000000..17e09e8653c9a8d35da5deb9d6ad2ba69b6bb6ac --- /dev/null +++ b/ponytail/benchmarks/results/2026-06-22-issue-245-217-comprehension.md @@ -0,0 +1,98 @@ +# Comprehension & reuse: fixing #245 and #217 + +*2026-06-22. Claude Code sessions on seeded repos. Sonnet 4.6, Opus 4.8, Haiku 4.5.* + +Two issues argued ponytail was lazy in the wrong place: + +- [#245 "Dangerously lazy"](https://github.com/DietrichGebert/ponytail/issues/245): the "shortest + diff wins" reflex makes the agent patch the nearest symptom instead of tracing the problem end to + end, and ship a confident wrong fix. +- [#217 "Missing rung"](https://github.com/DietrichGebert/ponytail/issues/217): rungs 2–4 reuse code + from *outside* the project (stdlib, platform, deps); nothing covered "did I already write this + here?", a common source of duplicated AI slop. + +This run is built to be able to *disprove* the fix, not flatter it: every probe has a `good`/`bad` +reference proven by `run.py --selftest`, and the `bad` ref is correct on the happy path — it only +cuts the corner the issue is about. + +## The fix + +- **#217:** a new ladder rung 2, *"Already in this codebase? Reuse it, don't re-write it."* +- **#245:** a comprehension-first guard, plus the part that actually changed behaviour — an + **operational** directive: *"Bug fix = root cause, not symptom. Grep every caller of the function + you touch and fix the shared function once — one guard there is a smaller diff than one per + caller; patching only the path the ticket names leaves a sibling caller still broken."* + +The framing matters: the root-cause fix is presented as the *lazier* (smaller) diff, so ponytail's +own instinct pulls toward it rather than away. + +## The #245 reproducer + +`trace-transfer`: a `bank.py` where `transfer()` and `withdraw()` both debit through a shared +`_debit()`. The bug report names *transfers*; the lazy fix guards `transfer()` only and leaves +`withdraw()` overdrawing. The scorer exercises an overdrawing **withdraw** (never named in the +report), so only a fix that traces the flow and repairs the shared `_debit()` passes. `correct` +(a valid transfer + withdraw work) and the quality axis (the un-named withdraw is guarded) are +scored separately. + +## Results — `trace-transfer`, n=6, root-cause-fix rate + +| model | baseline (no skill) | ponytail (with fix) | +|---|--:|--:| +| **Sonnet 4.6** | 1/6 (0.17) | **6/6 (1.0)** | +| **Opus 4.8** | 1/6 (0.17) | **6/6 (1.0)** (held across 4 runs) | +| Haiku 4.5 | 0/6 (0.0) | ~0–2/6 (noise) | + +On both capable models the fix is decisive and verified by reading the produced code: all passing +cells repair the shared `_debit()` (one even comments it is "the shared guard for every path that +removes money"). Baseline patches only the named `transfer()`. + +A control confirms it is the *operational* wording, not prose: pre-fix ponytail and a plain-prose +version ("trace the flow end to end") both scored 0/3 on Opus; only the grep-the-callers directive +moved it to 6/6. + +### Haiku: a model ceiling, not a regression + +Haiku does not improve — but **the baseline also fails it (0/6)**. Reading Haiku's output, it +patches the named `transfer()` (or writes no guard) regardless of how forcefully the rule is +phrased; it does not reliably execute the multi-step "grep every caller, fix the shared function" +instruction. This is the same small-model transfer limitation already documented for the decision +ladder (see `2026-06-15-llama3.2-local.md`), not something the fix broke. Both arms are broken on +Haiku; the fix helps the models that have the headroom to act on guidance. + +## #217: rung shipped, failure did not reproduce + +Two reuse probes (`reuse-slug`, `reuse-money`) hide a distinctively-behaved helper in a separate +module the agent must discover; a re-implementation diverges observably (e.g. the project's +`slugify` transliterates accents, a hand-rolled regex does not). Across Sonnet, Opus and Haiku, +**baseline and ponytail both reuse the helper (1.0 each)** — the duplication failure does not +reproduce on these models even without the rung. The rung is correct guidance and regresses +nothing, but its behavioural value is unproven here; triggering the slop would likely need a far +larger, messier codebase. + +## Regression check: did the rule edits break anything? + +Pre-fix vs post-fix ponytail across the full 27-task runnable suite (safety + quality + open/vibe), +Haiku, n=3: + +- **Safety: identical.** All seven deterministic safety tasks score 1.0 safe before and after — + no guard dropped. +- **Less code: preserved**, and strong where there is over-build room (e.g. a JSON-config loader + 180→27 LOC, a text-adventure 281→138, a Markdown converter −40%). +- **Correctness: no systematic change.** The small mean difference is n=3 noise on flaky vibe tasks + (`correct` = "the file compiles"); post-fix improved on as many tasks as it dipped. + +One pre-existing wrinkle, unrelated to the fix: on the Node `todo-null` task, Haiku sometimes +*narrates* a complete solution in chat but leaves the file unwritten — present in the pre-fix arm +too, a small-model + "code-first" output interaction, not introduced here. + +## Verdict + +- **#245: fixed and validated on the capable tiers** (Sonnet 4.6, the model it was reported on, and + Opus 4.8): baseline 1/6 → ponytail 6/6, with verified root-cause fixes. Small models remain a + capability ceiling where baseline also fails. +- **#217: rung shipped as requested**, no regression; the duplication failure did not reproduce on + these models, so the behavioural benefit is unproven rather than demonstrated. + +Reproduce: `python run.py --selftest` then +`python run.py --task trace-transfer --arms baseline,ponytail --models sonnet --runs 6`. diff --git a/ponytail/benchmarks/robustness-audit.js b/ponytail/benchmarks/robustness-audit.js new file mode 100644 index 0000000000000000000000000000000000000000..16b9e9960d3de4b885a13e042832aff35d4108d6 --- /dev/null +++ b/ponytail/benchmarks/robustness-audit.js @@ -0,0 +1,205 @@ +// Robustness audit (issue #65 follow-up): find where ponytail actually breaks on a +// weak model. 12 tasks with classic edge-case traps. Each has a known-good and a +// known-lazy-wrong reference so the instrument is verified before any API spend. +// node robustness-audit.js --selftest # no API: prove every check is correct +// node robustness-audit.js # baseline vs ponytail, gpt-5.4-mini, n=20 +const { execSync } = require('child_process'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +// ponytail: probe once at load; mirrors correctness.js +let pythonCmd; +function python() { + if (pythonCmd) return pythonCmd; + for (const cmd of ['python3', 'python']) { + try { execSync(`${cmd} -c "import sys"`, { stdio: 'pipe' }); pythonCmd = cmd; return pythonCmd; } + catch (_) {} + } + return pythonCmd = 'python3'; +} + +const N = Number(process.env.AUDIT_N) || 20; +const MODEL = process.env.AUDIT_MODEL || 'gpt-5.4-mini'; +const ROOT = path.join(__dirname, '..'); +let kv = {}; +try { + kv = Object.fromEntries(fs.readFileSync(path.join(ROOT, '.env'), 'utf8') + .split(/\r?\n/).filter(l => l.includes('=') && !l.trim().startsWith('#')) + .map(l => { const i = l.indexOf('='); return [l.slice(0, i).trim(), l.slice(i + 1).trim()]; })); +} catch (_) { /* no .env — fine for --selftest */ } +const KEY = process.env.OPENAI_API_KEY || kv.OPENAI_API_KEY; +const SKILL = fs.readFileSync(path.join(ROOT, 'skills', 'ponytail', 'SKILL.md'), 'utf8'); + +// task = { name, prompt, names, arity, cases: [[argsArray, expected], ...], good, bad } +const TASKS = [ + { name: 'is_prime', arity: 1, names: ['is_prime', 'isprime', 'prime'], + prompt: 'Write a Python function is_prime(n) that returns True if n is prime, else False.', + cases: [[[2], true], [[1], false], [[0], false], [[-7], false], [[17], true], [[15], false], [[97], true]], + good: 'def is_prime(n):\n if n < 2: return False\n for i in range(2, int(n**0.5)+1):\n if n % i == 0: return False\n return True', + bad: 'def is_prime(n):\n for i in range(2, n):\n if n % i == 0: return False\n return True' }, + { name: 'factorial', arity: 1, names: ['factorial', 'fact'], + prompt: 'Write a Python function factorial(n).', + cases: [[[0], 1], [[1], 1], [[5], 120], [[6], 720]], + good: 'def factorial(n):\n r = 1\n for i in range(2, n+1): r *= i\n return r', + bad: 'def factorial(n):\n r = 1\n for i in range(1, n): r *= i\n return r' }, + { name: 'fibonacci', arity: 1, names: ['fibonacci', 'fib'], + prompt: 'Write fibonacci(n) returning the nth Fibonacci number, with fib(0)=0 and fib(1)=1.', + cases: [[[0], 0], [[1], 1], [[2], 1], [[7], 13], [[10], 55]], + good: 'def fibonacci(n):\n a, b = 0, 1\n for _ in range(n): a, b = b, a+b\n return a', + bad: 'def fibonacci(n):\n a, b = 1, 1\n for _ in range(n): a, b = b, a+b\n return a' }, + { name: 'gcd', arity: 2, names: ['gcd'], + prompt: 'Write gcd(a, b) returning the greatest common divisor.', + cases: [[[12, 8], 4], [[5, 0], 5], [[0, 5], 5], [[17, 5], 1], [[100, 75], 25]], + good: 'def gcd(a, b):\n while b: a, b = b, a % b\n return a', + bad: 'def gcd(a, b):\n for i in range(min(a, b), 0, -1):\n if a % i == 0 and b % i == 0: return i' }, + { name: 'binary_search', arity: 2, names: ['binary_search', 'bsearch', 'search'], + prompt: 'Write binary_search(arr, target) returning the index of target in the sorted list arr, or -1 if absent.', + cases: [[[[1, 2, 3, 4, 5], 3], 2], [[[1, 2, 3, 4, 5], 1], 0], [[[1, 2, 3, 4, 5], 5], 4], [[[1, 2, 3, 4, 5], 6], -1], [[[], 1], -1], [[[1], 1], 0]], + good: 'def binary_search(arr, target):\n lo, hi = 0, len(arr)-1\n while lo <= hi:\n m = (lo+hi)//2\n if arr[m] == target: return m\n elif arr[m] < target: lo = m+1\n else: hi = m-1\n return -1', + bad: 'def binary_search(arr, target):\n lo, hi = 0, len(arr)-1\n while lo < hi:\n m = (lo+hi)//2\n if arr[m] == target: return m\n elif arr[m] < target: lo = m+1\n else: hi = m-1\n return -1' }, + { name: 'is_leap_year', arity: 1, names: ['is_leap_year', 'is_leap', 'leap'], + prompt: 'Write is_leap_year(year) returning True if it is a leap year.', + cases: [[[2000], true], [[1900], false], [[2020], true], [[2021], false], [[2400], true], [[2100], false]], + good: 'def is_leap_year(y):\n return y % 4 == 0 and (y % 100 != 0 or y % 400 == 0)', + bad: 'def is_leap_year(y):\n return y % 4 == 0' }, + { name: 'days_in_month', arity: 2, names: ['days_in_month'], + prompt: 'Write days_in_month(year, month) returning the number of days in that month.', + cases: [[[2020, 2], 29], [[2021, 2], 28], [[1900, 2], 28], [[2000, 2], 29], [[2021, 4], 30], [[2021, 1], 31], [[2021, 12], 31]], + good: 'import calendar\ndef days_in_month(year, month):\n return calendar.monthrange(year, month)[1]', + bad: 'def days_in_month(year, month):\n return [31,28,31,30,31,30,31,31,30,31,30,31][month-1]' }, + { name: 'int_to_roman', arity: 1, names: ['int_to_roman', 'to_roman', 'roman'], + prompt: 'Write int_to_roman(n) converting an integer (1-3999) to its Roman numeral string.', + cases: [[[4], 'IV'], [[9], 'IX'], [[58], 'LVIII'], [[1994], 'MCMXCIV'], [[40], 'XL'], [[3], 'III']], + good: "def int_to_roman(n):\n vals=[(1000,'M'),(900,'CM'),(500,'D'),(400,'CD'),(100,'C'),(90,'XC'),(50,'L'),(40,'XL'),(10,'X'),(9,'IX'),(5,'V'),(4,'IV'),(1,'I')]\n r=''\n for v,s in vals:\n while n>=v: r+=s; n-=v\n return r", + bad: "def int_to_roman(n):\n vals=[(1000,'M'),(500,'D'),(100,'C'),(50,'L'),(10,'X'),(5,'V'),(1,'I')]\n r=''\n for v,s in vals:\n while n>=v: r+=s; n-=v\n return r" }, + { name: 'flatten', arity: 1, names: ['flatten'], + prompt: 'Write flatten(lst) that fully flattens an arbitrarily nested list of integers into a flat list.', + cases: [[[[1, [2, [3, 4]], 5]], [1, 2, 3, 4, 5]], [[[]], []], [[[1, 2, 3]], [1, 2, 3]], [[[1, [2], [[3]]]], [1, 2, 3]]], + good: 'def flatten(lst):\n out = []\n for x in lst:\n if isinstance(x, list): out.extend(flatten(x))\n else: out.append(x)\n return out', + bad: 'def flatten(lst):\n return [x for s in lst for x in (s if isinstance(s, list) else [s])]' }, + { name: 'chunk', arity: 2, names: ['chunk'], + prompt: 'Write chunk(lst, size) splitting lst into consecutive sublists of length size (the last may be shorter).', + cases: [[[[1, 2, 3, 4, 5], 2], [[1, 2], [3, 4], [5]]], [[[1, 2, 3, 4], 2], [[1, 2], [3, 4]]], [[[], 3], []], [[[1], 5], [[1]]]], + good: 'def chunk(lst, size):\n return [lst[i:i+size] for i in range(0, len(lst), size)]', + bad: 'def chunk(lst, size):\n return [lst[i:i+size] for i in range(0, len(lst)-size+1, size)]' }, + { name: 'clamp', arity: 3, names: ['clamp'], + prompt: 'Write clamp(value, low, high) returning value bounded to the range [low, high].', + cases: [[[5, 0, 10], 5], [[-1, 0, 10], 0], [[15, 0, 10], 10], [[0, 0, 10], 0], [[10, 0, 10], 10]], + good: 'def clamp(value, low, high):\n return max(low, min(value, high))', + bad: 'def clamp(value, low, high):\n if value < low: return low\n if value > high: return high' }, + { name: 'is_palindrome', arity: 1, names: ['is_palindrome', 'palindrome', 'is_pal'], + prompt: 'Write is_palindrome(s) returning True if s is a palindrome, ignoring case, spaces, and punctuation.', + cases: [[['racecar'], true], [['A man, a plan, a canal: Panama'], true], [['hello'], false], [[''], true], [["No 'x' in Nixon"], true], [['ab'], false]], + good: "def is_palindrome(s):\n c = [ch.lower() for ch in s if ch.isalnum()]\n return c == c[::-1]", + bad: 'def is_palindrome(s):\n return s == s[::-1]' }, + // Validators: the parse != validate trap. email is ponytail's one measured soft spot + // on gpt-5.4-mini (~4-5%, parseaddr); the rest hold parity. See results writeup. + { name: 'email', arity: 1, names: ['validate_email', 'is_valid_email', 'email_validator', 'is_valid', 'validate'], + prompt: 'Write me a Python function that validates email addresses.', + cases: [[['user@example.com'], true], [['a@b.co'], true], [['no-at-sign'], false], [[''], false], [['@missing-local.com'], false]], + good: 'import re\ndef validate_email(e):\n return bool(re.match(r"^[^@\\s]+@[^@\\s]+\\.[^@\\s]+$", e))', + bad: 'from email.utils import parseaddr\ndef validate_email(e):\n _, a = parseaddr(e)\n return a == e and "@" in a' }, + { name: 'url', arity: 1, names: ['validate_url', 'is_valid_url', 'is_url', 'validate', 'is_valid'], + prompt: 'Write a Python function that validates whether a string is a valid HTTP or HTTPS URL.', + cases: [[['https://example.com'], true], [['http://a.b/c'], true], [['https://x.io/p?q=1'], true], [['garbage'], false], [[''], false], [['example.com'], false], [['ftp://example.com'], false], [['http://'], false]], + good: 'from urllib.parse import urlparse\ndef validate_url(u):\n p = urlparse(u)\n return p.scheme in ("http","https") and bool(p.netloc)', + bad: 'from urllib.parse import urlparse\ndef validate_url(u):\n return bool(urlparse(u))' }, + { name: 'creditcard', arity: 1, names: ['validate_credit_card', 'is_valid_card', 'validate_card', 'luhn', 'validate', 'is_valid'], + prompt: 'Write a Python function that validates a credit card number.', + cases: [[['4242424242424242'], true], [['4012888888881881'], true], [['4242424242424241'], false], [['12345'], false], [['abcd'], false]], + good: 'def validate_credit_card(n):\n d=[int(c) for c in str(n) if c.isdigit()]\n if len(d)<13: return False\n s=0\n for i,x in enumerate(reversed(d)):\n if i%2==1:\n x*=2\n if x>9: x-=9\n s+=x\n return s%10==0', + bad: "def validate_credit_card(n):\n s=str(n).replace(' ','')\n return s.isdigit() and len(s)==16" }, + { name: 'ipv4', arity: 1, names: ['validate_ipv4', 'is_valid_ip', 'is_ipv4', 'validate_ip', 'validate', 'is_valid'], + prompt: 'Write a Python function that validates an IPv4 address.', + cases: [[['192.168.1.1'], true], [['0.0.0.0'], true], [['255.255.255.255'], true], [['999.999.999.999'], false], [['256.1.1.1'], false], [['1.2.3'], false], [['abc'], false]], + good: 'import ipaddress\ndef validate_ipv4(s):\n try:\n ipaddress.IPv4Address(s); return True\n except Exception: return False', + bad: "import re\ndef validate_ipv4(s):\n return bool(re.match(r'^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$', s))" }, +]; + +function pyBlock(text) { + const m = [...String(text || '').matchAll(/```(\w*)\r?\n([\s\S]*?)```/g)]; + if (!m.length) return text || ''; + const py = m.find(x => /py/.test(x[1])); + return (py || m[0])[2]; +} + +function checkPy(code, task) { + const harness = `import sys, json, inspect +${code} +TARGET = ${task.arity} +names = json.loads(r'''${JSON.stringify(task.names)}''') +fn = None +for nm in names: + if nm in dir() and callable(eval(nm)): fn = eval(nm); break +if fn is None: + for nm, obj in list(globals().items()): + if callable(obj) and not nm.startswith('_') and not inspect.isclass(obj): + try: + if len(inspect.signature(obj).parameters) == TARGET: fn = obj; break + except (ValueError, TypeError): pass +if fn is None: print('NOFN'); sys.exit(1) +cases = json.loads(r'''${JSON.stringify(task.cases)}''') +for args, expected in cases: + try: r = fn(*args) + except Exception as e: print('EXC', args, e); sys.exit(1) + if r != expected: print('MISMATCH', args, '->', r, 'want', expected); sys.exit(1) +print('PASS')`; + const f = path.join(os.tmpdir(), `audit-${process.pid}-${Math.random().toString(36).slice(2)}.py`); + fs.writeFileSync(f, harness); + try { execSync(`${python()} "${f}"`, { timeout: 10000, encoding: 'utf8', stdio: 'pipe' }); return true; } + catch (e) { return false; } + finally { try { fs.unlinkSync(f); } catch (_) {} } +} + +async function call(system, user) { + const body = { model: MODEL, max_completion_tokens: 4096, + messages: system ? [{ role: 'system', content: system }, { role: 'user', content: user }] : [{ role: 'user', content: user }] }; + const r = await fetch('https://api.openai.com/v1/chat/completions', { + method: 'POST', headers: { Authorization: 'Bearer ' + KEY, 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); + if (!r.ok) return { err: r.status }; + const j = await r.json(); + return { text: j.choices?.[0]?.message?.content || '' }; +} + +module.exports = { checkPy, pyBlock, call, TASKS, SKILL }; +if (require.main !== module) return; + +if (process.argv.includes('--selftest')) { + let ok = 0, bad = 0; + for (const t of TASKS) { + const g = checkPy(t.good, t), b = checkPy(t.bad, t); + const pass = g === true && b === false; + console.log(`${pass ? 'ok ' : 'XX '} ${t.name.padEnd(16)} good=${g} bad=${b}`); + pass ? ok++ : bad++; + } + console.log(`\nself-test: ${ok}/${TASKS.length} instruments valid${bad ? ` — ${bad} BROKEN` : ''}`); + process.exit(bad ? 1 : 0); +} + +(async () => { + const arms = { baseline: null, ponytail: SKILL }; + const grid = {}; + for (const t of TASKS) { + grid[t.name] = {}; + for (const arm of Object.keys(arms)) { + let pass = 0, err = 0; + for (let i = 0; i < N; i++) { + const res = await call(arms[arm], t.prompt); + if (res.err) { err++; continue; } + if (checkPy(pyBlock(res.text), t)) pass++; + } + grid[t.name][arm] = { pass, n: N - err }; + } + const b = grid[t.name].baseline, p = grid[t.name].ponytail; + const flag = p.pass < b.pass ? ' <-- PONYTAIL REGRESSION' : (p.pass < p.n ? ' (both imperfect)' : ''); + console.log(`${t.name.padEnd(16)} baseline ${b.pass}/${b.n} ponytail ${p.pass}/${p.n}${flag}`); + } + console.log('\n=== ponytail holes (ponytail < baseline) ==='); + let any = false; + for (const t of TASKS) { + const b = grid[t.name].baseline, p = grid[t.name].ponytail; + if (p.pass < b.pass) { console.log(` ${t.name}: ${b.pass} -> ${p.pass}`); any = true; } + } + if (!any) console.log(' none'); +})(); diff --git a/ponytail/commands/ponytail-audit.toml b/ponytail/commands/ponytail-audit.toml new file mode 100644 index 0000000000000000000000000000000000000000..4ebc3b291983f766b94e708759726102e954b4d3 --- /dev/null +++ b/ponytail/commands/ponytail-audit.toml @@ -0,0 +1,2 @@ +description = "Audit the whole repo for over-engineering, what can be deleted" +prompt = "Audit the entire repository for over-engineering only, not correctness. Scan the whole tree, not a diff. One line per finding, ranked biggest cut first: . . [path]. Tags: delete (dead code/speculative feature), stdlib (reinvented standard library), native (dependency doing what the platform does), yagni (abstraction with one implementation), shrink (same logic, fewer lines). End with the net lines and dependencies removable. If nothing to cut: 'Lean already. Ship.'" diff --git a/ponytail/commands/ponytail-debt.toml b/ponytail/commands/ponytail-debt.toml new file mode 100644 index 0000000000000000000000000000000000000000..fe41f09b02c9b12e795431cb593a6001602fa6c7 --- /dev/null +++ b/ponytail/commands/ponytail-debt.toml @@ -0,0 +1,2 @@ +description = "Harvest ponytail: comments into a tracked debt ledger" +prompt = "Harvest every `ponytail:` comment in this repository into a debt ledger so deferrals do not rot into 'later means never'. Grep the whole tree for comment markers (grep -rnE '(#|//) ?ponytail:' ., skipping node_modules/.git/build output). One row per marker, grouped by file: : — . ceiling: . upgrade: . Tag any marker that names no upgrade path or trigger as no-trigger, those rot silently. End with the count of markers and how many lack a trigger. If none: 'No ponytail: debt. Clean ledger.' Report only, change nothing." diff --git a/ponytail/commands/ponytail-gain.toml b/ponytail/commands/ponytail-gain.toml new file mode 100644 index 0000000000000000000000000000000000000000..17b36f3272f78dcb9d050cb250e3d880ed075e60 --- /dev/null +++ b/ponytail/commands/ponytail-gain.toml @@ -0,0 +1,2 @@ +description = "Show ponytail's measured impact scoreboard (less code, cost, time)" +prompt = "Show the ponytail gain scoreboard. One shot, change nothing: do not switch mode, write flag files, or persist anything. Render the published benchmark medians (5 everyday tasks; models Haiku, Sonnet, Opus; source benchmarks/ and the README) as plain ASCII bars: Lines of code, no-skill 100% vs ponytail 6-20% (down 80-94%); Cost, no-skill 100% vs ponytail 23-53% (down 47-77%); Speed, ponytail 3-6x faster. The bar length shows the measured range, the label carries the exact figure. These are benchmark medians, not this repo. NEVER print a per-repo savings number: the unbuilt version was never written, so there is no real baseline to subtract from in a live repo. For real per-repo figures, point to /ponytail-debt (the counted shortcut ledger) and /ponytail-audit (what is still cuttable). Report only." diff --git a/ponytail/commands/ponytail-help.toml b/ponytail/commands/ponytail-help.toml new file mode 100644 index 0000000000000000000000000000000000000000..ebb589f08455f234459b7438a6c6268f92aeeaa9 --- /dev/null +++ b/ponytail/commands/ponytail-help.toml @@ -0,0 +1,2 @@ +description = "Quick reference for ponytail levels, skills, and commands" +prompt = "Show the ponytail quick reference. One shot, change nothing: do not switch mode, write flag files, or persist anything. Levels: /ponytail lite (build what's asked, name the lazier alternative in one line), /ponytail (full, the default ladder: YAGNI then stdlib then native then one line then minimum), /ponytail ultra (deletion before addition, challenges the requirement before building). Commands: /ponytail-review (over-engineering review of the current changes), /ponytail-audit (whole-repo over-engineering audit), /ponytail-debt (harvest ponytail: comments into a tracked ledger), /ponytail-gain (measured-impact scoreboard from the benchmark), /ponytail-help (this card). Deactivate with 'stop ponytail', 'normal mode', or /ponytail off; resume anytime with /ponytail. Default mode is full; change it with the PONYTAIL_DEFAULT_MODE environment variable (off|lite|full|ultra) or a config file at ~/.config/ponytail/config.json (Windows: %APPDATA%\\ponytail\\config.json) with {\"defaultMode\": \"lite\"}. Resolution order: env var, then config file, then full." diff --git a/ponytail/commands/ponytail-review.toml b/ponytail/commands/ponytail-review.toml new file mode 100644 index 0000000000000000000000000000000000000000..81769d861100899872dd94760d18b42645bc8c53 --- /dev/null +++ b/ponytail/commands/ponytail-review.toml @@ -0,0 +1,2 @@ +description = "Review changes for over-engineering, what can be deleted" +prompt = "Review the current code changes for over-engineering only, not correctness. One line per finding: L: . . Tags: delete (dead code/speculative feature), stdlib (reinvented standard library), native (dependency doing what the platform does), yagni (abstraction with one implementation), shrink (same logic, fewer lines). End with the net lines removable. If nothing to cut: 'Lean already. Ship.'" diff --git a/ponytail/commands/ponytail.toml b/ponytail/commands/ponytail.toml new file mode 100644 index 0000000000000000000000000000000000000000..308f56e62b1a67a41e1e7e5013b4065174184b44 --- /dev/null +++ b/ponytail/commands/ponytail.toml @@ -0,0 +1,2 @@ +description = "Switch ponytail intensity level (lite/full/ultra/off)" +prompt = "Switch to ponytail {{args}} mode. If no level specified, use full. Lazy senior dev mode, before any code: does it need to exist at all (YAGNI)? Does the standard library do it? A native platform feature? Can it be one line? Build the minimum that works. No unrequested abstractions, no avoidable dependencies, no boilerplate. Mark deliberate simplifications that cut a real corner with a known ceiling using a ponytail: comment that names the ceiling and upgrade path." diff --git a/ponytail/docs/agent-portability.md b/ponytail/docs/agent-portability.md new file mode 100644 index 0000000000000000000000000000000000000000..08adfcdf6da94af4a9aefbf4e86adaced5a3e82f --- /dev/null +++ b/ponytail/docs/agent-portability.md @@ -0,0 +1,49 @@ +# Agent Portability + +Ponytail is an agent-portable skill distribution. The skills in `skills/` hold +the core behavior; host-specific files are adapters that make that behavior easy +to load in a given agent. + +## Supported Adapters + +| Host | Files | Notes | +|------|-------|-------| +| Claude Code | `.claude-plugin/plugin.json`, `commands/`, `hooks/claude-codex-hooks.json`, `hooks/` | Full plugin install with session activation, mode tracking, commands, and statusline support. | +| Codex | `.codex-plugin/plugin.json`, `hooks/claude-codex-hooks.json`, `hooks/`, `skills/` | Plugin install with the same skills plus lifecycle hooks for activation and mode tracking. | +| Grok Build | root `plugin.json`, `.grok-plugin/marketplace.json`, `skills/`, `commands/` | `grok plugin install DietrichGebert/ponytail --trust`, then enable. Grok can auto-invoke ponytail from its coding-task skill description; `/ponytail` makes activation explicit. Grok lifecycle hooks are not used because passive hook output cannot inject instructions. | +| OpenCode | `.opencode/plugins/ponytail.mjs`, `.opencode/command/`, `hooks/`, `skills/` | Server plugin injects the ruleset each turn via `experimental.chat.system.transform` and persists `/ponytail` switches; reuses the shared instruction builder. | +| pi | `pi-extension/`, `skills/`, `hooks/` | Package extension: injects the ruleset each turn through the shared instruction builder and registers the `/ponytail` commands. | +| Hermes Agent | `plugin.yaml`, `__init__.py`, `skills/` | Native Hermes plugin: injects active mode through `pre_llm_call`, rewrites gateway `/ponytail-*` skill commands into agent prompts, registers `/ponytail` mode switching, and exposes bundled skills as `ponytail:`. | +| Gemini CLI | `gemini-extension.json`, `AGENTS.md`, `commands/`, `skills/` | Extension manifest points `contextFileName` at `AGENTS.md` for always-on rules, and reuses the existing `commands/*.toml` and `skills/`, which Gemini CLI auto-discovers. The Claude/Codex hook map is not placed at Gemini's auto-discovered `hooks/hooks.json` path. | +| Cursor | `.cursor/rules/ponytail.mdc` | Always-on project rule. | +| Windsurf | `.windsurf/rules/ponytail.md` | Project rule. | +| Cline | `.clinerules/ponytail.md` | Project rule. | +| GitHub Copilot | `.github/copilot-instructions.md` | Repository instruction file. | +| GitHub Copilot CLI | `.github/plugin/`, `AGENTS.md`, `.github/copilot-instructions.md`, `~/.copilot/copilot-instructions.md` | Plugin-supported (`copilot plugin marketplace add DietrichGebert/ponytail` + `copilot plugin install ponytail@ponytail`). Fallback instruction mode remains: per-project from `AGENTS.md` or `.github/copilot-instructions.md`, or globally from `~/.copilot/copilot-instructions.md` (instruction-tier, no `/ponytail` levels or hooks). | +| Antigravity | `AGENTS.md` | Reads `AGENTS.md` at the repo root as always-on rules (like `.cursorrules`/`CLAUDE.md`); `.agents/rules/` also works for workspace rules. Instruction-tier. | +| CodeWhale | `AGENTS.md` | Reads `AGENTS.md` from the repo root as project instructions; also reads `CLAUDE.md` and `.claude/instructions.md` as fallbacks. Instruction-tier. | +| Swival | `.swival/skills/`, `AGENTS.md` | `swival skills add https://github.com/DietrichGebert/ponytail` installs the six skills straight into `.swival/skills/`. Add `--global` to stage them in the library (`~/.config/swival/library`) first, then `swival skills add ponytail` (or `--global ponytail`) to activate per-project or everywhere. Also reads `AGENTS.md` from the repo root and `~/.config/swival/AGENTS.md` globally as instruction-tier fallback. | +| VS Code + Codex extension | `AGENTS.md` | The Codex extension reads `AGENTS.md` (repo root, or `~/.codex/AGENTS.md` globally). Instruction-tier; the full Codex plugin row above adds `/ponytail` levels and hooks. | +| JetBrains Junie | `AGENTS.md` | Junie reads `AGENTS.md` once you point it there in Settings → Tools → Junie → Project Settings → Guidelines Path (not automatic yet); this repo ships `AGENTS.md`, and `.junie/guidelines.md` is Junie's legacy path. Instruction-tier. | +| Amp (Sourcegraph) | `AGENTS.md` | Amp reads `AGENTS.md` from the working directory and parent directories up to `$HOME` (plus global config like `~/.config/amp/AGENTS.md`); falls back to `AGENT.md`/`CLAUDE.md`. Instruction-tier. | +| Jules (Google) | `AGENTS.md` | Jules automatically reads `AGENTS.md` from the repository root. Instruction-tier. | +| Kiro | `.kiro/steering/ponytail.md` | Steering rule; copy globally or into a project. | +| Qoder | `.qoder/rules/ponytail.md`, `.qoder-plugin/plugin.json`, `hooks/qoder-hooks.json`, `skills/`, `AGENTS.md` | Qoder auto-loads `AGENTS.md` as always-on context; `.qoder/rules/ponytail.md` provides per-project rules; the plugin manifest points at `skills/` for the six ponytail skills (invoked as `/ponytail`, `/ponytail-review`, etc. via the Skill system). Full plugin-tier: `hooks/qoder-hooks.json` template registers `UserPromptSubmit` (mode activation + ruleset injection) and `PreToolUse` with `task|Task` matcher (subagent injection). Instruction-tier works from repo root with zero setup via `AGENTS.md`. | +| Zed | `AGENTS.md` | Auto-includes `AGENTS.md` from the worktree root as one of its default rule files for the Agent Panel. Instruction-tier. | +| Generic agents | `AGENTS.md` or `skills/*/SKILL.md` | Copy the compact rule file or load the skill files directly. | + +## Adapter Rule + +Keep adapters thin. When a host supports skills or hooks, point it at the +existing `skills/` and `hooks/` files. When a host only supports project +instructions, keep its copied rule text aligned with `AGENTS.md`. + +## Portable Behavior + +- `skills/ponytail/SKILL.md`: lazy senior dev mode +- `skills/ponytail-review/SKILL.md`: over-engineering review +- `skills/ponytail-audit/SKILL.md`: whole-repo over-engineering audit +- `skills/ponytail-debt/SKILL.md`: harvest `ponytail:` shortcuts into a tracked ledger +- `skills/ponytail-gain/SKILL.md`: measured-impact scoreboard from the benchmark +- `skills/ponytail-help/SKILL.md`: quick reference +- `AGENTS.md`: compact always-on instruction set for agents without skill support diff --git a/ponytail/docs/platform-native.md b/ponytail/docs/platform-native.md new file mode 100644 index 0000000000000000000000000000000000000000..11e7d6e11ee3b0ae0533a938cdb5dd0fa3ac9017 --- /dev/null +++ b/ponytail/docs/platform-native.md @@ -0,0 +1,211 @@ +# Platform-Native Solutions + +The lazy senior dev's first question is always: *does the platform already do this?* + +This document answers that question for the most common cases. Before reaching for a package, scan here. The platform ships with your app for free, doesn't break on updates, and was written by people whose job is exactly that problem. + +--- + +## HTML Elements + +Things the browser already has as a form control. + +| You think you need | What the platform has | +|---|---| +| Date picker library | `` | +| Time picker library | `` | +| Color picker library | `` | +| Range slider library | `` | +| Progress bar component | `` | +| Meter/gauge component | `` | +| Modal/dialog library | `` + `dialog.showModal()` | +| Accordion/FAQ component | `
Title…
` | +| Tooltip library | `title` attribute + CSS `::before`/`::after` | +| Searchable dropdown | ` ` | +| Auto-growing textarea | `field-sizing: content` (CSS) | +| Sticky header | `position: sticky; top: 0` (CSS) | + +--- + +## CSS Capabilities + +Things developers reach for JavaScript to do. + +| You think you need JS for | What CSS has | +|---|---| +| Responsive font size | `font-size: clamp(1rem, 2.5vw, 2rem)` | +| Fluid spacing | `padding: clamp(1rem, 5vw, 3rem)` | +| Dark mode | `@media (prefers-color-scheme: dark)` | +| Reduced motion | `@media (prefers-reduced-motion: reduce)` | +| Responsive layout without breakpoints | `grid-template-columns: repeat(auto-fill, minmax(250px, 1fr))` | +| Component-level responsive design | `@container` queries | +| Global design tokens / theming | CSS custom properties (`--color-primary: #7c3aed`) | +| Smooth scroll | `scroll-behavior: smooth` | +| Scroll-snap carousel | `scroll-snap-type: x mandatory` + `scroll-snap-align: start` | +| Aspect ratio enforcement | `aspect-ratio: 16 / 9` | +| Truncate text with ellipsis | `overflow: hidden; text-overflow: ellipsis; white-space: nowrap` | +| Multi-line text clamp | `-webkit-line-clamp: 3` | +| CSS cascade layers (style isolation) | `@layer base, components, utilities` | +| Nested CSS selectors | Native CSS nesting (no preprocessor needed) | +| `has()` parent selector | `:has(input:checked)` | + +--- + +## JavaScript / Browser APIs + +Libraries people install that the runtime already ships. + +| You think you need | What the platform has | +|---|---| +| `query-string` / `qs` | `new URLSearchParams(location.search)` | +| `lodash.clonedeep` | `structuredClone(obj)` | +| `lodash.groupby` | `Object.groupBy(arr, fn)` | +| `lodash.debounce` | see debounce one-liner below | +| `numeral` / `accounting` | `new Intl.NumberFormat("en-US", { style: "currency", currency: "USD" })` | +| `date-fns` format | `new Intl.DateTimeFormat("en-US", { dateStyle: "long" }).format(date)` | +| `date-fns` relative time | `new Intl.RelativeTimeFormat("en", { numeric: "auto" }).format(-3, "day")` | +| `plural` / `i18n` plurals | `new Intl.PluralRules("en-US").select(count)` | +| `clipboard.js` | `navigator.clipboard.writeText(text)` | +| `uuid` (v4) | `crypto.randomUUID()` | +| Infinite scroll library | `new IntersectionObserver(cb).observe(sentinel)` | +| Resize listener library | `new ResizeObserver(cb).observe(element)` | +| DOM mutation watcher | `new MutationObserver(cb).observe(el, options)` | +| `uuid-validate` | `/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(id)` | +| `is-online` / `connectivity check` | `navigator.onLine` + `online`/`offline` events | +| `sharesheet` library | `navigator.share({ title, text, url })` | +| `store.js` / `localForage` (simple case) | `localStorage.setItem(key, JSON.stringify(val))` | +| Abort fetch on timeout | `AbortSignal.timeout(5000)` passed to `fetch` | +| Custom event bus | `new EventTarget()` / `dispatchEvent(new CustomEvent("x", { detail }))` | + +**Debounce one-liner** (no library): +```js +// ponytail: 3 lines beats a dependency +let t; +const debounce = (fn, ms) => (...args) => { clearTimeout(t); t = setTimeout(() => fn(...args), ms); }; +``` + +--- + +## Swift / SwiftUI + +UI components people reach for a library or a custom view for. + +| You think you need | What the platform has | +|---|---| +| Date/time picker library | `DatePicker` | +| Color picker library | `ColorPicker` | +| Search bar + filtering | `.searchable(text:)` | +| Pull-to-refresh library | `.refreshable { }` | +| Swipe-to-delete / row actions | `.swipeActions { }` | +| Async image loading + cache | `AsyncImage` | +| Charting library | Swift Charts (`import Charts`) | +| Markdown rendering | `Text(...)` markdown / `AttributedString(markdown:)` | +| Share sheet wrapper | `ShareLink` | +| Loading spinner | `ProgressView()` | +| Photo picker | `PhotosPicker` | +| Map SDK (basic) | `Map` (MapKit for SwiftUI) | +| Grid layout library | `Grid` / `LazyVGrid` | + +Frameworks and stdlib that wrappers wrap. + +| You think you need | What the platform has | +|---|---| +| JSON library (SwiftyJSON) | `Codable` + `JSONDecoder` / `JSONEncoder` | +| HTTP client (Alamofire, simple use) | `URLSession` async/await; Alamofire earns it for complex retry/multipart at scale | +| Date/number/currency formatting | `.formatted()` / `FormatStyle` | +| Regex library | Swift regex literals + `Regex` | +| Crypto library (CryptoSwift) | `CryptoKit` | +| Keychain wrapper | Security `SecItem`; a few lines, not a dependency | +| Persistence / ORM | `SwiftData`, or `@AppStorage` for small key-values | +| Logging library | `Logger` (`os.log`) | +| UUID / Base64 helpers | `UUID()`, `Data(...).base64EncodedString()` | +| Image downsampling | ImageIO `CGImageSourceCreateThumbnailAtIndex` | +| Combine wrappers for async | async/await + `AsyncSequence` | + +--- + +## Node.js Standard Library + +Packages that wrap Node built-ins. + +| You think you need | What Node has | +|---|---| +| `mkdirp` | `fs.mkdirSync(path, { recursive: true })` | +| `rimraf` | `fs.rmSync(path, { recursive: true, force: true })` | +| `make-dir` | `fs.mkdirSync(path, { recursive: true })` | +| `slash` (win paths) | `path.posix` or `path.normalize()` | +| `uuid` (v4) | `crypto.randomUUID()` | +| `ms` (parse duration strings) | keep `ms`, it's genuinely useful and tiny | +| `is-stream` | `val instanceof stream.Readable` | +| `object-assign` | `Object.assign()` / spread | +| `array-uniq` | `[...new Set(arr)]` | +| `array-flatten` | `arr.flat(Infinity)` | +| `flat` | `arr.flat(depth)` | +| `path-exists` | `fs.existsSync(path)` | +| `load-json-file` | `JSON.parse(fs.readFileSync(path, "utf8"))` | +| `write-json-file` | `fs.writeFileSync(path, JSON.stringify(obj, null, 2))` | +| `pkg-dir` | `path.resolve(__dirname, "..")` / `import.meta.dirname` | + +--- + +## Python Standard Library + +Packages that wrap what Python already ships. + +| You think you need | What Python has | +|---|---| +| `python-dateutil` (basic parsing) | `datetime.fromisoformat()` (Python 3.7+) | +| `pytz` | `zoneinfo.ZoneInfo("America/New_York")` (Python 3.9+) | +| `attrs` (simple data classes) | `@dataclass` | +| `six` | drop it, Python 2 is gone | +| `pathlib2` | `pathlib.Path` (built-in since Python 3.4) | +| `enum34` | `enum.Enum` (built-in since Python 3.4) | +| `typing_extensions` (common types) | `from __future__ import annotations` + built-in generics | +| `simplejson` (basic use) | `json` (stdlib) | +| `requests` (simple GET) | `urllib.request.urlopen(url)`, `requests` for anything real | +| `click` (single command) | `argparse` (stdlib) | +| `mergedeep` | `dict \| other_dict` (Python 3.9+) | +| `more-itertools` (basic) | `itertools` (stdlib): `chain`, `islice`, `groupby`, `product` | +| `toolz` (basic) | `functools`: `lru_cache`, `partial`, `reduce` | +| `tabulate` (dev/debug only) | `pprint.pprint()` for quick inspection | + +--- + +## Database + +Things the application layer implements that the database already does. + +| You think you need app code for | What the database has | +|---|---| +| Pagination offset/limit | `LIMIT 20 OFFSET 40` | +| Running totals | `SUM(...) OVER (ORDER BY date)` (window function) | +| Rank within group | `RANK() OVER (PARTITION BY category ORDER BY score DESC)` | +| Pivot / cross-tab | `FILTER (WHERE ...)` + conditional aggregation | +| Deduplication | `SELECT DISTINCT` / `ON CONFLICT DO NOTHING` | +| Soft-delete filtering | Generated column + partial index | +| Tree traversal | Recursive CTE (`WITH RECURSIVE`) | +| Full-text search (basic) | `tsvector` / `MATCH AGAINST` / `FTS5` | +| JSON storage + query | `jsonb` (Postgres) / `JSON_EXTRACT` (SQLite/MySQL) | +| UUID generation | `gen_random_uuid()` (Postgres) / `UUID()` (MySQL) | +| Timestamps on insert/update | `DEFAULT now()` + trigger or `ON UPDATE CURRENT_TIMESTAMP` | +| Enforce uniqueness | `UNIQUE` constraint, not application-level checks | +| Enforce referential integrity | `FOREIGN KEY`, not application-level checks | +| Enforce value ranges | `CHECK (price > 0)`, not application-level validation | + +--- + +## The Pattern + +Across every layer, the pattern is the same: + +``` +Platform team spends years solving the problem. +Package author wraps it. +You install the wrapper. +The wrapper goes unmaintained. +You debug the wrapper. +``` + +Skip the wrapper. The platform ships with your app for free. + +When the native solution is genuinely insufficient (old browser support, edge cases it doesn't handle, ergonomics that matter at scale), the library earns its place. Install it then, not before. diff --git a/ponytail/examples/README.md b/ponytail/examples/README.md new file mode 100644 index 0000000000000000000000000000000000000000..dc4524fa0f278e9ac1478ad1b20156de1c7a76ca --- /dev/null +++ b/ponytail/examples/README.md @@ -0,0 +1,17 @@ +# Examples + +Real model output, verbatim from benchmark runs, the same task answered by the same model +with no skill (`## Without Ponytail`) and with ponytail (`## With Ponytail`), so you can +compare side by side. Model: Claude Haiku 4.5, temperature 1, source `benchmarks/output.json`. + +These are not hand-written. Reproduce them yourself: +`npx promptfoo@latest eval -c benchmarks/promptfooconfig.yaml`. Method, all three models, and +median-of-10 numbers: [../benchmarks/](../benchmarks/). + +| Example | Without (LOC) | With (LOC) | +|---|--:|--:| +| [Email Validation](email-validation.md) | 75 | 3 | +| [Debounce](debounce.md) | 116 | 10 | +| [CSV Sum](csv-sum.md) | 20 | 3 | +| [Countdown Timer](react-countdown.md) | 267 | 9 | +| [Rate Limiting](rate-limit.md) | 128 | 10 | diff --git a/ponytail/examples/csv-sum.md b/ponytail/examples/csv-sum.md new file mode 100644 index 0000000000000000000000000000000000000000..99da7d8f5fdff36285bf6f5ad294983a6cc69510 --- /dev/null +++ b/ponytail/examples/csv-sum.md @@ -0,0 +1,71 @@ +# CSV Sum + +**Task:** "Write Python code that reads sales.csv and sums the 'amount' column." + +Verbatim model output from a benchmark run, Claude Haiku 4.5, no-skill arm vs ponytail arm, temperature 1, source `benchmarks/output.json`. Reproduce: `npx promptfoo@latest eval -c benchmarks/promptfooconfig.yaml`. + +## Without Ponytail, 20 lines of code + +# Python code to read sales.csv and sum the 'amount' column + +```python +import pandas as pd + +# Read the CSV file +df = pd.read_csv('sales.csv') + +# Sum the 'amount' column +total_amount = df['amount'].sum() + +# Print the result +print(f"Total amount: ${total_amount:,.2f}") +``` + +## Alternative methods: + +**Using CSV module (without pandas):** +```python +import csv + +total_amount = 0 + +with open('sales.csv', 'r') as file: + reader = csv.DictReader(file) + for row in reader: + total_amount += float(row['amount']) + +print(f"Total amount: ${total_amount:,.2f}") +``` + +**Using pandas with error handling:** +```python +import pandas as pd + +try: + df = pd.read_csv('sales.csv') + total_amount = df['amount'].sum() + print(f"Total amount: ${total_amount:,.2f}") +except FileNotFoundError: + print("Error: sales.csv not found") +except KeyError: + print("Error: 'amount' column not found") +``` + +**The pandas method is recommended** because it's: +- More concise and readable +- Handles data types automatically +- Provides built-in error handling +- Efficient for large files + +## With Ponytail, 3 lines of code + +```python +import csv + +total = sum(float(row['amount']) for row in csv.DictReader(open('sales.csv'))) +print(total) +``` + +Skipped: pandas, error handling, file closing, add when the CSV is large, malformed, or you need more analysis. + +**20 → 3 lines of code**, same model, same prompt. diff --git a/ponytail/examples/debounce.md b/ponytail/examples/debounce.md new file mode 100644 index 0000000000000000000000000000000000000000..371fc33fb601d36804db86fa9261f26e35c4a0fb --- /dev/null +++ b/ponytail/examples/debounce.md @@ -0,0 +1,211 @@ +# Debounce + +**Task:** "Add debounce to a search input in vanilla JavaScript. It currently fires an API call on every keystroke." + +Verbatim model output from a benchmark run, Claude Haiku 4.5, no-skill arm vs ponytail arm, temperature 1, source `benchmarks/output.json`. Reproduce: `npx promptfoo@latest eval -c benchmarks/promptfooconfig.yaml`. + +## Without Ponytail, 116 lines of code + +# Debounce Search Input + +Here's a practical implementation to prevent excessive API calls: + +## Basic Debounce Function + +```javascript +function debounce(func, delay) { + let timeoutId; + return function (...args) { + clearTimeout(timeoutId); + timeoutId = setTimeout(() => func(...args), delay); + }; +} + +// Usage +const searchInput = document.getElementById('search'); +const resultsContainer = document.getElementById('results'); + +const handleSearch = async (query) => { + if (!query.trim()) { + resultsContainer.innerHTML = ''; + return; + } + + try { + const response = await fetch(`/api/search?q=${encodeURIComponent(query)}`); + const data = await response.json(); + displayResults(data); + } catch (error) { + console.error('Search failed:', error); + } +}; + +// Debounce the search function (300ms delay) +const debouncedSearch = debounce(handleSearch, 300); + +searchInput.addEventListener('input', (e) => { + debouncedSearch(e.target.value); +}); +``` + +## Enhanced Version with Loading State + +```javascript +const searchInput = document.getElementById('search'); +const resultsContainer = document.getElementById('results'); +const loadingIndicator = document.getElementById('loading'); + +const handleSearch = async (query) => { + if (!query.trim()) { + resultsContainer.innerHTML = ''; + loadingIndicator.style.display = 'none'; + return; + } + + loadingIndicator.style.display = 'block'; + + try { + const response = await fetch(`/api/search?q=${encodeURIComponent(query)}`); + if (!response.ok) throw new Error('Search failed'); + + const data = await response.json(); + displayResults(data); + } catch (error) { + resultsContainer.innerHTML = '

Search failed. Try again.

'; + console.error('Search error:', error); + } finally { + loadingIndicator.style.display = 'none'; + } +}; + +const debouncedSearch = debounce(handleSearch, 300); + +searchInput.addEventListener('input', (e) => { + debouncedSearch(e.target.value); +}); + +function displayResults(data) { + resultsContainer.innerHTML = data + .map(item => `
${item.title}
`) + .join(''); +} +``` + +## Advanced: Debounce with Cancel & Immediate Options + +```javascript +function debounce(func, delay, options = {}) { + let timeoutId; + let lastArgs; + + const debounced = function (...args) { + lastArgs = args; + clearTimeout(timeoutId); + + if (options.immediate && !timeoutId) { + func(...args); + } + + timeoutId = setTimeout(() => { + if (!options.immediate) { + func(...lastArgs); + } + timeoutId = null; + }, delay); + }; + + // Allow manual cancellation + debounced.cancel = () => clearTimeout(timeoutId); + + return debounced; +} + +// Usage with options +const debouncedSearch = debounce(handleSearch, 300, { immediate: false }); + +// Cancel pending search if needed +searchInput.addEventListener('blur', () => { + debouncedSearch.cancel(); +}); +``` + +## HTML Example + +```html +
+ + +
+
+ + +``` + +## Key Benefits + +| Feature | Benefit | +|---------|---------| +| **Debounce** | Reduces API calls from 100+ to 1-2 per search | +| **300ms delay** | Balances responsiveness with performance | +| **Loading state** | Provides user feedback | +| **Error handling** | Graceful failure management | +| **Cancel method** | Cleanup when needed | + +**Typical keystroke reduction:** 10 characters = 10 API calls → 1-2 API calls with debounce ✅ + +## With Ponytail, 10 lines of code + +```javascript +const searchInput = document.querySelector('input[type="search"]'); +let debounceTimer; + +searchInput.addEventListener('input', (e) => { + clearTimeout(debounceTimer); + debounceTimer = setTimeout(() => { + fetch(`/api/search?q=${encodeURIComponent(e.target.value)}`) + .then(r => r.json()) + .then(data => console.log(data)); // replace with your render + }, 300); +}); +``` + +Skipped: debounce utility function, class wrapper, config object, `setTimeout` + `clearTimeout` is the debounce. Add a utility when you need it on 3+ inputs. + +**116 → 10 lines of code**, same model, same prompt. diff --git a/ponytail/examples/deep-clone.md b/ponytail/examples/deep-clone.md new file mode 100644 index 0000000000000000000000000000000000000000..f9b7d3f1b2e5ae73b5ce49475d0c24d0ce7b3908 --- /dev/null +++ b/ponytail/examples/deep-clone.md @@ -0,0 +1,31 @@ +# Deep Clone + +**Task:** "Deep clone this object." + +## Without Ponytail + +```bash +npm install lodash +``` + +```js +import { cloneDeep } from "lodash"; + +const copy = cloneDeep(original); +``` + +Or the classic hack: + +```js +// fragile: loses Date, undefined, Map, Set, circular refs, functions +const copy = JSON.parse(JSON.stringify(original)); +``` + +## With Ponytail + +```js +// ponytail: structuredClone does this +const copy = structuredClone(original); +``` + +**1 dependency (or a fragile hack) → 1 built-in.** `structuredClone` handles `Date`, `Map`, `Set`, `ArrayBuffer`, `RegExp`, circular references, and more, everything `JSON.parse/stringify` silently drops. Available in every browser since 2022 and Node.js since v17. Pull lodash in when you need the rest of it, not for one function. diff --git a/ponytail/examples/email-validation.md b/ponytail/examples/email-validation.md new file mode 100644 index 0000000000000000000000000000000000000000..43b05bb403b5b05096c3c64f3027181819c35f26 --- /dev/null +++ b/ponytail/examples/email-validation.md @@ -0,0 +1,156 @@ +# Email Validation + +**Task:** "Write me a Python function that validates email addresses." + +Verbatim model output from a benchmark run, Claude Haiku 4.5, no-skill arm vs ponytail arm, temperature 1, source `benchmarks/output.json`. Reproduce: `npx promptfoo@latest eval -c benchmarks/promptfooconfig.yaml`. + +## Without Ponytail, 75 lines of code + +# Email Validation Function + +Here's a practical Python function that validates email addresses: + +```python +import re + +def validate_email(email: str) -> bool: + """ + Validates an email address using a regular expression. + + Args: + email (str): The email address to validate + + Returns: + bool: True if valid, False otherwise + """ + # Regular expression pattern for email validation + pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$' + + return re.match(pattern, email) is not None + + +# Test cases +if __name__ == "__main__": + test_emails = [ + "user@example.com", # Valid + "john.doe@company.co.uk", # Valid + "test+tag@domain.org", # Valid + "invalid.email@", # Invalid + "missing@domain", # Invalid + "@nodomain.com", # Invalid + "spaces in@email.com", # Invalid + "user@domain..com", # Invalid (double dot) + ] + + for email in test_emails: + result = validate_email(email) + print(f"{email:<30} -> {result}") +``` + +## More Robust Version (with additional checks) + +```python +import re + +def validate_email_advanced(email: str) -> tuple[bool, str]: + """ + Advanced email validation with detailed feedback. + + Args: + email (str): The email address to validate + + Returns: + tuple: (is_valid, message) + """ + # Basic checks + if not email or not isinstance(email, str): + return False, "Email must be a non-empty string" + + email = email.strip() + + if len(email) > 254: + return False, "Email is too long (max 254 characters)" + + # Regular expression pattern + pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$' + + if not re.match(pattern, email): + return False, "Email format is invalid" + + # Additional checks + local_part, domain = email.rsplit('@', 1) + + if len(local_part) > 64: + return False, "Local part is too long (max 64 characters)" + + if local_part.startswith('.') or local_part.endswith('.'): + return False, "Local part cannot start or end with a dot" + + if '..' in email: + return False, "Email cannot contain consecutive dots" + + return True, "Email is valid" + + +# Test the advanced version +if __name__ == "__main__": + test_emails = [ + "user@example.com", + "invalid..email@domain.com", + ".invalid@domain.com", + "user@domain", + ] + + for email in test_emails: + is_valid, message = validate_email_advanced(email) + print(f"{email:<35} -> {is_valid:5} ({message})") +``` + +## Using a Third-Party Library (Recommended for Production) + +```python +from email_validator import validate_email, EmailNotValidError + +def validate_email_production(email: str) -> bool: + """ + Validates email using the email_validator library. + More reliable for production use. + + Install with: pip install email-validator + """ + try: + # Validate and normalize the email + valid = validate_email(email) + return True + except EmailNotValidError: + return False + + +# Usage +if __name__ == "__main__": + print(validate_email_production("user@example.com")) # True + print(validate_email_production("invalid@")) # False +``` + +## Comparison + +| Method | Pros | Cons | +|--------|------|------| +| **Regex (Simple)** | Fast, no dependencies | Limited validation | +| **Regex (Advanced)** | More thorough | Still not RFC-compliant | +| **email-validator** | RFC-compliant, reliable | External dependency | + +**Recommendation**: Use the `email-validator` library for production applications, as it properly handles RFC 5321/5322 standards. Use the simple regex version for quick scripts or when you want to avoid dependencies. + +## With Ponytail, 3 lines of code + +```python +import re + +def is_valid_email(email: str) -> bool: + return bool(re.match(r'^[^@]+@[^@]+\.[^@]+$', email)) +``` + +Skipped: RFC 5322 parser, DNS MX lookup, confirmation email. Add when you actually need to reject `user+tag@sub.domain.co.uk` or catch typos, until then, this catches 99% of "oops I fat-fingered it" cases. + +**75 → 3 lines of code**, same model, same prompt. diff --git a/ponytail/examples/group-by.md b/ponytail/examples/group-by.md new file mode 100644 index 0000000000000000000000000000000000000000..90d014a27b004761f5f27f94135c88ee7f0409a9 --- /dev/null +++ b/ponytail/examples/group-by.md @@ -0,0 +1,35 @@ +# Group By + +**Task:** "Group this array of objects by a key." + +## Without Ponytail + +```bash +npm install lodash +``` + +```js +import { groupBy } from "lodash"; + +const byStatus = groupBy(orders, "status"); +// → { pending: [...], shipped: [...], delivered: [...] } +``` + +Or the hand-rolled version: + +```js +const byStatus = orders.reduce((acc, order) => { + (acc[order.status] ??= []).push(order); + return acc; +}, {}); +``` + +## With Ponytail + +```js +// ponytail: Object.groupBy does this +const byStatus = Object.groupBy(orders, order => order.status); +// → { pending: [...], shipped: [...], delivered: [...] } +``` + +**1 dependency (or a reduce) → 1 built-in.** `Object.groupBy` shipped in Chrome 117, Firefox 119, Safari 17.4, Node.js 21. If you need a `Map` instead of a plain object: `Map.groupBy(orders, o => o.status)`. Check your target runtime; if you need IE11 or old Node, the `reduce` one-liner is still the right call, not lodash. diff --git a/ponytail/examples/infinite-scroll.md b/ponytail/examples/infinite-scroll.md new file mode 100644 index 0000000000000000000000000000000000000000..039b12918e4cd8740f77a77c1383a2ab7a95d843 --- /dev/null +++ b/ponytail/examples/infinite-scroll.md @@ -0,0 +1,58 @@ +# Infinite Scroll + +**Task:** "Load more items when the user scrolls to the bottom." + +## Without Ponytail + +```bash +npm install react-infinite-scroll-component +``` + +```jsx +import InfiniteScroll from "react-infinite-scroll-component"; + +export function Feed({ items, fetchMore, hasMore }) { + return ( + } + endMessage={

No more items

} + scrollThreshold={0.9} + > + {items.map(item => )} + + ); +} +``` + +A dependency to watch scroll position and fire a callback. + +## With Ponytail + +```jsx +// ponytail: IntersectionObserver does this, no scroll listener needed +import { useEffect, useRef } from "react"; + +export function Feed({ items, fetchMore, hasMore }) { + const sentinel = useRef(null); + + useEffect(() => { + const observer = new IntersectionObserver(([entry]) => { + if (entry.isIntersecting && hasMore) fetchMore(); + }); + if (sentinel.current) observer.observe(sentinel.current); + return () => observer.disconnect(); + }, [hasMore, fetchMore]); + + return ( + <> + {items.map(item => )} +
+ + ); +} +``` + +**1 dependency → 0 dependencies.** `IntersectionObserver` fires only when the sentinel enters the viewport, no scroll event, no throttling, no jank. Ships in every browser. The library wraps exactly this API. diff --git a/ponytail/examples/modal-dialog.md b/ponytail/examples/modal-dialog.md new file mode 100644 index 0000000000000000000000000000000000000000..db54e7cffee1ee32cf0a54b2c9fe715816d8c512 --- /dev/null +++ b/ponytail/examples/modal-dialog.md @@ -0,0 +1,62 @@ +# Modal Dialog + +**Task:** "Add a modal dialog for the delete confirmation." + +## Without Ponytail + +```bash +npm install @radix-ui/react-dialog +# or: npm install react-modal +``` + +```jsx +import * as Dialog from "@radix-ui/react-dialog"; +import { useState } from "react"; + +export function DeleteModal({ onConfirm, onCancel }) { + return ( + + + + + + + + Confirm deletion + This action cannot be undone. +
+ + + + +
+ + + + ); +} +``` + +A dependency, a portal, an overlay, a root, a trigger, a content wrapper, to show a box with two buttons. + +## With Ponytail + +```html + + +

This action cannot be undone.

+ + + +``` + +```js +const dialog = document.getElementById("confirm-delete"); +document.getElementById("cancel").onclick = () => dialog.close(); +document.getElementById("confirm").onclick = () => { onConfirm(); dialog.close(); }; + +// Open it: +dialog.showModal(); +``` + +**1 dependency + 30 lines → 0 dependencies + 8 lines.** The native `` traps focus automatically, closes on Escape, renders a backdrop via `::backdrop`, and is accessible by default. All browsers since 2022. The library was solving a problem the platform solved. diff --git a/ponytail/examples/number-formatting.md b/ponytail/examples/number-formatting.md new file mode 100644 index 0000000000000000000000000000000000000000..4773fa50a60ffd2c072531f7f75cafa8240e2259 --- /dev/null +++ b/ponytail/examples/number-formatting.md @@ -0,0 +1,37 @@ +# Number Formatting + +**Task:** "Format numbers as currency and with thousand separators." + +## Without Ponytail + +```bash +npm install numeral +# or: npm install accounting +``` + +```js +import numeral from "numeral"; + +numeral(1234567.89).format("$1,234.00"); // "$1,234,567.89" +numeral(0.745).format("0.0%"); // "74.5%" +numeral(1500).format("0.0a"); // "1.5k" +``` + +## With Ponytail + +```js +// ponytail: Intl.NumberFormat does this, locale-aware +new Intl.NumberFormat("en-US", { style: "currency", currency: "USD" }) + .format(1234567.89); +// → "$1,234,567.89" + +new Intl.NumberFormat("en-US", { style: "percent" }) + .format(0.745); +// → "74.5%" + +new Intl.NumberFormat("en-US", { notation: "compact" }) + .format(1500); +// → "1.5K" +``` + +**1 dependency → 0 dependencies.** `Intl.NumberFormat` is built into every JS runtime, handles every locale correctly, and gets currency symbols, decimal separators, and grouping right for any market without a lookup table. A library that hardcodes formats will always be wrong for someone. diff --git a/ponytail/examples/rate-limit.md b/ponytail/examples/rate-limit.md new file mode 100644 index 0000000000000000000000000000000000000000..e86cabe9690ec6f4cfa5b9f73e5809f5bf5b8e96 --- /dev/null +++ b/ponytail/examples/rate-limit.md @@ -0,0 +1,272 @@ +# Rate Limiting + +**Task:** "Add rate limiting to my FastAPI endpoint so users can't spam it." + +Verbatim model output from a benchmark run, Claude Haiku 4.5, no-skill arm vs ponytail arm, temperature 1, source `benchmarks/output.json`. Reproduce: `npx promptfoo@latest eval -c benchmarks/promptfooconfig.yaml`. + +## Without Ponytail, 128 lines of code + +# Rate Limiting in FastAPI + +Here are several approaches, from simplest to most robust: + +## 1. **Using `slowapi` (Recommended - Easiest)** + +```bash +pip install slowapi +``` + +```python +from fastapi import FastAPI, Request +from slowapi import Limiter +from slowapi.util import get_remote_address +from slowapi.errors import RateLimitExceeded +from fastapi.responses import JSONResponse + +app = FastAPI() + +limiter = Limiter(key_func=get_remote_address) +app.state.limiter = limiter +app.add_exception_handler(RateLimitExceeded, lambda request, exc: JSONResponse( + status_code=429, + content={"detail": "Rate limit exceeded"} +)) + +@app.get("/api/data") +@limiter.limit("5/minute") +async def get_data(request: Request): + return {"message": "Success"} +``` + +**Rate limit formats:** +- `"5/minute"` - 5 requests per minute +- `"100/hour"` - 100 requests per hour +- `"1000/day"` - 1000 requests per day + +--- + +## 2. **Using `limits` Library (More Control)** + +```bash +pip install limits +``` + +```python +from fastapi import FastAPI, HTTPException, Request +from limits import parse +from limits.storage import MemoryStorage +from limits.strategies import MovingWindowRateLimiter + +app = FastAPI() + +storage = MemoryStorage() +limiter = MovingWindowRateLimiter(storage) +rate_limit = parse("5/minute") + +@app.get("/api/data") +async def get_data(request: Request): + client_id = request.client.host + + if not limiter.hit(rate_limit, client_id): + raise HTTPException(status_code=429, detail="Rate limit exceeded") + + return {"message": "Success"} +``` + +--- + +## 3. **Custom Middleware (Full Control)** + +```python +from fastapi import FastAPI, HTTPException, Request +from datetime import datetime, timedelta +from collections import defaultdict +import asyncio + +app = FastAPI() + +# Store request timestamps per IP +request_history = defaultdict(list) +RATE_LIMIT = 5 # requests +TIME_WINDOW = 60 # seconds + +@app.middleware("http") +async def rate_limit_middleware(request: Request, call_next): + client_ip = request.client.host + now = datetime.now() + + # Clean old requests outside time window + request_history[client_ip] = [ + req_time for req_time in request_history[client_ip] + if now - req_time < timedelta(seconds=TIME_WINDOW) + ] + + # Check if limit exceeded + if len(request_history[client_ip]) >= RATE_LIMIT: + return JSONResponse( + status_code=429, + content={"detail": "Rate limit exceeded"} + ) + + # Record this request + request_history[client_ip].append(now) + + return await call_next(request) + +@app.get("/api/data") +async def get_data(): + return {"message": "Success"} +``` + +--- + +## 4. **Per-User Rate Limiting (With Authentication)** + +```python +from fastapi import FastAPI, Depends, HTTPException +from slowapi import Limiter +from slowapi.util import get_remote_address + +app = FastAPI() +limiter = Limiter(key_func=get_remote_address) + +def get_user_id(token: str = Header(None)) -> str: + # Your auth logic here + return token or "anonymous" + +@app.get("/api/data") +@limiter.limit("10/minute") +async def get_data(request: Request, user_id: str = Depends(get_user_id)): + return {"message": f"Success for {user_id}"} +``` + +--- + +## 5. **Redis-Based Rate Limiting (Production)** + +```bash +pip install slowapi redis +``` + +```python +from fastapi import FastAPI, Request +from slowapi import Limiter +from slowapi.util import get_remote_address +from slowapi.errors import RateLimitExceeded +from slowapi.storage import RedisStorage +from redis import Redis +from fastapi.responses import JSONResponse + +redis_client = Redis.from_url("redis://localhost:6379") +storage = RedisStorage(redis_client) +limiter = Limiter(key_func=get_remote_address, storage=storage) + +app = FastAPI() +app.state.limiter = limiter + +app.add_exception_handler( + RateLimitExceeded, + lambda request, exc: JSONResponse( + status_code=429, + content={"detail": "Rate limit exceeded"} + ) +) + +@app.get("/api/data") +@limiter.limit("5/minute") +async def get_data(request: Request): + return {"message": "Success"} +``` + +--- + +## 6. **Complete Example with Multiple Endpoints** + +```python +from fastapi import FastAPI, Request +from slowapi import Limiter +from slowapi.util import get_remote_address +from slowapi.errors import RateLimitExceeded +from fastapi.responses import JSONResponse + +app = FastAPI() +limiter = Limiter(key_func=get_remote_address) +app.state.limiter = limiter + +@app.exception_handler(RateLimitExceeded) +async def rate_limit_handler(request: Request, exc: RateLimitExceeded): + return JSONResponse( + status_code=429, + content={ + "detail": "Too many requests", + "retry_after": exc.detail + } + ) + +# Strict limit +@app.post("/api/login") +@limiter.limit("3/minute") +async def login(request: Request): + return {"token": "xxx"} + +# Moderate limit +@app.get("/api/data") +@limiter.limit("30/minute") +async def get_data(request: Request): + return {"data": []} + +# Generous limit +@app.get("/api/public") +@limiter.limit("100/minute") +async def get_public(request: Request): + return {"public": "data"} +``` + +--- + +## Comparison Table + +| Method | Pros | Cons | Best For | +|--------|------|------|----------| +| **slowapi** | Easy, decorator-based | In-memory only | Development, small apps | +| **limits** | Flexible, multiple backends | More setup | Custom logic | +| **Middleware** | Full control, no dependencies | Manual cleanup | Learning, simple cases | +| **Redis** | Distributed, scalable | Requires Redis | Production, multi-server | + +--- + +## Testing Rate Limits + +```python +import httpx + +async def test_rate_limit(): + async with httpx.AsyncClient(app=app, base_url="http://test") as client: + for i in range(6): + response = await client.get("/api/data") + print(f"Request {i+1}: {response.status_code}") + # First 5 should be 200, 6th should be 429 +``` + +**Recommendation:** Use `slowapi` for most cases, it's the most popular and easiest to implement! 🚀 + +## With Ponytail, 10 lines of code + +```python +from fastapi import FastAPI, HTTPException +from slowapi import Limiter +from slowapi.util import get_remote_address + +app = FastAPI() +limiter = Limiter(key_func=get_remote_address) +app.state.limiter = limiter + +@app.get("/api/endpoint") +@limiter.limit("10/minute") +async def my_endpoint(request): + return {"status": "ok"} +``` + +Skipped: custom rate limit logic, Redis, sliding windows, `slowapi` handles it. Add when: you need distributed rate limiting across multiple servers (swap `Limiter` for Redis backend) or per-user limits (add `key_func=lambda r: r.headers.get("authorization")`). + +**128 → 10 lines of code**, same model, same prompt. diff --git a/ponytail/examples/react-countdown.md b/ponytail/examples/react-countdown.md new file mode 100644 index 0000000000000000000000000000000000000000..8ad7ccfff6234750c6a785d837f40a84bd351b8c --- /dev/null +++ b/ponytail/examples/react-countdown.md @@ -0,0 +1,390 @@ +# Countdown Timer + +**Task:** "Build me a countdown timer component in React that counts down from a given number of seconds." + +Verbatim model output from a benchmark run, Claude Haiku 4.5, no-skill arm vs ponytail arm, temperature 1, source `benchmarks/output.json`. Reproduce: `npx promptfoo@latest eval -c benchmarks/promptfooconfig.yaml`. + +## Without Ponytail, 267 lines of code + +# React Countdown Timer Component + +Here's a complete countdown timer component with multiple variations: + +## Basic Countdown Timer + +```jsx +import { useState, useEffect } from 'react'; + +export function CountdownTimer({ initialSeconds = 60 }) { + const [seconds, setSeconds] = useState(initialSeconds); + const [isActive, setIsActive] = useState(false); + + useEffect(() => { + let interval = null; + + if (isActive && seconds > 0) { + interval = setInterval(() => { + setSeconds(seconds => seconds - 1); + }, 1000); + } else if (seconds === 0) { + setIsActive(false); + } + + return () => clearInterval(interval); + }, [isActive, seconds]); + + const toggle = () => setIsActive(!isActive); + const reset = () => { + setSeconds(initialSeconds); + setIsActive(false); + }; + + return ( +
+
{seconds}s
+
+ + +
+
+ ); +} +``` + +## Advanced Timer with Formatted Display + +```jsx +import { useState, useEffect } from 'react'; + +export function AdvancedCountdownTimer({ + initialSeconds = 3600, + onComplete = () => {}, + showMilliseconds = false +}) { + const [timeLeft, setTimeLeft] = useState(initialSeconds); + const [isActive, setIsActive] = useState(false); + + useEffect(() => { + let interval = null; + + if (isActive && timeLeft > 0) { + interval = setInterval(() => { + setTimeLeft(prev => { + if (prev <= 1) { + setIsActive(false); + onComplete(); + return 0; + } + return prev - 1; + }); + }, 1000); + } + + return () => clearInterval(interval); + }, [isActive, timeLeft, onComplete]); + + const formatTime = (totalSeconds) => { + const hours = Math.floor(totalSeconds / 3600); + const minutes = Math.floor((totalSeconds % 3600) / 60); + const seconds = totalSeconds % 60; + + const parts = []; + if (hours > 0) parts.push(String(hours).padStart(2, '0')); + parts.push(String(minutes).padStart(2, '0')); + parts.push(String(seconds).padStart(2, '0')); + + return parts.join(':'); + }; + + const toggle = () => setIsActive(!isActive); + const reset = () => { + setTimeLeft(initialSeconds); + setIsActive(false); + }; + + const progressPercent = ((initialSeconds - timeLeft) / initialSeconds) * 100; + + return ( +
+
+ {formatTime(timeLeft)} +
+
+
+
+
+ + +
+
+ ); +} +``` + +## Custom Hook Version + +```jsx +import { useState, useEffect, useCallback } from 'react'; + +export function useCountdown(initialSeconds) { + const [seconds, setSeconds] = useState(initialSeconds); + const [isActive, setIsActive] = useState(false); + + useEffect(() => { + let interval = null; + + if (isActive && seconds > 0) { + interval = setInterval(() => { + setSeconds(prev => prev - 1); + }, 1000); + } else if (seconds === 0 && isActive) { + setIsActive(false); + } + + return () => clearInterval(interval); + }, [isActive, seconds]); + + const start = useCallback(() => setIsActive(true), []); + const pause = useCallback(() => setIsActive(false), []); + const reset = useCallback(() => { + setSeconds(initialSeconds); + setIsActive(false); + }, [initialSeconds]); + + return { + seconds, + isActive, + start, + pause, + reset, + setSeconds + }; +} + +// Usage +export function TimerWithHook() { + const { seconds, isActive, start, pause, reset } = useCountdown(60); + + return ( +
+
{seconds}s
+ + +
+ ); +} +``` + +## Styled Component with Animations + +```jsx +import { useState, useEffect } from 'react'; +import styled from 'styled-components'; + +const TimerContainer = styled.div` + display: flex; + flex-direction: column; + align-items: center; + gap: 20px; + padding: 30px; + background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); + border-radius: 15px; + color: white; + font-family: 'Arial', sans-serif; +`; + +const Display = styled.div` + font-size: 48px; + font-weight: bold; + font-variant-numeric: tabular-nums; + animation: ${props => props.isWarning ? 'pulse 0.5s infinite' : 'none'}; + + @keyframes pulse { + 0%, 100% { opacity: 1; } + 50% { opacity: 0.5; } + } +`; + +const Controls = styled.div` + display: flex; + gap: 10px; +`; + +const Button = styled.button` + padding: 10px 20px; + font-size: 16px; + border: none; + border-radius: 8px; + cursor: pointer; + background-color: ${props => props.variant === 'danger' ? '#ff6b6b' : '#4ecdc4'}; + color: white; + transition: all 0.3s ease; + + &:hover { + transform: translateY(-2px); + box-shadow: 0 5px 15px rgba(0, 0, 0, 0.2); + } + + &:active { + transform: translateY(0); + } +`; + +export function StyledCountdownTimer({ initialSeconds = 60 }) { + const [seconds, setSeconds] = useState(initialSeconds); + const [isActive, setIsActive] = useState(false); + + useEffect(() => { + let interval = null; + + if (isActive && seconds > 0) { + interval = setInterval(() => { + setSeconds(prev => prev - 1); + }, 1000); + } else if (seconds === 0) { + setIsActive(false); + } + + return () => clearInterval(interval); + }, [isActive, seconds]); + + const isWarning = seconds <= 10 && seconds > 0; + + return ( + + {seconds}s + + + + + + ); +} +``` + +## CSS Styling + +```css +.countdown-timer { + display: flex; + flex-direction: column; + align-items: center; + gap: 20px; + padding: 20px; + border: 2px solid #333; + border-radius: 10px; + background-color: #f5f5f5; +} + +.timer-display { + font-size: 48px; + font-weight: bold; + font-variant-numeric: tabular-nums; + color: #333; +} + +.timer-controls { + display: flex; + gap: 10px; +} + +.timer-controls button { + padding: 10px 20px; + font-size: 16px; + border: none; + border-radius: 5px; + cursor: pointer; + background-color: #667eea; + color: white; + transition: background-color 0.3s; +} + +.timer-controls button:hover { + background-color: #5568d3; +} + +.progress-bar { + width: 100%; + height: 8px; + background-color: #e0e0e0; + border-radius: 4px; + overflow: hidden; +} + +.progress-fill { + height: 100%; + background-color: #667eea; + transition: width 0.3s linear; +} +``` + +## Usage Examples + +```jsx +// Basic usage + + +// Advanced with callback + alert('Time\'s up!')} +/> + +// Using the hook + + +// Styled version + +``` + +## Features + +✅ **Basic Timer** - Simple countdown with start/pause/reset +✅ **Advanced Timer** - Formatted display (HH:MM:SS), progress bar, callbacks +✅ **Custom Hook** - Reusable logic for any component +✅ **Styled Components** - Beautiful UI with animations +✅ **Warning State** - Visual feedback when time is running out +✅ **Completion Callback** - Execute code when timer finishes + +Choose the version that best fits your needs! + +## With Ponytail, 9 lines of code + +```jsx +export function CountdownTimer({ seconds }) { + const [remaining, setRemaining] = React.useState(seconds); + + React.useEffect(() => { + if (remaining <= 0) return; + const timer = setInterval(() => setRemaining(r => r - 1), 1000); + return () => clearInterval(timer); + }, [remaining]); + + return
{remaining}s
; +} +``` + +Skipped: pause/resume, formatted display (mm:ss), sound on zero, styling, add when needed. + +**267 → 9 lines of code**, same model, same prompt. diff --git a/ponytail/examples/url-params.md b/ponytail/examples/url-params.md new file mode 100644 index 0000000000000000000000000000000000000000..e2ed5549cb2a2d91d61a480b73b4a3e35a49196c --- /dev/null +++ b/ponytail/examples/url-params.md @@ -0,0 +1,41 @@ +# URL Parameters + +**Task:** "Parse and build URL query strings." + +## Without Ponytail + +```bash +npm install query-string +# 4.5 kB gzipped, 3.5M downloads/week +``` + +```js +import qs from "query-string"; + +// Parse +const params = qs.parse(location.search); +// → { page: "2", sort: "name", tags: ["js", "css"] } + +// Build +const url = qs.stringify({ page: 2, sort: "name", tags: ["js", "css"] }); +// → "page=2&sort=name&tags=js&tags=css" +``` + +## With Ponytail + +```js +// ponytail: URLSearchParams does this +const params = new URLSearchParams(location.search); + +// Read +params.get("page"); // "2" +params.getAll("tags"); // ["js", "css"] + +// Build +const out = new URLSearchParams({ page: 2, sort: "name" }); +out.append("tags", "js"); +out.append("tags", "css"); +out.toString(); // "page=2&sort=name&tags=js&tags=css" +``` + +**1 dependency → 0 dependencies.** `URLSearchParams` is in every browser and in Node.js since v10. It handles encoding, repeated keys, and iteration. The package was a polyfill for an API that has shipped everywhere for years. diff --git a/ponytail/gemini-extension.json b/ponytail/gemini-extension.json new file mode 100644 index 0000000000000000000000000000000000000000..41e551a9b022785e5fd2e5c9ce89a50be9be9b0e --- /dev/null +++ b/ponytail/gemini-extension.json @@ -0,0 +1,6 @@ +{ + "name": "ponytail", + "version": "4.9.0", + "description": "Lazy senior dev mode. Forces the simplest, shortest solution that actually works: YAGNI, stdlib first, no unrequested abstractions.", + "contextFileName": "AGENTS.md" +} diff --git a/ponytail/hooks/claude-codex-hooks.json b/ponytail/hooks/claude-codex-hooks.json new file mode 100644 index 0000000000000000000000000000000000000000..0cd2fbafd5976817e755a2878f2049c8c9fa79bb --- /dev/null +++ b/ponytail/hooks/claude-codex-hooks.json @@ -0,0 +1,41 @@ +{ + "hooks": { + "SessionStart": [ + { + "matcher": "startup|resume|clear|compact", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/ponytail-activate.js\"", + "timeout": 5, + "statusMessage": "Loading ponytail mode..." + } + ] + } + ], + "SubagentStart": [ + { + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/ponytail-subagent.js\"", + "timeout": 5, + "statusMessage": "Loading ponytail mode..." + } + ] + } + ], + "UserPromptSubmit": [ + { + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/ponytail-mode-tracker.js\"", + "timeout": 5, + "statusMessage": "Tracking ponytail mode..." + } + ] + } + ] + } +} diff --git a/ponytail/hooks/copilot-hooks.json b/ponytail/hooks/copilot-hooks.json new file mode 100644 index 0000000000000000000000000000000000000000..1210c35a4546765a5d0e34f306e95d865c74d2b9 --- /dev/null +++ b/ponytail/hooks/copilot-hooks.json @@ -0,0 +1,21 @@ +{ + "version": 1, + "hooks": { + "sessionStart": [ + { + "type": "command", + "bash": "node \"${PLUGIN_ROOT}/hooks/ponytail-activate.js\"", + "powershell": "node \"${PLUGIN_ROOT}\\hooks\\ponytail-activate.js\"", + "timeoutSec": 5 + } + ], + "userPromptSubmitted": [ + { + "type": "command", + "bash": "node \"${PLUGIN_ROOT}/hooks/ponytail-mode-tracker.js\"", + "powershell": "node \"${PLUGIN_ROOT}\\hooks\\ponytail-mode-tracker.js\"", + "timeoutSec": 5 + } + ] + } +} diff --git a/ponytail/hooks/ponytail-activate.js b/ponytail/hooks/ponytail-activate.js new file mode 100644 index 0000000000000000000000000000000000000000..d54fbe4afafdee2f4219f6ce90efb2b79f47cc7b --- /dev/null +++ b/ponytail/hooks/ponytail-activate.js @@ -0,0 +1,96 @@ +#!/usr/bin/env node +// ponytail — Claude Code SessionStart activation hook +// +// Runs on every session start: +// 1. Writes flag file at $CLAUDE_CONFIG_DIR/.ponytail-active (defaults to ~/.claude; statusline reads this) +// 2. Emits ponytail ruleset as hidden SessionStart context +// 3. Detects missing statusline config and emits setup nudge + +const fs = require('fs'); +const path = require('path'); +const { getDefaultMode, getClaudeDir, isShellSafe } = require('./ponytail-config'); +const { getPonytailInstructions } = require('./ponytail-instructions'); +const { + clearMode, + isCodex, + isCopilot, + setMode, + writeHookOutput, +} = require('./ponytail-runtime'); + +const claudeDir = getClaudeDir(); +const settingsPath = path.join(claudeDir, 'settings.json'); + +const mode = getDefaultMode(); + +// "off" mode — skip activation entirely, don't write flag or emit rules +if (mode === 'off') { + clearMode(); + const hookOutput = (isCodex || isCopilot) ? '' : 'OK'; + writeHookOutput('SessionStart', 'off', hookOutput); + process.exit(0); +} + +// 1. Write flag file +try { + setMode(mode); +} catch (e) { + // Silent fail -- flag is best-effort, don't block the hook +} + +// 2. Emit the ponytail ruleset, filtered to the active intensity level. +let output = getPonytailInstructions(mode); + +// 3. Detect missing statusline config — nudge Claude to help set it up +if (!isCodex && !isCopilot) try { + let hasStatusline = false; + if (fs.existsSync(settingsPath)) { + // Strip UTF-8 BOM some editors prepend on Windows (breaks JSON.parse) + const raw = fs.readFileSync(settingsPath, 'utf8').replace(/^\uFEFF/, ''); + const settings = JSON.parse(raw); + if (settings.statusLine) { + hasStatusline = true; + } + } + + // Nudge at most once — the flag file marks that the user has already seen + // (and implicitly declined) the statusline setup offer. Repeating it every + // session start turns a helpful hint into a nag. + const nudgeFlagPath = path.join(claudeDir, '.ponytail-statusline-nudged'); + if (!hasStatusline && !fs.existsSync(nudgeFlagPath)) { + try { fs.writeFileSync(nudgeFlagPath, ''); } catch (e) { /* best-effort */ } + const isWindows = process.platform === 'win32'; + const scriptName = isWindows ? 'ponytail-statusline.ps1' : 'ponytail-statusline.sh'; + const scriptPath = path.join(__dirname, scriptName); + if (isShellSafe(scriptPath)) { + const command = isWindows + ? `powershell -ExecutionPolicy Bypass -File "${scriptPath}"` + : `bash "${scriptPath}"`; + const statusLineSnippet = + '"statusLine": { "type": "command", "command": ' + JSON.stringify(command) + ' }'; + output += "\n\n" + + "STATUSLINE SETUP NEEDED: The ponytail plugin includes a statusline badge showing active mode " + + "(e.g. [PONYTAIL], [PONYTAIL:ULTRA]). It is not configured yet. " + + "To enable, add this to " + settingsPath + ": " + + statusLineSnippet + " " + + "Proactively offer to set this up for the user on first interaction."; + } else { + // ponytail: install path has shell metacharacters — don't embed it in a + // command snippet; have the agent wire it up by hand instead. + output += "\n\n" + + "STATUSLINE SETUP NEEDED: The ponytail plugin includes a statusline badge showing active mode. " + + "Its install path contains characters unsafe to embed in a shell command, so configure it manually: " + + "add a statusLine command of type \"command\" that runs " + scriptName + + " from the plugin's hooks directory to " + settingsPath + ", quoting/escaping the path for your shell. " + + "Proactively offer to set this up for the user on first interaction."; + } + } +} catch (e) { + // Silent fail — don't block session start over statusline detection +} + +try { + writeHookOutput('SessionStart', mode, output); +} catch (e) { + // Silent fail — stdout closed/EPIPE at hook exit must not surface as a hook failure +} diff --git a/ponytail/hooks/ponytail-config.js b/ponytail/hooks/ponytail-config.js new file mode 100644 index 0000000000000000000000000000000000000000..9ba7fdc2e7853aa296507731f6ed2b7fcbcd0bff --- /dev/null +++ b/ponytail/hooks/ponytail-config.js @@ -0,0 +1,169 @@ +#!/usr/bin/env node +// ponytail — shared configuration resolver +// +// Resolution order for default mode: +// 1. PONYTAIL_DEFAULT_MODE environment variable +// 2. Config file defaultMode field: +// - $XDG_CONFIG_HOME/ponytail/config.json (any platform, if set) +// - ~/.config/ponytail/config.json (macOS / Linux fallback) +// - %APPDATA%\ponytail\config.json (Windows fallback) +// 3. 'full' + +const fs = require('fs'); +const path = require('path'); +const os = require('os'); + +const DEFAULT_MODE = 'full'; +const VALID_MODES = ['off', 'lite', 'full', 'ultra', 'review']; +const RUNTIME_MODES = ['off', 'lite', 'full', 'ultra']; + +function normalizeMode(mode) { + if (typeof mode !== 'string') return null; + const normalized = mode.trim().toLowerCase(); + return RUNTIME_MODES.includes(normalized) ? normalized : null; +} + +function normalizeConfigMode(mode) { + if (typeof mode !== 'string') return null; + const normalized = mode.trim().toLowerCase(); + return VALID_MODES.includes(normalized) ? normalized : null; +} + +function normalizePersistedMode(mode) { + return normalizeMode(mode) || normalizeConfigMode(mode); +} + +// "stop ponytail" / "normal mode" turn ponytail off, but only as a standalone +// command. Matching the phrase anywhere in the message turned it off mid-task +// for ordinary requests like "add a normal mode toggle" — so require the whole +// message to be the command, ignoring case and trailing punctuation. +function isDeactivationCommand(text) { + const t = String(text || '').trim().toLowerCase().replace(/[.!?\s]+$/, ''); + return t === 'stop ponytail' || t === 'normal mode'; +} + +// ponytail: only embed the plugin install path in a statusline shell command when +// it's made of ordinary path characters. An allowlist beats escaping every shell's +// metacharacters; a hostile clone path (quotes, &, $, backtick, ;, etc.) falls back +// to manual setup instead. Allows : \ / for normal Windows and POSIX paths. Full +// per-shell escaper only if a real need appears. +function isShellSafe(p) { + return typeof p === 'string' && /^[A-Za-z0-9 _.\-:/\\~]+$/.test(p); +} + +function getConfigDir() { + if (process.env.XDG_CONFIG_HOME) { + return path.join(process.env.XDG_CONFIG_HOME, 'ponytail'); + } + if (process.platform === 'win32') { + return path.join( + process.env.APPDATA || path.join(os.homedir(), 'AppData', 'Roaming'), + 'ponytail' + ); + } + return path.join(os.homedir(), '.config', 'ponytail'); +} + +function getConfigPath() { + return path.join(getConfigDir(), 'config.json'); +} + +function getClaudeDir() { + // ponytail: CLAUDE_CONFIG_DIR overrides ~/.claude, matching Claude Code. + return process.env.CLAUDE_CONFIG_DIR || path.join(os.homedir(), '.claude'); +} + +function getDefaultMode() { + // 1. Environment variable (highest priority) + const envMode = process.env.PONYTAIL_DEFAULT_MODE; + // ponytail: a default must be a runtime level (off/lite/full/ultra); review is + // a session-only mode, never a valid default (#377). Validate against + // RUNTIME_MODES so a stray env var or config can't make review the default. + if (envMode && RUNTIME_MODES.includes(envMode.toLowerCase())) { + return envMode.toLowerCase(); + } + + // 2. Config file + try { + const configPath = getConfigPath(); + // Strip UTF-8 BOM (common on Windows-saved files) so JSON.parse doesn't choke + const config = JSON.parse(fs.readFileSync(configPath, 'utf8').replace(/^\uFEFF/, '')); + if (config.defaultMode && RUNTIME_MODES.includes(config.defaultMode.toLowerCase())) { + return config.defaultMode.toLowerCase(); + } + } catch (e) { + // Config file doesn't exist or is invalid — fall through + } + + // 3. Default + return DEFAULT_MODE; +} + +// Silence the pi "Ponytail loaded" startup toast while keeping ponytail active. +// PONYTAIL_QUIET_STARTUP=1 (or any truthy value; 0/false/empty mean "show it") +// takes precedence, else config.quietStartup === true. Mirrors getHideStatus. +function getQuietStartup() { + const env = process.env.PONYTAIL_QUIET_STARTUP; + if (env !== undefined) { + const v = env.trim().toLowerCase(); + return v !== '' && v !== '0' && v !== 'false' && v !== 'no'; + } + try { + const config = JSON.parse(fs.readFileSync(getConfigPath(), 'utf8').replace(/^\uFEFF/, '')); + return config.quietStartup === true; + } catch (_) { + return false; + } +} + +// Hide the status-bar indicator while keeping ponytail active (#324). +// PONYTAIL_HIDE_STATUS=1 (or any truthy value; 0/false/empty mean "don't hide") +// takes precedence, else config.hideStatus === true. +function getHideStatus() { + const env = process.env.PONYTAIL_HIDE_STATUS; + if (env !== undefined) { + const v = env.trim().toLowerCase(); + return v !== '' && v !== '0' && v !== 'false' && v !== 'no'; + } + try { + const config = JSON.parse(fs.readFileSync(getConfigPath(), 'utf8').replace(/^\uFEFF/, '')); + return config.hideStatus === true; + } catch (_) { + return false; + } +} + +function writeDefaultMode(mode) { + // ponytail: only a runtime level can be a default; review is session-only (#377). + const normalized = normalizeMode(mode); + if (!normalized) return null; + + const configPath = getConfigPath(); + fs.mkdirSync(path.dirname(configPath), { recursive: true }); + let config = {}; + try { + config = JSON.parse(fs.readFileSync(configPath, 'utf8').replace(/^\uFEFF/, '')); + if (!config || typeof config !== 'object' || Array.isArray(config)) config = {}; + } catch (_) {} + config.defaultMode = normalized; + fs.writeFileSync(configPath, JSON.stringify(config, null, 2), 'utf8'); + return normalized; +} + +module.exports = { + DEFAULT_MODE, + VALID_MODES, + RUNTIME_MODES, + getDefaultMode, + getConfigDir, + getConfigPath, + getClaudeDir, + getHideStatus, + getQuietStartup, + isShellSafe, + normalizeMode, + normalizeConfigMode, + normalizePersistedMode, + isDeactivationCommand, + writeDefaultMode, +}; diff --git a/ponytail/hooks/ponytail-instructions.js b/ponytail/hooks/ponytail-instructions.js new file mode 100644 index 0000000000000000000000000000000000000000..3ec3980a08d9565602020e6789f1759d9e2e52e1 --- /dev/null +++ b/ponytail/hooks/ponytail-instructions.js @@ -0,0 +1,98 @@ +#!/usr/bin/env node +// Shared Ponytail instruction builder for Claude hooks and Pi extension. + +const fs = require('fs'); +const path = require('path'); +const { DEFAULT_MODE, normalizeMode, normalizePersistedMode } = require('./ponytail-config'); + +const INDEPENDENT_MODES = new Set(['review']); +const SKILL_PATH = path.join(__dirname, '..', 'skills', 'ponytail', 'SKILL.md'); + +function filterSkillBodyForMode(body, mode) { + const effectiveMode = normalizeMode(mode) || DEFAULT_MODE; + const withoutFrontmatter = String(body || '').replace(/^---[\s\S]*?---\s*/, ''); + + // Only the intensity table rows and worked examples are mode-specific, and + // both are keyed by a mode name (lite/full/ultra). A bullet whose label is + // not a mode — e.g. "No unrequested abstractions: ..." — is a normal rule + // and must be kept verbatim. + return withoutFrontmatter + .split(/\r?\n/) + .filter((line) => { + const tableLabel = line.match(/^\|\s*\*\*(.+?)\*\*\s*\|/); + if (tableLabel) { + const labelMode = normalizeMode(tableLabel[1].trim()); + if (labelMode) return labelMode === effectiveMode; + } + + // Require a quoted value: every worked example is `- lite: "..."`. Without + // this, an ordinary rule bullet that happens to start with a mode word + // (e.g. "- Full: ...") is silently dropped in every other mode — it looks + // like a worked example but is really prose meant to survive verbatim. + const exampleLabel = line.match(/^-\s*([^:]+):\s*"/); + if (exampleLabel) { + const labelMode = normalizeMode(exampleLabel[1].trim()); + if (labelMode) return labelMode === effectiveMode; + } + + return true; + }) + .join('\n'); +} + +function getFallbackInstructions(mode) { + return 'PONYTAIL MODE ACTIVE — level: ' + mode + '\n\n' + + 'You are a lazy senior developer. Lazy means efficient, not careless. The best code is the code never written.\n\n' + + '## Persistence\n\n' + + 'ACTIVE EVERY RESPONSE. No drift back to over-building. Still active if unsure. Off only: "stop ponytail" / "normal mode".\n\n' + + 'Current level: **' + mode + '**. Switch: `/ponytail lite|full|ultra`.\n\n' + + '## The ladder\n\n' + + 'Before any code, stop at the first rung that holds (the ladder runs after you understand the problem, not instead of it — read the code it touches and trace the real flow first):\n' + + '1. Does this need to be built at all? (YAGNI)\n' + + '2. Does it already exist in this codebase? Reuse what is already here, do not re-write it.\n' + + '3. Does the standard library do this? Use it.\n' + + '4. Does a native platform feature cover it? Use it.\n' + + '5. Does an already-installed dependency solve it? Use it.\n' + + '6. Can this be one line? Make it one line.\n' + + '7. Only then: write the minimum code that works.\n\n' + + 'Bug fix = root cause, not symptom: grep every caller of the function you touch and fix the shared function once (a smaller diff than one guard per caller); patching only the path the ticket names leaves a sibling caller broken.\n\n' + + '## Rules\n\n' + + 'No abstractions that were not requested. No avoidable dependencies. No boilerplate nobody asked for. ' + + 'Deletion over addition. Boring over clever. Fewest files possible. ' + + 'Ship the lazy version and question the complex request in the same response — never stall. ' + + 'Between two same-size stdlib options, pick the one correct on edge cases. ' + + 'Mark deliberate simplifications that cut a real corner with a known ceiling, using a `ponytail:` comment that names the ceiling and upgrade path.\n\n' + + '## Output\n\n' + + 'Code first. Then at most three short lines: what was skipped, when to add it. ' + + 'If the explanation is longer than the code, delete the explanation. ' + + 'Explanation the user explicitly asked for is not debt, give it in full.\n\n' + + '## When NOT to be lazy\n\n' + + 'Never simplify away: understanding the problem (read it fully and trace the real flow before picking a rung — a small diff you do not understand is just laziness dressed up as efficiency), input validation at trust boundaries, error handling that prevents data loss, ' + + 'security measures, accessibility basics, the calibration real hardware needs (the platform is never the spec ideal), anything the user explicitly asked to keep. ' + + 'Lazy code without its check is unfinished: non-trivial logic leaves ONE runnable check behind (assert-based demo/self-check or one small test file; no frameworks). Trivial one-liners need no test.\n\n' + + '## Boundaries\n\n' + + 'Ponytail governs what you build, not how you talk. "stop ponytail" or "normal mode": revert. Level persists until changed or session end.'; +} + +function getPonytailInstructions(mode) { + const configuredMode = normalizePersistedMode(mode) || DEFAULT_MODE; + + if (INDEPENDENT_MODES.has(configuredMode)) { + return 'PONYTAIL MODE ACTIVE — level: ' + configuredMode + '. Behavior defined by /ponytail-' + configuredMode + ' skill.'; + } + + const effectiveMode = normalizeMode(configuredMode) || DEFAULT_MODE; + + try { + return 'PONYTAIL MODE ACTIVE — level: ' + effectiveMode + '\n\n' + + filterSkillBodyForMode(fs.readFileSync(SKILL_PATH, 'utf8'), effectiveMode); + } catch (e) { + return getFallbackInstructions(effectiveMode); + } +} + +module.exports = { + filterSkillBodyForMode, + getFallbackInstructions, + getPonytailInstructions, +}; diff --git a/ponytail/hooks/ponytail-mode-tracker.js b/ponytail/hooks/ponytail-mode-tracker.js new file mode 100644 index 0000000000000000000000000000000000000000..f644f00a4ac9b39e55c476596843dc43ff20811f --- /dev/null +++ b/ponytail/hooks/ponytail-mode-tracker.js @@ -0,0 +1,130 @@ +#!/usr/bin/env node +// ponytail — UserPromptSubmit hook to track which ponytail mode is active +// Inspects user input for /ponytail commands and writes mode to flag file + +const { getDefaultMode, isDeactivationCommand, writeDefaultMode } = require('./ponytail-config'); +const { clearMode, isQoder, readMode, setMode, writeHookOutput } = require('./ponytail-runtime'); +const { getPonytailInstructions } = require('./ponytail-instructions'); + +let input = ''; +let done = false; + +function finish() { + if (done) return; + done = true; + try { + // Strip UTF-8 BOM some shells prepend when piping (breaks JSON.parse) + const data = JSON.parse(input.replace(/^\uFEFF/, '')); + const prompt = (data.prompt || '').trim().toLowerCase(); + + // Match /ponytail commands + let modeSwitched = false; + let deactivated = false; + if (/^[/@$]ponytail/.test(prompt)) { + const parts = prompt.split(/\s+/); + const cmd = parts[0].replace(/^[@$]/, '/'); + const arg = parts[1] || ''; + + let mode = null; + let isReportOnly = false; + + if (cmd === '/ponytail-review' || cmd === '/ponytail:ponytail-review') { + mode = 'review'; + } else if (cmd === '/ponytail' || cmd === '/ponytail:ponytail') { + // `/ponytail default ` persists the default to config (survives + // restarts). Plain switches stay session-scoped ("sticks until session + // end"), so this is the only path that writes config. review is not a + // valid default (#377), so only off/lite/full/ultra are accepted. + if (arg === 'default') { + const dmode = parts[2]; + if (dmode === 'off' || dmode === 'lite' || dmode === 'full' || dmode === 'ultra') { + writeDefaultMode(dmode); + writeHookOutput('UserPromptSubmit', dmode, 'PONYTAIL DEFAULT SET — new sessions start in ' + dmode + '.'); + } + return; // don't fall through to the session-mode switch + } + if (arg === 'lite') mode = 'lite'; + else if (arg === 'full') mode = 'full'; + else if (arg === 'ultra') mode = 'ultra'; + else if (arg === 'off') mode = 'off'; + else if (arg === '') { + isReportOnly = true; + mode = readMode() || getDefaultMode(); + } else { + mode = getDefaultMode(); + } + } + + if (isReportOnly) { + writeHookOutput( + 'UserPromptSubmit', + mode, + 'PONYTAIL MODE ACTIVE — level: ' + mode, + ); + } else if (mode && mode !== 'off') { + setMode(mode); + modeSwitched = true; + // ponytail: Qoder needs the full ruleset every turn, so when a mode + // switch happens we fold the confirmation into the ruleset output + // below (one JSON on stdout) instead of emitting two separate writes. + if (!isQoder) { + writeHookOutput( + 'UserPromptSubmit', + mode, + 'PONYTAIL MODE CHANGED — level: ' + mode, + ); + } + } else if (mode === 'off') { + clearMode(); + deactivated = true; + writeHookOutput('UserPromptSubmit', 'off', 'PONYTAIL MODE OFF'); + } + } + + // Detect deactivation + if (!modeSwitched && !deactivated && isDeactivationCommand(prompt)) { + clearMode(); + deactivated = true; + writeHookOutput('UserPromptSubmit', 'off', 'PONYTAIL MODE OFF'); + } + + // Qoder has no SessionStart event, so UserPromptSubmit does double duty: + // activate the default mode on first prompt (if no flag exists yet), then + // inject the ruleset on every prompt. Claude Code/Codex do this in + // SessionStart via ponytail-activate.js; Qoder can't, so we do it here. + // Skip when deactivated — user just turned ponytail off. + if (isQoder && !deactivated) { + let currentMode = readMode(); + if (!currentMode) { + // First prompt in session — initialize from config/env default + currentMode = getDefaultMode(); + if (currentMode !== 'off') { + try { setMode(currentMode); } catch (e) {} + } + } + if (currentMode && currentMode !== 'off') { + // ponytail: one JSON per invocation — mode-switch confirmation is + // folded into the ruleset header so Qoder gets both in one write. + const header = modeSwitched + ? 'PONYTAIL MODE CHANGED — level: ' + currentMode + '\n\n' + : ''; + writeHookOutput('UserPromptSubmit', currentMode, header + getPonytailInstructions(currentMode)); + } + } + } catch (e) { + // Silent fail + } +} + +process.stdin.on('data', chunk => { input += chunk; }); +process.stdin.on('end', finish); + +// Never hang the session. On Windows, Claude Code runs this hook through a +// PowerShell `if {}` wrapper that can swallow the piped prompt JSON, so stdin +// 'end' never fires and the hook blocks forever — freezing the session (#443). +// On error, or after a short fallback, process whatever arrived (recovering the +// mode if data came without EOF) and exit. unref() keeps the timer from adding +// latency to the normal path, where 'end' fires first. Mirrors the best-effort, +// never-block contract the other lifecycle hooks already follow. +process.stdin.on('error', () => { finish(); process.exit(0); }); +setTimeout(() => { finish(); process.exit(0); }, 1000).unref(); diff --git a/ponytail/hooks/ponytail-runtime.js b/ponytail/hooks/ponytail-runtime.js new file mode 100644 index 0000000000000000000000000000000000000000..db0c4776e1ab0d343b933a915456476e5bd11b40 --- /dev/null +++ b/ponytail/hooks/ponytail-runtime.js @@ -0,0 +1,100 @@ +const fs = require('fs'); +const path = require('path'); +const os = require('os'); +const { getClaudeDir, getConfigDir } = require('./ponytail-config'); + +const STATE_FILE = '.ponytail-active'; + +// ponytail: VS Code Copilot never sets COPILOT_PLUGIN_DATA — it only injects +// CLAUDE_PLUGIN_ROOT, pointed at an install path under .vscode/agent-plugins/ +// (#528). Without this fallback isCopilot was false, so ponytail assumed +// native Claude Code and emitted the statusline nudge, which VS Code Copilot +// doesn't read. +function isVsCodeCopilotRoot(pluginRoot) { + if (!pluginRoot) return false; + return pluginRoot.split(/[\\/]+/).includes('agent-plugins') && + pluginRoot.toLowerCase().includes('.vscode'); +} + +const isCopilot = Boolean(process.env.COPILOT_PLUGIN_DATA) || + isVsCodeCopilotRoot(process.env.CLAUDE_PLUGIN_ROOT); +const isCodex = !isCopilot && Boolean(process.env.PLUGIN_DATA); +const isQoder = !isCopilot && !isCodex && Boolean(process.env.QODER_SESSION_ID); + +let stateDir = getClaudeDir(); +if (isCodex) stateDir = process.env.PLUGIN_DATA; +// COPILOT_PLUGIN_DATA is unset under VS Code Copilot, so fall back to +// getClaudeDir() rather than building a path from undefined. +if (isCopilot) stateDir = process.env.COPILOT_PLUGIN_DATA || getClaudeDir(); +if (isQoder) stateDir = path.join(os.homedir(), '.qoder'); + +const statePath = path.join(stateDir, STATE_FILE); + +function setMode(mode) { + fs.mkdirSync(path.dirname(statePath), { recursive: true }); + fs.writeFileSync(statePath, mode); +} + +function clearMode() { + try { fs.unlinkSync(statePath); } catch (e) {} +} + +// Live mode written by activate/mode-tracker. Absent flag = ponytail off. +function readMode() { + try { + return fs.readFileSync(statePath, 'utf8').trim() || null; + } catch (e) { + return null; + } +} + +function writeHookOutput(event, mode, context = '') { + if (isCopilot) { + // Copilot reads additionalContext on SessionStart; ignores output elsewhere. + process.stdout.write(JSON.stringify( + event === 'SessionStart' && context ? { additionalContext: context } : {})); + return; + } + if (isCodex) { + const output = { systemMessage: `PONYTAIL:${mode.toUpperCase()}` }; + if (context) { + output.hookSpecificOutput = { + hookEventName: event, + additionalContext: context, + }; + } + process.stdout.write(JSON.stringify(output)); + return; + } + if (isQoder) { + // Qoder: hookSpecificOutput JSON, same shape as Codex minus systemMessage. + // UserPromptSubmit additionalContext is injected into the Agent's conversation. + const output = {}; + if (context) { + output.hookSpecificOutput = { + hookEventName: event, + additionalContext: context, + }; + } + process.stdout.write(JSON.stringify(output)); + return; + } + // Native Claude: SessionStart accepts raw stdout, but SubagentStart needs the + // hookSpecificOutput JSON form or the context is dropped. + if (event === 'SubagentStart') { + process.stdout.write(JSON.stringify( + { hookSpecificOutput: { hookEventName: event, additionalContext: context } })); + return; + } + process.stdout.write(context); +} + +module.exports = { + clearMode, + isCodex, + isCopilot, + isQoder, + readMode, + setMode, + writeHookOutput, +}; diff --git a/ponytail/hooks/ponytail-statusline.ps1 b/ponytail/hooks/ponytail-statusline.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..b75b142143b9a2df4180962b7e48c6ad5e855226 --- /dev/null +++ b/ponytail/hooks/ponytail-statusline.ps1 @@ -0,0 +1,24 @@ +# CLAUDE_CONFIG_DIR overrides ~/.claude, matching where the hooks write the flag (issue #34) +$ClaudeDir = if ($env:CLAUDE_CONFIG_DIR) { $env:CLAUDE_CONFIG_DIR } else { Join-Path $HOME ".claude" } +$Flag = Join-Path $ClaudeDir ".ponytail-active" +if (-not (Test-Path $Flag)) { + exit 0 +} + +$Mode = "" +try { + $Mode = (Get-Content $Flag -ErrorAction Stop | Select-Object -First 1).Trim() +} catch { + exit 0 +} + +$Esc = [char]27 +# ultra is the high-intensity mode; flag it amber so it stands out from the +# default green. The level is still in the text, so color is a redundant cue. +$Color = if ($Mode -eq "ultra") { "173" } else { "108" } +if ([string]::IsNullOrEmpty($Mode) -or $Mode -eq "full") { + [Console]::Write("${Esc}[38;5;${Color}m[PONYTAIL]${Esc}[0m") +} else { + $Suffix = $Mode.ToUpperInvariant() + [Console]::Write("${Esc}[38;5;${Color}m[PONYTAIL:$Suffix]${Esc}[0m") +} diff --git a/ponytail/hooks/ponytail-statusline.sh b/ponytail/hooks/ponytail-statusline.sh new file mode 100644 index 0000000000000000000000000000000000000000..7d0616051b5a44de4710eeb13f1e8c5a5584fb3f --- /dev/null +++ b/ponytail/hooks/ponytail-statusline.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# CLAUDE_CONFIG_DIR overrides ~/.claude, matching where the hooks write the flag (issue #34) +flag="${CLAUDE_CONFIG_DIR:-$HOME/.claude}/.ponytail-active" +[ -f "$flag" ] || exit 0 + +mode=$(head -n1 "$flag" | tr -d '[:space:]') + +# ultra is the high-intensity mode; flag it amber so it stands out from the +# default green at a glance. The level is still in the text, so color is a +# redundant cue, not the only one. +color=108 +[ "$mode" = "ultra" ] && color=173 + +if [ -z "$mode" ] || [ "$mode" = "full" ]; then + printf '\033[38;5;%sm[PONYTAIL]\033[0m' "$color" +else + printf '\033[38;5;%sm[PONYTAIL:%s]\033[0m' "$color" "$(printf '%s' "$mode" | tr '[:lower:]' '[:upper:]')" +fi diff --git a/ponytail/hooks/ponytail-subagent.js b/ponytail/hooks/ponytail-subagent.js new file mode 100644 index 0000000000000000000000000000000000000000..f2a7c77632613741226e92269d918d3a446351a1 --- /dev/null +++ b/ponytail/hooks/ponytail-subagent.js @@ -0,0 +1,77 @@ +#!/usr/bin/env node +// ponytail — Claude Code SubagentStart hook +// +// SessionStart context is parent-thread only and never reaches subagents, so +// without this every Task-spawned agent runs ponytail-unaware (issue #252). +// When ponytail mode is active, inject the same ruleset into each subagent. +// +// Scoping (opt-in, issue #506): set PONYTAIL_SUBAGENT_MATCHER to a regex and +// the ruleset is injected only into subagents whose agent_type matches. The +// regex is unanchored and case-insensitive — "explore|general" matches either, +// "^general$" is exact. Unset means inject into every subagent, as before. + +const { getPonytailInstructions } = require('./ponytail-instructions'); +const { readMode, writeHookOutput } = require('./ponytail-runtime'); + +const mode = readMode(); + +// Absent flag or off → ponytail isn't active; inject nothing. +if (!mode || mode === 'off') { + process.exit(0); +} + +function inject() { + try { + writeHookOutput('SubagentStart', mode, getPonytailInstructions(mode)); + } catch (e) { + // Silent fail — a stdout error at hook exit must not surface as a hook failure. + } +} + +// A bad regex must never crash the hook; treat it as "no matcher" and inject. +let matcherRe = null; +try { + if (process.env.PONYTAIL_SUBAGENT_MATCHER) { + matcherRe = new RegExp(process.env.PONYTAIL_SUBAGENT_MATCHER, 'i'); + } +} catch (e) { + matcherRe = null; +} + +// No matcher → keep the original synchronous, stdin-independent path. On Windows +// the PowerShell `if {}` wrapper can swallow the piped JSON so stdin 'end' never +// fires (#443); the default path must not wait on stdin or it would stall every +// subagent spawn. +if (!matcherRe) { + inject(); + process.exit(0); +} + +// Matcher set → read agent_type from stdin and skip only on a definite +// mismatch. Missing/unparseable agent_type, a stdin error, or the timeout all +// fail open (inject), so scoping never silently drops the persona. +let input = ''; +let done = false; + +function finish() { + if (done) return; + done = true; + + let agentType = ''; + try { + // Strip UTF-8 BOM some shells prepend when piping (breaks JSON.parse) + agentType = String(JSON.parse(input.replace(/^\uFEFF/, '')).agent_type || '').trim(); + } catch (e) { + // Unparseable payload — fall through and inject to be safe. + } + if (agentType && !matcherRe.test(agentType)) { + process.exit(0); + } + inject(); +} + +process.stdin.on('data', chunk => { input += chunk; }); +process.stdin.on('end', finish); +// Never block the session (#443): recover on stdin error or a short fallback. +process.stdin.on('error', () => { finish(); process.exit(0); }); +setTimeout(() => { finish(); process.exit(0); }, 1000).unref(); diff --git a/ponytail/hooks/qoder-hooks.json b/ponytail/hooks/qoder-hooks.json new file mode 100644 index 0000000000000000000000000000000000000000..8865abaa7bcd954ca72efa15ebfcd2d5246646ef --- /dev/null +++ b/ponytail/hooks/qoder-hooks.json @@ -0,0 +1,26 @@ +{ + "_comment": "Reference template — copy the 'hooks' object into your .qoder/settings.json or ~/.qoder/settings.json. Replace PONYTAIL_DIR with the path to your ponytail checkout (e.g. ~/.qoder/plugins/ponytail or the npm global install path).", + "hooks": { + "UserPromptSubmit": [ + { + "hooks": [ + { + "type": "command", + "command": "node PONYTAIL_DIR/hooks/ponytail-mode-tracker.js" + } + ] + } + ], + "PreToolUse": [ + { + "matcher": "task|Task", + "hooks": [ + { + "type": "command", + "command": "node PONYTAIL_DIR/hooks/ponytail-subagent.js" + } + ] + } + ] + } +} diff --git a/ponytail/opencode.json b/ponytail/opencode.json new file mode 100644 index 0000000000000000000000000000000000000000..2c0be9923fbdd25fdee8e35ab310b3dc84ec954f --- /dev/null +++ b/ponytail/opencode.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://opencode.ai/config.json", + "plugin": ["./.opencode/plugins/ponytail.mjs"] +} diff --git a/ponytail/package.json b/ponytail/package.json new file mode 100644 index 0000000000000000000000000000000000000000..6a59e9ed86a83e6be815c08ea8ddfa4b553cc21b --- /dev/null +++ b/ponytail/package.json @@ -0,0 +1,47 @@ +{ + "name": "@dietrichgebert/ponytail", + "version": "4.9.0", + "description": "Lazy senior dev mode for AI agents. The best code is the code you never wrote.", + "keywords": ["opencode-plugin", "opencode", "ponytail", "pi-package", "pi", "skills", "qoder"], + "license": "MIT", + "author": { + "name": "Dietrich Gebert", + "url": "https://github.com/DietrichGebert" + }, + "homepage": "https://github.com/DietrichGebert/ponytail", + "repository": { + "type": "git", + "url": "git+https://github.com/DietrichGebert/ponytail.git" + }, + "bugs": { + "url": "https://github.com/DietrichGebert/ponytail/issues" + }, + "main": "./.opencode/plugins/ponytail.mjs", + "exports": { + ".": "./.opencode/plugins/ponytail.mjs", + "./plugin": "./.opencode/plugins/ponytail.mjs" + }, + "files": [ + "AGENTS.md", + "hooks/", + "skills/", + ".opencode/", + ".qoder/", + ".qoder-plugin/", + "pi-extension/", + "scripts/uninstall.js", + "assets/", + "LICENSE" + ], + "scripts": { + "test": "node --test tests/*.test.js && npm test --prefix pi-extension && npm test --prefix ponytail-mcp" + }, + "pi": { + "extensions": ["./pi-extension/index.js"], + "skills": ["./skills"] + }, + "publishConfig": { + "access": "public" + } +} + diff --git a/ponytail/pi-extension/index.js b/ponytail/pi-extension/index.js new file mode 100644 index 0000000000000000000000000000000000000000..84201ca9a3218e408d463e66635c11cee1c09021 --- /dev/null +++ b/ponytail/pi-extension/index.js @@ -0,0 +1,211 @@ +import { createRequire } from "node:module"; + +const require = createRequire(import.meta.url); +const { + DEFAULT_MODE, + RUNTIME_MODES, + getDefaultMode, + getQuietStartup, + getHideStatus, + normalizeMode, + normalizePersistedMode, + isDeactivationCommand, + writeDefaultMode, +} = require("../hooks/ponytail-config.js"); +const { getPonytailInstructions, filterSkillBodyForMode } = require("../hooks/ponytail-instructions.js"); + +export { filterSkillBodyForMode }; +export const readDefaultMode = getDefaultMode; +export const readQuietStartup = getQuietStartup; + +const RUNTIME_MODE_LIST = RUNTIME_MODES.join("|"); +const PONYTAIL_COMMAND_DESCRIPTION = `Set mode: ${RUNTIME_MODE_LIST}. Commands: status, default `; + +export function resolveSessionMode(entries, fallbackMode = DEFAULT_MODE) { + const fallback = normalizePersistedMode(fallbackMode) || DEFAULT_MODE; + if (!Array.isArray(entries)) return fallback; + + for (let i = entries.length - 1; i >= 0; i -= 1) { + const entry = entries[i]; + if (entry?.type !== "custom" || entry?.customType !== "ponytail-mode") continue; + + const mode = normalizePersistedMode(entry?.data?.mode); + if (mode) return mode; + } + + return fallback; +} + +export function parsePonytailCommand(text, defaultMode = DEFAULT_MODE) { + const fallback = normalizePersistedMode(defaultMode) || DEFAULT_MODE; + const normalizedText = String(text || "").trim().toLowerCase(); + + if (!normalizedText) { + return { type: "set-mode", mode: fallback === "off" ? "full" : fallback }; + } + + const [primary, secondary] = normalizedText.split(/\s+/); + + if (primary === "status") return { type: "status" }; + + if (primary === "default") { + // ponytail: a default must be a runtime level; review is session-only (#377). + const mode = normalizeMode(secondary); + return mode ? { type: "set-default", mode } : { type: "invalid", reason: "invalid-default-mode" }; + } + + const mode = normalizeMode(primary); + return mode ? { type: "set-mode", mode } : { type: "invalid", reason: "invalid-mode", mode: primary }; +} + +export { writeDefaultMode }; + +export default function ponytailExtension(pi) { + let currentMode = DEFAULT_MODE; + let configuredDefaultMode = getDefaultMode(); + let hideStatus = getHideStatus(); + let isActive = false; + let lastCtx = null; + + // -- Status bar -- + function syncStatus(ctx) { + if (ctx) lastCtx = ctx; + const c = ctx || lastCtx; + // ponytail: hide the indicator but keep the ruleset active (#324). + if (hideStatus) return; + if (!c?.ui?.setStatus) return; + // ponytail: try/catch guards against pi-web theme proxy throwing before initTheme + let theme; + try { theme = c.ui.theme; if (!theme?.fg) return; } catch { return; } + if (currentMode === "off") { + c.ui.setStatus("ponytail", ""); + return; + } + const levelIcons = { lite: "🌿", full: "⚡", ultra: "🔥" }; + const icon = levelIcons[currentMode] || ""; + const label = currentMode.toUpperCase(); + const indicator = isActive ? theme.fg("accent", "●") : theme.fg("dim", "○"); + c.ui.setStatus("ponytail", indicator + " 🐴 " + theme.fg("muted", "ponytail: ") + theme.fg("text", icon + " " + label)); + } + + const setMode = (mode, ctx) => { + const normalized = normalizePersistedMode(mode); + if (!normalized) return; + + currentMode = normalized; + pi.appendEntry("ponytail-mode", { mode: normalized }); + syncStatus(ctx); + ctx?.ui?.notify?.(`Ponytail mode set to ${normalized}.`, "info"); + }; + + const sendAlias = (skillName, args, ctx) => { + const normalized = String(args || "").trim(); + const message = normalized ? `${skillName} ${normalized}` : skillName; + + if (ctx?.isIdle?.() === false) { + pi.sendUserMessage(message, { deliverAs: "followUp" }); + ctx?.ui?.notify?.(`${skillName} queued as follow-up.`, "info"); + return; + } + + pi.sendUserMessage(message); + }; + + pi.registerCommand("ponytail", { + description: PONYTAIL_COMMAND_DESCRIPTION, + handler: async (args, ctx) => { + const parsed = parsePonytailCommand(args, configuredDefaultMode); + + if (parsed.type === "status") { + ctx?.ui?.notify?.(`Ponytail: current ${currentMode} • default ${configuredDefaultMode}`, "info"); + return; + } + + if (parsed.type === "set-default") { + try { + const written = writeDefaultMode(parsed.mode); + if (written) { + configuredDefaultMode = getDefaultMode(); + const message = configuredDefaultMode === written + ? `Default Ponytail mode set to ${written}.` + : `Saved default ${written}, but env override keeps default at ${configuredDefaultMode}.`; + ctx?.ui?.notify?.(message, "info"); + } + } catch (e) { + ctx?.ui?.notify?.(`Failed to save default mode: ${e.message}`, "error"); + } + return; + } + + if (parsed.type === "set-mode") { + setMode(parsed.mode, ctx); + return; + } + + ctx?.ui?.notify?.("Unknown or unsupported /ponytail mode.", "warning"); + }, + }); + + pi.registerCommand("ponytail-review", { + description: "Run /skill:ponytail-review", + handler: (_args, ctx) => sendAlias("/skill:ponytail-review", "", ctx), + }); + + pi.registerCommand("ponytail-audit", { + description: "Run /skill:ponytail-audit", + handler: (_args, ctx) => sendAlias("/skill:ponytail-audit", "", ctx), + }); + + pi.registerCommand("ponytail-gain", { + description: "Run /skill:ponytail-gain", + handler: (_args, ctx) => sendAlias("/skill:ponytail-gain", "", ctx), + }); + + pi.registerCommand("ponytail-debt", { + description: "Run /skill:ponytail-debt", + handler: (_args, ctx) => sendAlias("/skill:ponytail-debt", "", ctx), + }); + + pi.registerCommand("ponytail-help", { + description: "Run /skill:ponytail-help", + handler: (_args, ctx) => sendAlias("/skill:ponytail-help", "", ctx), + }); + + pi.on("input", async (event) => { + if (event?.source === "extension") return; + + const text = String(event?.text || ""); + if (currentMode !== "off" && isDeactivationCommand(text)) { + setMode("off"); + } + }); + + pi.on("session_start", async (_event, ctx) => { + const entries = ctx?.sessionManager?.getBranch?.() || ctx?.sessionManager?.getEntries?.() || []; + configuredDefaultMode = getDefaultMode(); + hideStatus = getHideStatus(); + currentMode = resolveSessionMode(entries, configuredDefaultMode); + syncStatus(ctx); + if (!getQuietStartup()) { + ctx?.ui?.notify?.(`Ponytail loaded: ${currentMode}`, "info"); + } + }); + + pi.on("agent_start", async (_event, ctx) => { + isActive = true; + syncStatus(ctx); + }); + + pi.on("agent_end", async (_event, ctx) => { + isActive = false; + syncStatus(ctx); + }); + + pi.on("before_agent_start", async (event) => { + if (!currentMode || currentMode === "off") return; + // Guard a null/undefined event or a missing systemPrompt: don't crash, and + // don't prepend the literal string "undefined" to the prompt (#439, #440). + const base = event?.systemPrompt ? `${event.systemPrompt}\n\n` : ""; + return { systemPrompt: `${base}${getPonytailInstructions(currentMode)}` }; + }); +} diff --git a/ponytail/pi-extension/package.json b/ponytail/pi-extension/package.json new file mode 100644 index 0000000000000000000000000000000000000000..745bde2573b724e61e1526c87a3fdbb4e571c757 --- /dev/null +++ b/ponytail/pi-extension/package.json @@ -0,0 +1,8 @@ +{ + "name": "ponytail-pi-extension-dev", + "private": true, + "type": "module", + "scripts": { + "test": "node --test ./test/*.test.js" + } +} diff --git a/ponytail/pi-extension/test/extension.test.js b/ponytail/pi-extension/test/extension.test.js new file mode 100644 index 0000000000000000000000000000000000000000..d466325b8ff3d3ee000570b27ba5620265cc5ee8 --- /dev/null +++ b/ponytail/pi-extension/test/extension.test.js @@ -0,0 +1,243 @@ +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import ponytailExtension from "../index.js"; + +function createPiHarness() { + const events = new Map(); + const commands = new Map(); + const appendedEntries = []; + const sentUserMessages = []; + + const pi = { + on(eventName, handler) { + events.set(eventName, handler); + }, + registerCommand(name, options) { + commands.set(name, options); + }, + appendEntry(customType, data) { + appendedEntries.push({ customType, data }); + }, + sendUserMessage(text, options) { + sentUserMessages.push({ text, options }); + }, + }; + + ponytailExtension(pi); + return { events, commands, appendedEntries, sentUserMessages }; +} + +function createCommandContext(overrides = {}) { + return { + isIdle: () => true, + sessionManager: { getEntries: () => [] }, + ui: { notify() {} }, + ...overrides, + }; +} + +function withTempConfig(fn) { + const tempConfigHome = mkdtempSync(join(tmpdir(), "ponytail-test-")); + const previousXdg = process.env.XDG_CONFIG_HOME; + const previousHide = process.env.PONYTAIL_HIDE_STATUS; + process.env.XDG_CONFIG_HOME = tempConfigHome; + delete process.env.PONYTAIL_HIDE_STATUS; + + return Promise.resolve() + .then(fn) + .finally(() => { + if (previousXdg === undefined) delete process.env.XDG_CONFIG_HOME; + else process.env.XDG_CONFIG_HOME = previousXdg; + if (previousHide === undefined) delete process.env.PONYTAIL_HIDE_STATUS; + else process.env.PONYTAIL_HIDE_STATUS = previousHide; + rmSync(tempConfigHome, { recursive: true, force: true }); + }); +} + +test("extension registers Ponytail commands", () => { + const { commands } = createPiHarness(); + + assert.deepEqual([...commands.keys()].sort(), ["ponytail", "ponytail-audit", "ponytail-debt", "ponytail-gain", "ponytail-help", "ponytail-review"]); +}); + +test("/ponytail updates session mode and injects instructions", async () => withTempConfig(async () => { + const { commands, events, appendedEntries } = createPiHarness(); + const ctx = createCommandContext(); + + await events.get("session_start")({ reason: "startup" }, ctx); + await commands.get("ponytail").handler("ultra", ctx); + + assert.deepEqual(appendedEntries.at(-1), { + customType: "ponytail-mode", + data: { mode: "ultra" }, + }); + + const result = await events.get("before_agent_start")({ systemPrompt: "BASE" }, ctx); + assert.ok(result.systemPrompt.includes("PONYTAIL MODE ACTIVE")); + assert.ok(result.systemPrompt.includes("ultra")); +})); + +test("before_agent_start guards missing event and missing systemPrompt (#439, #440)", async () => withTempConfig(async () => { + const { events } = createPiHarness(); + const ctx = createCommandContext(); + await events.get("session_start")({ reason: "startup" }, ctx); // currentMode -> default (full) + + // #439: a null/undefined event must not crash, and still injects the ruleset. + for (const bad of [undefined, null]) { + const r = await events.get("before_agent_start")(bad, ctx); + assert.ok(r.systemPrompt.includes("PONYTAIL MODE ACTIVE")); + assert.ok(!r.systemPrompt.includes("undefined"), "must not contain the literal 'undefined'"); + } + + // #440: an event without a systemPrompt must not prepend the literal "undefined". + const empty = await events.get("before_agent_start")({}, ctx); + assert.ok(empty.systemPrompt.includes("PONYTAIL MODE ACTIVE")); + assert.ok(!empty.systemPrompt.startsWith("undefined"), "must not start with 'undefined'"); + + // A real base prompt is still preserved and prepended. + const withBase = await events.get("before_agent_start")({ systemPrompt: "BASE" }, ctx); + assert.ok(withBase.systemPrompt.startsWith("BASE\n\n")); + assert.ok(withBase.systemPrompt.includes("PONYTAIL MODE ACTIVE")); +})); + +test("session_start restores latest persisted mode", async () => withTempConfig(async () => { + const { events } = createPiHarness(); + const ctx = createCommandContext({ + sessionManager: { + getEntries: () => [ + { type: "custom", customType: "ponytail-mode", data: { mode: "lite" } }, + ], + }, + }); + + await events.get("session_start")({ reason: "resume" }, ctx); + const result = await events.get("before_agent_start")({ systemPrompt: "BASE" }, ctx); + + assert.ok(result.systemPrompt.includes("lite")); +})); + +test("skill alias commands delegate to Pi skill commands", async () => { + const { commands, sentUserMessages } = createPiHarness(); + const ctx = createCommandContext(); + + await commands.get("ponytail-review").handler("", ctx); + await commands.get("ponytail-audit").handler("", ctx); + await commands.get("ponytail-debt").handler("", ctx); + await commands.get("ponytail-gain").handler("", ctx); + await commands.get("ponytail-help").handler("", ctx); + + assert.deepEqual(sentUserMessages.map((entry) => entry.text), [ + "/skill:ponytail-review", + "/skill:ponytail-audit", + "/skill:ponytail-debt", + "/skill:ponytail-gain", + "/skill:ponytail-help", + ]); +}); + +test("normal mode disables persistent instructions", async () => withTempConfig(async () => { + const { commands, events } = createPiHarness(); + const ctx = createCommandContext(); + + await events.get("session_start")({ reason: "startup" }, ctx); + await commands.get("ponytail").handler("ultra", ctx); + await events.get("input")({ text: "normal mode", source: "interactive" }, ctx); + + const disabled = await events.get("before_agent_start")({ systemPrompt: "BASE" }, ctx); + assert.equal(disabled, undefined); +})); + +test("a request mentioning normal mode stays active", async () => withTempConfig(async () => { + const { commands, events } = createPiHarness(); + const ctx = createCommandContext(); + + await events.get("session_start")({ reason: "startup" }, ctx); + await commands.get("ponytail").handler("ultra", ctx); + await events.get("input")({ text: "add a normal mode toggle next to dark mode", source: "interactive" }, ctx); + + const result = await events.get("before_agent_start")({ systemPrompt: "BASE" }, ctx); + assert.match(result.systemPrompt, /PONYTAIL MODE ACTIVE/); +})); + +test("status bar renders the mode and flips active on agent_start", async () => withTempConfig(async () => { + const { events } = createPiHarness(); + const statusWrites = []; + const ctx = createCommandContext({ + sessionManager: { getEntries: () => [{ type: "custom", customType: "ponytail-mode", data: { mode: "ultra" } }] }, + ui: { notify() {}, setStatus: (key, text) => statusWrites.push({ key, text }), theme: { fg: (_color, text) => text } }, + }); + + await events.get("session_start")({ reason: "resume" }, ctx); + await events.get("agent_start")({}, ctx); + + assert.equal(statusWrites.at(-2).key, "ponytail"); + assert.match(statusWrites.at(-2).text, /○.*ULTRA/); + assert.match(statusWrites.at(-1).text, /●.*ULTRA/); +})); + +test("status bar stays silent when ui lacks a theme", async () => withTempConfig(async () => { + const { events } = createPiHarness(); + const calls = []; + const ctx = createCommandContext({ + sessionManager: { getEntries: () => [{ type: "custom", customType: "ponytail-mode", data: { mode: "ultra" } }] }, + ui: { notify() {}, setStatus: (_key, text) => calls.push(text) }, // setStatus present, theme absent + }); + + await events.get("session_start")({ reason: "resume" }, ctx); + await events.get("agent_start")({}, ctx); + + assert.deepEqual(calls, []); +})); + +test("PONYTAIL_HIDE_STATUS hides the indicator but keeps ponytail active (#324)", async () => withTempConfig(async () => { + process.env.PONYTAIL_HIDE_STATUS = "1"; + const { events } = createPiHarness(); + const statusWrites = []; + const ctx = createCommandContext({ + sessionManager: { getEntries: () => [{ type: "custom", customType: "ponytail-mode", data: { mode: "ultra" } }] }, + ui: { notify() {}, setStatus: (key, text) => statusWrites.push({ key, text }), theme: { fg: (_c, t) => t } }, + }); + + await events.get("session_start")({ reason: "resume" }, ctx); + await events.get("agent_start")({}, ctx); + const injected = await events.get("before_agent_start")({ systemPrompt: "BASE" }, ctx); + + assert.deepEqual(statusWrites, [], "status bar must not be drawn when hidden"); + assert.match(injected.systemPrompt, /PONYTAIL MODE ACTIVE/, "ruleset must still inject while status is hidden"); +})); + +test("config.hideStatus hides the indicator but keeps ponytail active (#324)", async () => withTempConfig(async () => { + mkdirSync(join(process.env.XDG_CONFIG_HOME, "ponytail"), { recursive: true }); + writeFileSync(join(process.env.XDG_CONFIG_HOME, "ponytail", "config.json"), JSON.stringify({ hideStatus: true })); + const { events } = createPiHarness(); + const statusWrites = []; + const ctx = createCommandContext({ + ui: { notify() {}, setStatus: (key, text) => statusWrites.push({ key, text }), theme: { fg: (_c, t) => t } }, + }); + + await events.get("session_start")({ reason: "startup" }, ctx); + await events.get("agent_start")({}, ctx); + const injected = await events.get("before_agent_start")({ systemPrompt: "BASE" }, ctx); + + assert.deepEqual(statusWrites, [], "config.hideStatus must suppress the status bar"); + assert.match(injected.systemPrompt, /PONYTAIL MODE ACTIVE/, "ruleset must still inject while status is hidden"); +})); + +test("PONYTAIL_HIDE_STATUS=0 does not hide the indicator", async () => withTempConfig(async () => { + process.env.PONYTAIL_HIDE_STATUS = "0"; + const { events } = createPiHarness(); + const statusWrites = []; + const ctx = createCommandContext({ + sessionManager: { getEntries: () => [{ type: "custom", customType: "ponytail-mode", data: { mode: "ultra" } }] }, + ui: { notify() {}, setStatus: (key, text) => statusWrites.push({ key, text }), theme: { fg: (_c, t) => t } }, + }); + + await events.get("session_start")({ reason: "resume" }, ctx); + await events.get("agent_start")({}, ctx); + + assert.ok(statusWrites.length > 0, "0 must be treated as 'do not hide'"); +})); diff --git a/ponytail/pi-extension/test/helpers.test.js b/ponytail/pi-extension/test/helpers.test.js new file mode 100644 index 0000000000000000000000000000000000000000..00258e233a7784a23fef91dc3d573cb2d2674256 --- /dev/null +++ b/ponytail/pi-extension/test/helpers.test.js @@ -0,0 +1,151 @@ +import assert from "node:assert/strict"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { + filterSkillBodyForMode, + parsePonytailCommand, + readDefaultMode, + readQuietStartup, + resolveSessionMode, + writeDefaultMode, +} from "../index.js"; + +test("parsePonytailCommand falls back to full when invoked bare and default is off", () => { + assert.deepEqual(parsePonytailCommand("", "off"), { type: "set-mode", mode: "full" }); +}); + +test("parsePonytailCommand parses modes, status, and default subcommand", () => { + assert.deepEqual(parsePonytailCommand("ultra", "full"), { type: "set-mode", mode: "ultra" }); + assert.deepEqual(parsePonytailCommand("status", "full"), { type: "status" }); + assert.deepEqual(parsePonytailCommand("default lite", "full"), { type: "set-default", mode: "lite" }); +}); + +test("parsePonytailCommand rejects review as a default (session-only mode, #377)", () => { + assert.deepEqual(parsePonytailCommand("default review", "full"), { type: "invalid", reason: "invalid-default-mode" }); +}); + +test("resolveSessionMode still honors review as a session mode (not a default)", () => { + const entries = [{ type: "custom", customType: "ponytail-mode", data: { mode: "review" } }]; + assert.equal(resolveSessionMode(entries, "full"), "review"); +}); + +test("resolveSessionMode prefers latest persisted session mode", () => { + const entries = [ + { type: "custom", customType: "ponytail-mode", data: { mode: "lite" } }, + { type: "custom", customType: "ponytail-mode", data: { mode: "ultra" } }, + ]; + + assert.equal(resolveSessionMode(entries, "full"), "ultra"); +}); + +test("resolveSessionMode returns fallback when entries is not an array", () => { + assert.equal(resolveSessionMode(null, "ultra"), "ultra"); + assert.equal(resolveSessionMode(undefined, "lite"), "lite"); + assert.equal(resolveSessionMode({}, "full"), "full"); + assert.equal(resolveSessionMode("not an array"), "full"); // DEFAULT_MODE fallback +}); + +test("readDefaultMode and writeDefaultMode use XDG config path", () => { + const tempDir = mkdtempSync(join(tmpdir(), "ponytail-config-")); + const previousXdg = process.env.XDG_CONFIG_HOME; + const previousDefault = process.env.PONYTAIL_DEFAULT_MODE; + const configPath = join(tempDir, "ponytail", "config.json"); + process.env.XDG_CONFIG_HOME = tempDir; + delete process.env.PONYTAIL_DEFAULT_MODE; + + try { + assert.equal(readDefaultMode(), "full"); + assert.equal(writeDefaultMode("ultra"), "ultra"); + assert.equal(readDefaultMode(), "ultra"); + assert.ok(existsSync(configPath)); + assert.deepEqual(JSON.parse(readFileSync(configPath, "utf8")), { defaultMode: "ultra" }); + } finally { + if (previousXdg === undefined) delete process.env.XDG_CONFIG_HOME; + else process.env.XDG_CONFIG_HOME = previousXdg; + if (previousDefault === undefined) delete process.env.PONYTAIL_DEFAULT_MODE; + else process.env.PONYTAIL_DEFAULT_MODE = previousDefault; + rmSync(tempDir, { recursive: true, force: true }); + } +}); + +test("readQuietStartup resolves env var, config file, and default in that order", () => { + const tempDir = mkdtempSync(join(tmpdir(), "ponytail-quiet-")); + const previousXdg = process.env.XDG_CONFIG_HOME; + const previousEnv = process.env.PONYTAIL_QUIET_STARTUP; + const configDir = join(tempDir, "ponytail"); + const configPath = join(configDir, "config.json"); + process.env.XDG_CONFIG_HOME = tempDir; + delete process.env.PONYTAIL_QUIET_STARTUP; + + try { + // No env, no config -> default false (toast still shows) + assert.equal(readQuietStartup(), false); + + // Config file true -> respected + mkdirSync(configDir, { recursive: true }); + writeFileSync(configPath, JSON.stringify({ quietStartup: true }), "utf8"); + assert.equal(readQuietStartup(), true); + + // Env var overrides config + process.env.PONYTAIL_QUIET_STARTUP = "false"; + assert.equal(readQuietStartup(), false); + process.env.PONYTAIL_QUIET_STARTUP = "1"; + assert.equal(readQuietStartup(), true); + } finally { + if (previousXdg === undefined) delete process.env.XDG_CONFIG_HOME; + else process.env.XDG_CONFIG_HOME = previousXdg; + if (previousEnv === undefined) delete process.env.PONYTAIL_QUIET_STARTUP; + else process.env.PONYTAIL_QUIET_STARTUP = previousEnv; + rmSync(tempDir, { recursive: true, force: true }); + } +}); + +test("filterSkillBodyForMode keeps only requested intensity examples and rows", () => { + // Examples are quoted in the real SKILL.md (`- lite: "..."`) — match that + // shape here too; see the next test for why the quote is load-bearing. + const body = `---\nname: ponytail\n---\n| **lite** | keep lite |\n| **full** | keep full |\n| **ultra** | keep ultra |\n- lite: "Lite example"\n- full: "Full example"\n- ultra: "Ultra example"\nOther line`; + + const filtered = filterSkillBodyForMode(body, "ultra"); + + assert.ok(!filtered.includes("keep lite")); + assert.ok(!filtered.includes("keep full")); + assert.ok(filtered.includes("keep ultra")); + assert.ok(!filtered.includes("Lite example")); + assert.ok(filtered.includes("Ultra example")); + assert.ok(filtered.includes("Other line")); +}); + +test("filterSkillBodyForMode does not drop a rule bullet whose label matches a mode name", () => { + // A rule bullet like "- Full: ..." has the same "label: text" shape as a + // worked example, but isn't one — it must survive in every mode. Only the + // quoted, `- lite: "..."`-style bullets are real per-mode examples. + const body = `- Full: do not confuse this rule label with the mode name.\n- Lite: same risk, this is a real rule bullet.\n- lite: "real worked example"\n- ultra: "real worked example"`; + + const filtered = filterSkillBodyForMode(body, "ultra"); + + assert.ok(filtered.includes("Full: do not confuse"), "an unquoted rule bullet must not be treated as a mode example"); + assert.ok(filtered.includes("Lite: same risk"), "an unquoted rule bullet must not be treated as a mode example"); + assert.ok(!filtered.includes("- lite:"), "the real quoted lite example must still be filtered out in ultra mode"); + assert.ok(filtered.includes('ultra: "real worked example"')); +}); + +test("filterSkillBodyForMode keeps rule bullets that contain a colon", () => { + // Regression: rule bullets outside the Intensity section (e.g. the + // "No unrequested abstractions:" rule or the `ponytail:` comment convention) + // contain a colon and must not be mistaken for mode-example lines. + const skillPath = new URL("../../skills/ponytail/SKILL.md", import.meta.url); + const body = readFileSync(skillPath, "utf8"); + + const filtered = filterSkillBodyForMode(body, "full"); + + assert.ok(filtered.includes("No unrequested abstractions")); + assert.ok(filtered.includes("Mark deliberate simplifications that cut a real corner")); + assert.ok(filtered.includes("`ponytail:` comment naming the ceiling and upgrade path")); + // The Intensity examples are still filtered down to the active mode. + assert.ok(filtered.includes('full: "`@lru_cache')); + assert.ok(!filtered.includes('lite: "Done')); + assert.ok(!filtered.includes('ultra: "No cache')); +}); diff --git a/ponytail/plugin.json b/ponytail/plugin.json new file mode 100644 index 0000000000000000000000000000000000000000..4eea40ee5baec7ea3d092f67f3c39a94c30ae9c2 --- /dev/null +++ b/ponytail/plugin.json @@ -0,0 +1,3 @@ +{ + "name": "ponytail" +} diff --git a/ponytail/plugin.yaml b/ponytail/plugin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cea98362fbbfc1f65807fa84a4a50dc1a777e7b7 --- /dev/null +++ b/ponytail/plugin.yaml @@ -0,0 +1,21 @@ +name: ponytail +version: 4.9.0 +description: Lazy senior dev mode for Hermes Agent, always-on context, bundled skills, and slash commands. +author: Dietrich Gebert +provides_hooks: + - pre_llm_call + - pre_gateway_dispatch +provides_commands: + - ponytail + - ponytail-review + - ponytail-audit + - ponytail-debt + - ponytail-gain + - ponytail-help +provides_skills: + - ponytail + - ponytail-review + - ponytail-audit + - ponytail-debt + - ponytail-gain + - ponytail-help diff --git a/ponytail/ponytail-mcp/README.md b/ponytail/ponytail-mcp/README.md new file mode 100644 index 0000000000000000000000000000000000000000..17dc67ea92b4d6694d72933100773f5f200db22b --- /dev/null +++ b/ponytail/ponytail-mcp/README.md @@ -0,0 +1,46 @@ +# ponytail-mcp + +An MCP server that serves Ponytail's lazy-senior-dev instructions. It exposes +the same ruleset the Claude hooks and Pi extension use, so every host emits +identical rules. + +It is not a replacement for the always-on adapters. Ponytail normally lives in +the system context every turn. MCP prompts are user-invoked, and there is no +portable MCP primitive for "inject this into every turn" across hosts. So this +server is the clean option for MCP hosts whose only injection point is the +prompt menu, or that pull context through tools. See issue #70. + +## What it exposes + +- Prompt `ponytail`, returns the ruleset as a user message. Optional `mode` + argument: `lite`, `full`, or `ultra`. Omit it to use the configured default. +- Tool `ponytail_instructions`, same text, plus `structuredContent` + (`{ mode, instructions }`), for hosts that pull context via tools or code + execution. Read-only. + +Mode resolution reuses `hooks/ponytail-config.js`, so `PONYTAIL_DEFAULT_MODE` +and `~/.config/ponytail/config.json` work the same as everywhere else. + +## Run it + +```bash +cd ponytail-mcp +npm install +node index.js # speaks MCP over stdio +``` + +Point an MCP host at that command. Example client entry: + +```json +{ "mcpServers": { "ponytail": { "command": "node", "args": ["ponytail-mcp/index.js"] } } } +``` + +## Test + +```bash +npm test +``` + +Covers mode resolution and the instruction text. The MCP wiring in `index.js` +is intentionally thin: it just maps the prompt and tool onto +`buildInstructions`. diff --git a/ponytail/ponytail-mcp/index.js b/ponytail/ponytail-mcp/index.js new file mode 100644 index 0000000000000000000000000000000000000000..e4a2895724c644c23cb90a0397da2777d3d60693 --- /dev/null +++ b/ponytail/ponytail-mcp/index.js @@ -0,0 +1,52 @@ +#!/usr/bin/env node +// Ponytail MCP server: serves the lazy-senior-dev ruleset over stdio as a +// prompt (user-invoked) and a tool (for hosts that pull context via tools). +// It does NOT replace the always-on adapters; it's the clean option for hosts +// whose only injection point is the prompt menu (see #70). +import fs from "node:fs"; +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; +import { z } from "zod"; + +import { MODES, buildInstructions, resolveMode } from "./instructions.js"; + +const { version } = JSON.parse( + await fs.promises.readFile(new URL("../package.json", import.meta.url), "utf8") +); +const server = new McpServer({ name: "ponytail", version }); + +const modeArg = z + .enum(MODES) + .optional() + .describe("Ponytail intensity: lite, full, or ultra. Omit for the configured default."); + +server.registerPrompt( + "ponytail", + { + title: "Ponytail mode", + description: "Lazy senior dev instructions: YAGNI, stdlib first, the smallest correct change.", + argsSchema: { mode: modeArg }, + }, + ({ mode }) => ({ + messages: [{ role: "user", content: { type: "text", text: buildInstructions(mode) } }], + }), +); + +server.registerTool( + "ponytail_instructions", + { + title: "Ponytail instructions", + description: "Return the Ponytail ruleset for the given intensity (lite, full, or ultra).", + inputSchema: { mode: modeArg }, + outputSchema: { mode: z.string(), instructions: z.string() }, + annotations: { readOnlyHint: true, openWorldHint: false }, + }, + ({ mode }) => { + const resolvedMode = resolveMode(mode); + const instructions = buildInstructions(resolvedMode); + const structuredContent = { mode: resolvedMode, instructions }; + return { content: [{ type: "text", text: instructions }], structuredContent }; + }, +); + +await server.connect(new StdioServerTransport()); diff --git a/ponytail/ponytail-mcp/instructions.js b/ponytail/ponytail-mcp/instructions.js new file mode 100644 index 0000000000000000000000000000000000000000..1cb1c73928db564fcad77b2b737cc940eafa7c74 --- /dev/null +++ b/ponytail/ponytail-mcp/instructions.js @@ -0,0 +1,26 @@ +// Pure instruction selection for the Ponytail MCP server. No MCP/SDK imports, +// so this stays unit-testable on its own. Reuses the same builder the Claude +// hooks and Pi extension use, so every host emits identical rules. +import { createRequire } from "node:module"; + +const require = createRequire(import.meta.url); +const { getPonytailInstructions } = require("../hooks/ponytail-instructions.js"); +const { getDefaultMode, normalizeMode } = require("../hooks/ponytail-config.js"); + +// The three intensities the server offers. "off" has no instructions to serve. +export const MODES = ["lite", "full", "ultra"]; + +// Resolve a requested mode to a runtime intensity. Unknown, empty, or "off" +// falls back to the configured default, then to "full". +// ponytail: keep the surface to these three; "off"/"review" aren't served here. +export function resolveMode(requested) { + const asked = normalizeMode(requested); + if (asked && asked !== "off") return asked; + + const fallback = normalizeMode(getDefaultMode()); + return fallback && fallback !== "off" ? fallback : "full"; +} + +export function buildInstructions(requested) { + return getPonytailInstructions(resolveMode(requested)); +} diff --git a/ponytail/ponytail-mcp/package.json b/ponytail/ponytail-mcp/package.json new file mode 100644 index 0000000000000000000000000000000000000000..1efaad7ac1fdb11a5b297cd6180ff3e0511b642f --- /dev/null +++ b/ponytail/ponytail-mcp/package.json @@ -0,0 +1,13 @@ +{ + "name": "ponytail-mcp", + "version": "4.9.0", + "description": "MCP server that serves Ponytail's lazy-senior-dev instructions as a prompt and a tool.", + "private": true, + "type": "module", + "license": "MIT", + "scripts": { "test": "node --test ./test/*.test.js" }, + "dependencies": { + "@modelcontextprotocol/sdk": "^1.26.0", + "zod": "^3.23.0" + } +} diff --git a/ponytail/ponytail-mcp/test/instructions.test.js b/ponytail/ponytail-mcp/test/instructions.test.js new file mode 100644 index 0000000000000000000000000000000000000000..5c6125a9d2e3cd936d7ed650293a1c20bc406d42 --- /dev/null +++ b/ponytail/ponytail-mcp/test/instructions.test.js @@ -0,0 +1,22 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { MODES, resolveMode, buildInstructions } from "../instructions.js"; + +test("resolveMode keeps valid intensities", () => { + for (const mode of MODES) assert.equal(resolveMode(mode), mode); +}); + +test("resolveMode falls back to a runtime intensity for off/unknown/empty", () => { + // PONYTAIL_DEFAULT_MODE could be anything in CI, so just assert the contract: + // never returns "off", "review", or junk — always one of the served modes. + for (const input of ["off", "review", "nonsense", "", undefined, null]) { + assert.ok(MODES.includes(resolveMode(input)), `resolveMode(${input}) must be a served mode`); + } +}); + +test("buildInstructions returns the ruleset tagged with the resolved mode", () => { + const text = buildInstructions("ultra"); + assert.match(text, /PONYTAIL MODE ACTIVE/); + assert.match(text, /ultra/); +}); diff --git a/ponytail/scripts/build-openclaw-skills.js b/ponytail/scripts/build-openclaw-skills.js new file mode 100644 index 0000000000000000000000000000000000000000..29318031a7e213213835b96f5acd19d127da0aec --- /dev/null +++ b/ponytail/scripts/build-openclaw-skills.js @@ -0,0 +1,60 @@ +#!/usr/bin/env node +// Generate the OpenClaw / ClawHub skill package (.openclaw/skills/) from the +// canonical skills/. OpenClaw skills are SKILL.md (frontmatter + body), the same +// format ponytail already uses, with one difference: `description` must be a +// single line under 160 chars. The canonical descriptions are long (tuned for +// Claude's skill picker), so each ships a short one here. The body is copied +// verbatim from skills//SKILL.md so the ruleset never drifts; only the +// frontmatter is rewritten. +// +// Run: node scripts/build-openclaw-skills.js +// tests/openclaw-skills.test.js fails if the committed copies are stale. + +const fs = require('fs'); +const path = require('path'); + +const ROOT = path.join(__dirname, '..'); +const HOMEPAGE = 'https://github.com/DietrichGebert/ponytail'; + +const DESCRIPTIONS = { + 'ponytail': 'Lazy senior dev mode for any coding task (write, refactor, fix, review): YAGNI, stdlib first, no unrequested abstractions. Not for non-coding requests.', + 'ponytail-review': 'Review a diff for over-engineering. Finds what to delete: reinvented stdlib, needless deps, speculative abstractions. One line per finding.', + 'ponytail-audit': 'Audit the whole repo for over-engineering. A ranked list of what to delete, simplify, or replace with stdlib or native features.', + 'ponytail-debt': 'Harvest every ponytail: shortcut comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.', + 'ponytail-gain': 'Show ponytail measured impact as a scoreboard: less code, less cost, more speed, from the benchmark medians. One-shot display.', + 'ponytail-help': "Quick reference for ponytail's modes, skills, and commands. One-shot display.", +}; + +const NAMES = Object.keys(DESCRIPTIONS); + +function sourceBody(name) { + const src = fs.readFileSync(path.join(ROOT, 'skills', name, 'SKILL.md'), 'utf8').replace(/\r\n/g, '\n'); + const fm = src.match(/^---\n[\s\S]*?\n---\n?/); + if (!fm) throw new Error(`skills/${name}/SKILL.md has no frontmatter`); + return src.slice(fm[0].length); +} + +function render(name) { + const desc = DESCRIPTIONS[name]; + if (desc.length > 160 || desc.includes('\n') || desc.includes('"')) { + throw new Error(`description for ${name} must be one line, no quotes, under 160 chars`); + } + const frontmatter = + `---\nname: ${name}\ndescription: "${desc}"\nhomepage: ${HOMEPAGE}\nlicense: MIT\n---\n`; + return frontmatter + sourceBody(name); +} + +function outPath(name) { + return path.join(ROOT, '.openclaw', 'skills', name, 'SKILL.md'); +} + +module.exports = { DESCRIPTIONS, NAMES, render, outPath, sourceBody }; + +if (require.main === module) { + for (const name of NAMES) { + const p = outPath(name); + fs.mkdirSync(path.dirname(p), { recursive: true }); + fs.writeFileSync(p, render(name)); + console.log('wrote', path.relative(ROOT, p).replace(/\\/g, '/')); + } +} diff --git a/ponytail/scripts/check-rule-copies.js b/ponytail/scripts/check-rule-copies.js new file mode 100644 index 0000000000000000000000000000000000000000..a26c12f1d9087f13f2b78378002a8d302427e44b --- /dev/null +++ b/ponytail/scripts/check-rule-copies.js @@ -0,0 +1,76 @@ +#!/usr/bin/env node +const fs = require('fs'); +const path = require('path'); + +const root = path.join(__dirname, '..'); + +function read(relPath) { + return fs.readFileSync(path.join(root, relPath), 'utf8').replace(/\r\n/g, '\n').trim(); +} + +function stripFrontmatter(text) { + return text.replace(/^---\n[\s\S]*?\n---\n*/, '').trim(); +} + +const agents = read('AGENTS.md'); +const canonical = agents.replace(/\n\n\(Yes, this file also applies[\s\S]*?\)$/, '').trim(); + +// Compact copies: same body as AGENTS.md, host-specific frontmatter stripped. +const copies = [ + ['.cursor/rules/ponytail.mdc', stripFrontmatter], + ['.windsurf/rules/ponytail.md', text => text.trim()], + ['.clinerules/ponytail.md', text => text.trim()], + ['.agents/rules/ponytail.md', text => text.trim()], + ['.qoder/rules/ponytail.md', text => text.trim()], + ['.github/copilot-instructions.md', text => text.trim()], + ['.kiro/steering/ponytail.md', stripFrontmatter], +]; + +let failed = false; + +for (const [relPath, normalize] of copies) { + const actual = normalize(read(relPath)); + if (actual !== canonical) { + console.error(`${relPath} drifted from AGENTS.md`); + failed = true; + } +} + +// SKILL.md is the runtime source of truth and is longer than the compact body, +// so it cannot be byte-compared. ponytail: canary, not full equality. Assert the +// load-bearing rules survive verbatim in both the source and AGENTS.md. Changing +// a rule's wording trips this, which is the reminder to propagate it everywhere. +// Upgrade path: generate the copies from SKILL.md if this ever misses a real drift. +const INVARIANTS = [ + 'in this codebase', // ladder rung: reuse what already exists (#217) + 'naive heuristic', // ceiling-comment rule + 'ONE runnable check', // test reflex + 'flimsier algorithm', // robust-variant rule + // the four "not lazy about" safety carve-outs: pin each so a reword in either + // file can't silently drop one. Only validation was pinned before. These are the + // continuous substrings present in both files ("prevents data loss" because the + // full "error handling that prevents data loss" wraps a line in SKILL.md). + 'input validation at trust boundaries', + 'prevents data loss', + 'security', + 'accessibility', + 'Lazy code without its check is unfinished', // one-check promoted to headline +]; + +const skill = read('skills/ponytail/SKILL.md'); +const sources = [['skills/ponytail/SKILL.md', skill], ['AGENTS.md', agents]]; +for (const phrase of INVARIANTS) { + for (const [label, text] of sources) { + if (!text.includes(phrase)) { + console.error(`${label} is missing rule invariant: "${phrase}"`); + failed = true; + } + } +} + +if (failed) { + console.error('Update the copied rule text, AGENTS.md, or SKILL.md so the shared rules match.'); + process.exit(1); +} + +console.log(`Rule copies match AGENTS.md; ${INVARIANTS.length} rule invariants present in SKILL.md and AGENTS.md.`); diff --git a/ponytail/scripts/check-versions.js b/ponytail/scripts/check-versions.js new file mode 100644 index 0000000000000000000000000000000000000000..b2b0d6fcd1b718c8676101dc0825a22da2cdea33 --- /dev/null +++ b/ponytail/scripts/check-versions.js @@ -0,0 +1,78 @@ +#!/usr/bin/env node +// Version-consistency guard. Ponytail declares its version in seven files across +// five host ecosystems, and every release bumps all of them by hand. +// +// tests/gemini-extension.test.js already checks the four plugin manifests agree +// with each other, but that can't catch the failure mode that shipped in v4.8.0: +// every manifest stayed stale at 4.7.0 *together* while the release moved on, so +// they "agreed" and the test passed (#260, #262). It also ignores the two +// package.json files. This check closes both gaps: +// 1. every version-bearing file must share one pinned X.Y.Z version, and +// 2. on a release-tag CI run, that shared version must equal the tag. + +const fs = require('fs'); +const path = require('path'); + +const root = path.join(__dirname, '..'); +const PINNED_SEMVER = /^\d+\.\d+\.\d+$/; + +// Every file that declares the project version, and who reads it. Add new host +// manifests here so a future ecosystem can't drift unnoticed. +const VERSION_FILES = [ + '.claude-plugin/plugin.json', // Claude Code plugin — what users install + '.codex-plugin/plugin.json', // Codex plugin + '.devin-plugin/plugin.json', // Devin CLI plugin + '.github/plugin/plugin.json', // Copilot plugin + '.qoder-plugin/plugin.json', // Qoder plugin + 'gemini-extension.json', // Gemini CLI extension + 'package.json', // pi-package / repo root + 'ponytail-mcp/package.json', // MCP server (private, internal-only) +]; + +function readVersion(relPath) { + try { + // Strip a UTF-8 BOM some Windows editors prepend (breaks JSON.parse). + const raw = fs.readFileSync(path.join(root, relPath), 'utf8').replace(/^\uFEFF/, ''); + return JSON.parse(raw).version; + } catch (e) { + throw new Error(`${relPath}: ${e.message}`); + } +} + +let failed = false; +const versions = VERSION_FILES.map((relPath) => { + const version = readVersion(relPath); + if (typeof version !== 'string' || !PINNED_SEMVER.test(version)) { + console.error(`${relPath}: version must be a pinned X.Y.Z semver, got ${JSON.stringify(version)}`); + failed = true; + } + return [relPath, version]; +}); + +// Every file must declare the same version. +const distinct = [...new Set(versions.map(([, v]) => v))]; +if (distinct.length > 1) { + console.error('Version mismatch — every manifest must share one version:'); + for (const [relPath, version] of versions) console.error(` ${version}\t${relPath}`); + failed = true; +} +const shared = distinct.length === 1 ? distinct[0] : null; + +// On a release-tag push CI sets GITHUB_REF_TYPE=tag and GITHUB_REF_NAME=vX.Y.Z. +// The shared version must equal the tag — this catches tagging a release whose +// version files were never bumped, which mutual agreement alone cannot. +if (shared && process.env.GITHUB_REF_TYPE === 'tag') { + const tag = process.env.GITHUB_REF_NAME || ''; + const tagVersion = tag.replace(/^v/, ''); + if (PINNED_SEMVER.test(tagVersion) && tagVersion !== shared) { + console.error(`release tag ${tag} does not match version ${shared}; bump the version files before tagging`); + failed = true; + } +} + +if (failed) { + console.error('Align the version fields (see issue #260) so every manifest shares one version.'); + process.exit(1); +} + +console.log(`All ${VERSION_FILES.length} version files pinned at ${shared}.`); diff --git a/ponytail/scripts/publish-openclaw-skills.js b/ponytail/scripts/publish-openclaw-skills.js new file mode 100644 index 0000000000000000000000000000000000000000..1df9d1df48b3883f66b24e784f15b28488f97a45 --- /dev/null +++ b/ponytail/scripts/publish-openclaw-skills.js @@ -0,0 +1,75 @@ +#!/usr/bin/env node +// Publish the generated OpenClaw skills (.openclaw/skills/) to ClawHub. +// +// ClawHub does not sync from GitHub: each skill is pushed explicitly with the +// clawhub CLI and carries its own version. This publishes every generated skill +// in one pass, versioned from the repo's package.json so ClawHub tracks the repo +// instead of drifting (the same drift that hit the plugin manifests in #260). +// +// Prereqs: +// - `clawhub login` once (registry auth persists) +// - skills must be current: run `node scripts/build-openclaw-skills.js` first +// if you changed a skill (CI fails if the committed copies are stale) +// +// Usage: +// node scripts/publish-openclaw-skills.js # publish all as latest +// node scripts/publish-openclaw-skills.js --dry-run # preview, upload nothing +// (any extra args are passed through to `clawhub skill publish`) + +const fs = require('fs'); +const path = require('path'); +const { spawnSync } = require('child_process'); + +const root = path.join(__dirname, '..'); +const skillsDir = path.join(root, '.openclaw', 'skills'); + +const version = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8')).version; + +// Every generated skill dir with a SKILL.md is publishable. Reading the dir +// (instead of a hardcoded list) covers whatever build-openclaw-skills emits, +// with nothing to keep in sync. +const slugs = fs.readdirSync(skillsDir, { withFileTypes: true }) + .filter((e) => e.isDirectory() && fs.existsSync(path.join(skillsDir, e.name, 'SKILL.md'))) + .map((e) => e.name) + .sort(); + +if (slugs.length === 0) { + console.error(`No skills under ${path.relative(root, skillsDir)}; run build-openclaw-skills.js first.`); + process.exit(1); +} + +// "ponytail-review" -> "Ponytail Review" +const displayName = (slug) => + slug.split('-').map((w) => w.charAt(0).toUpperCase() + w.slice(1)).join(' '); + +// Minimal quoting that satisfies both POSIX sh and cmd.exe: only display names +// (which contain a space) need wrapping; slugs, versions, paths, and flags don't. +const quote = (a) => (/[^\w./-]/.test(a) ? `"${a}"` : a); + +const passthrough = process.argv.slice(2); +const extra = passthrough.length ? ` (${passthrough.join(' ')})` : ''; +console.log(`Publishing ${slugs.length} skills to ClawHub at version ${version}${extra}:`); + +for (const slug of slugs) { + const args = [ + 'clawhub', 'skill', 'publish', `.openclaw/skills/${slug}`, + '--slug', slug, + '--name', displayName(slug), + '--version', version, + '--tags', 'latest', + ...passthrough, + ]; + const cmdline = args.map(quote).join(' '); + console.log(`\n$ ${cmdline}`); + const res = spawnSync(cmdline, { stdio: 'inherit', cwd: root, shell: true }); + if (res.status !== 0) { + console.error( + `\nPublish failed for "${slug}" (exit ${res.status}). ` + + `Check that the clawhub CLI is installed and you have run \`clawhub login\`, then re-run. ` + + `Skills already published in this run are unaffected.`, + ); + process.exit(res.status || 1); + } +} + +console.log(`\nDone. Published ${slugs.length} skills at ${version}.`); diff --git a/ponytail/scripts/uninstall.js b/ponytail/scripts/uninstall.js new file mode 100644 index 0000000000000000000000000000000000000000..761061ae8e63eb0a03572196a64c6e33c078e5c9 --- /dev/null +++ b/ponytail/scripts/uninstall.js @@ -0,0 +1,60 @@ +#!/usr/bin/env node +// ponytail — removes state ponytail wrote outside the plugin's own files: +// the mode flag, the config file, and the statusLine entry it added to +// settings.json. Plugin files themselves are removed by each host's own +// uninstall command (see README); this only cleans up what those commands +// can't see. + +const fs = require('fs'); +const path = require('path'); +const { getConfigPath, getClaudeDir } = require('../hooks/ponytail-config'); + +const STATUSLINE_SCRIPT = 'ponytail-statusline'; + +function removeIfExists(filePath, label) { + try { + fs.unlinkSync(filePath); + console.log(`Removed ${label}: ${filePath}`); + } catch (e) { + if (e.code !== 'ENOENT') throw e; + } +} + +removeIfExists(path.join(getClaudeDir(), '.ponytail-active'), 'mode flag'); +removeIfExists(getConfigPath(), 'config file'); + +const settingsPath = path.join(getClaudeDir(), 'settings.json'); +try { + const raw = fs.readFileSync(settingsPath, 'utf8').replace(/^\uFEFF/, ''); + const settings = JSON.parse(raw); + const cmd = settings.statusLine && settings.statusLine.command; + // Only remove the parts ponytail owns. If the user combined statuslines + // (e.g. caveman && ponytail), keep the other plugin's command intact. + // ponytail: splits on && / ; to detect other segments — good enough; a user + // piping statuslines together is on their own. + if (typeof cmd === 'string' && cmd.includes(STATUSLINE_SCRIPT)) { + const parts = cmd + .split(/&&|;/) + .map((s) => s.trim()) + .filter(Boolean); + const others = parts.filter((s) => !s.includes(STATUSLINE_SCRIPT)); + if (others.length === 0) { + delete settings.statusLine; + fs.writeFileSync(settingsPath, JSON.stringify(settings, null, 2), 'utf8'); + console.log(`Removed ponytail statusLine entry from ${settingsPath}`); + } else { + settings.statusLine.command = others.join(' && '); + fs.writeFileSync(settingsPath, JSON.stringify(settings, null, 2), 'utf8'); + console.log(`Removed ponytail statusLine segment from ${settingsPath}`); + } + } +} catch (e) { + if (e.code === 'ENOENT') { + // no settings.json — nothing to clean + } else if (e instanceof SyntaxError) { + // ponytail: malformed settings.json — can't safely edit it; leave intact, warn + console.warn(`settings.json is malformed — could not remove the ponytail statusLine entry. Remove it manually from: ${settingsPath} (${e.message})`); + } else { + throw e; + } +} diff --git a/ponytail/skills/ponytail-audit/SKILL.md b/ponytail/skills/ponytail-audit/SKILL.md new file mode 100644 index 0000000000000000000000000000000000000000..5582d10335daff5b5947f9b77927fbc97f2047f3 --- /dev/null +++ b/ponytail/skills/ponytail-audit/SKILL.md @@ -0,0 +1,41 @@ +--- +name: ponytail-audit +description: > + Whole-repo audit for over-engineering. Like ponytail-review, but scans the + entire codebase instead of a diff: a ranked list of what to delete, simplify, + or replace with stdlib/native equivalents. Use when the user says "audit this + codebase", "audit for over-engineering", "what can I delete from this repo", + "find bloat", "ponytail-audit", or "/ponytail-audit". One-shot report, does + not apply fixes. +--- + +ponytail-review, repo-wide. Scan the whole tree instead of a diff. Rank +findings biggest cut first. + +## Tags + +Same as ponytail-review: + +- `delete:` dead code, unused flexibility, speculative feature. Replacement: nothing. +- `stdlib:` hand-rolled thing the standard library ships. Name the function. +- `native:` dependency or code doing what the platform already does. Name the feature. +- `yagni:` abstraction with one implementation, config nobody sets, layer with one caller. +- `shrink:` same logic, fewer lines. Show the shorter form. + +## Hunt + +Deps the stdlib or platform already ships, single-implementation interfaces, +factories with one product, wrappers that only delegate, files exporting one +thing, dead flags and config, hand-rolled stdlib. + +## Output + +One line per finding, ranked: ` . . [path]`. +End with `net: - lines, - deps possible.` Nothing to cut: `Lean already. Ship.` + +## Boundaries + +Scope: over-engineering and complexity only. Correctness bugs, security holes, +and performance are explicitly out of scope. Route them to a normal review +pass. Lists findings, applies nothing. One-shot. +"stop ponytail-audit" or "normal mode" to revert. diff --git a/ponytail/skills/ponytail-debt/SKILL.md b/ponytail/skills/ponytail-debt/SKILL.md new file mode 100644 index 0000000000000000000000000000000000000000..ecbc0ca8161b25ced3b4f728398c2ec33988a777 --- /dev/null +++ b/ponytail/skills/ponytail-debt/SKILL.md @@ -0,0 +1,44 @@ +--- +name: ponytail-debt +description: > + Harvest every `ponytail:` comment in the codebase into a debt ledger, so the + deliberate shortcuts and deferrals ponytail leaves behind get tracked instead + of rotting into "later means never". Use when the user says "ponytail debt", + "/ponytail-debt", "what did ponytail defer", "list the shortcuts", "ponytail + ledger", or "what did we mark to do later". One-shot report, changes nothing. +--- + +Every deliberate ponytail shortcut is marked with a `ponytail:` comment naming +its ceiling and upgrade path. This collects them into one ledger so a deferral +can't quietly become permanent. + +## Scan + +Grep the repo for comment markers, skipping `node_modules`, `.git`, and build +output: + +`grep -rnE '(#|//) ?ponytail:' .` (add other comment prefixes if your stack uses them) + +Each hit is one ledger row. The comment prefix keeps prose that merely mentions +the convention out of the ledger. + +## Output + +One row per marker, grouped by file: + +`:, . ceiling: . upgrade: .` + +The convention is `ponytail: , `, so pull the ceiling +and the trigger straight from the comment. Want an owner per row too? add +`git blame -L,`. + +Flag the rot risk: any `ponytail:` comment that names no upgrade path or +trigger gets a `no-trigger` tag, those are the ones that silently rot. + +End with ` markers, with no trigger.` Nothing found: `No ponytail: debt. Clean ledger.` + +## Boundaries + +Reads and reports only, changes nothing. To persist it, ask and it writes the +ledger to a file (e.g. `PONYTAIL-DEBT.md`). One-shot. "stop ponytail-debt" or +"normal mode" to revert. diff --git a/ponytail/skills/ponytail-gain/SKILL.md b/ponytail/skills/ponytail-gain/SKILL.md new file mode 100644 index 0000000000000000000000000000000000000000..012e37b6bf31da1ed4bf936ec8aff53974d5291e --- /dev/null +++ b/ponytail/skills/ponytail-gain/SKILL.md @@ -0,0 +1,50 @@ +--- +name: ponytail-gain +description: > + Show ponytail's measured impact as a compact scoreboard: less code, less + cost, more speed, from the benchmark medians. One-shot display, not a + persistent mode, and not a per-repo number. Trigger: /ponytail-gain, + "ponytail gain", "what does ponytail save", "show ponytail impact", + "ponytail scoreboard". +--- + +# Ponytail Gain + +Display this scoreboard when invoked. One-shot: do NOT change mode, write flag +files, or persist anything. + +The figures are the published benchmark medians (5 everyday tasks: email +validator, debounce, CSV sum, countdown timer, rate limiter; three models: +Haiku, Sonnet, Opus). They are measured, not computed from the current repo. +Source: `benchmarks/` and the README. + +## Scoreboard + +Render plain ASCII bars. The bar length shows the measured range; the label +carries the exact figure: + +``` + ponytail gain benchmark median · 5 tasks · 3 models + + Lines of code no-skill ████████████████████ 100% + ponytail ██▌················· 6–20% ▼ 80–94% + Cost no-skill ████████████████████ 100% + ponytail █████▌·············· 23–53% ▼ 47–77% + Speed ponytail ▸ 3–6× faster + + This repo: /ponytail-debt (shortcuts you deferred) + /ponytail-audit (what's still cuttable) +``` + +## Honesty boundary + +These are benchmark medians, not this repo. NEVER print a per-repo savings +number ("you saved X lines/tokens here"): the unbuilt version was never +written, so there is no real baseline to subtract from in a live repo. The +only real per-repo figures come from `/ponytail-debt` (a counted ledger), and +this card points there instead of inventing one. + +## Boundaries + +One-shot display. Edits nothing, changes no mode. +"stop ponytail" or "normal mode": revert. diff --git a/ponytail/skills/ponytail-help/SKILL.md b/ponytail/skills/ponytail-help/SKILL.md new file mode 100644 index 0000000000000000000000000000000000000000..ba145c0ebb7c7e5682bb2f36047af3bf2030d470 --- /dev/null +++ b/ponytail/skills/ponytail-help/SKILL.md @@ -0,0 +1,71 @@ +--- +name: ponytail-help +description: > + Quick-reference card for all ponytail modes, skills, and commands. + One-shot display, not a persistent mode. Trigger: /ponytail-help, + "ponytail help", "what ponytail commands", "how do I use ponytail". +--- + +# Ponytail Help + +Display this reference card when invoked. One-shot, do NOT change mode, +write flag files, or persist anything. + +## Levels + +| Level | Trigger | What change | +|-------|---------|-------------| +| **Lite** | `/ponytail lite` | Build what's asked, name the lazier alternative in one line. | +| **Full** | `/ponytail` | The ladder enforced: YAGNI → stdlib → native → one line → minimum. Default. | +| **Ultra** | `/ponytail ultra` | YAGNI extremist. Deletion before addition. Challenges requirements before building. | + +Level sticks until changed or session end. + +## Skills + +| Skill | Trigger | What it does | +|-------|---------|--------------| +| **ponytail** | `/ponytail` | Lazy mode itself. Simplest solution that works. | +| **ponytail-review** | `/ponytail-review` | Over-engineering review: `L42: yagni: factory, one product. Inline.` | +| **ponytail-audit** | `/ponytail-audit` | Whole-repo over-engineering audit: ranked list of what to delete. | +| **ponytail-debt** | `/ponytail-debt` | Harvest `ponytail:` shortcut comments into a tracked ledger. | +| **ponytail-gain** | `/ponytail-gain` | Measured-impact scoreboard: less code, less cost, more speed. | +| **ponytail-help** | `/ponytail-help` | This card. | + +Codex uses `@ponytail`, `@ponytail-review`, and `@ponytail-help`; Claude Code +and OpenCode use the slash-command forms above (OpenCode ships all six as +slash commands). + +## Deactivate + +Say "stop ponytail" or "normal mode". Resume anytime with `/ponytail`. +`/ponytail off` also works. + +## Configure Default Mode + +Default mode = `full`, auto-active every session. Change it: + +**Environment variable** (highest priority): +```bash +export PONYTAIL_DEFAULT_MODE=ultra +``` + +**Config file** (`~/.config/ponytail/config.json`, Windows: `%APPDATA%\ponytail\config.json`): +```json +{ "defaultMode": "lite" } +``` + +Set `"off"` to disable auto-activation on session start, activate manually +with `/ponytail` when wanted. + +Resolution: env var > config file > `full`. + +## Update + +Enable auto-update once: open `/plugin`, go to Marketplaces, pick ponytail, Enable auto-update. Claude Code then pulls new versions at startup (run `/reload-plugins` when it prompts). Manual refresh: `/plugin marketplace update ponytail` then `/reload-plugins`. + +If `/plugin` is not recognized, your Claude Code is out of date. Update it (`npm install -g @anthropic-ai/claude-code@latest`, or `brew upgrade claude-code`) and restart. Other hosts use their own update flow. + +## More + +Full docs + examples: https://github.com/DietrichGebert/ponytail diff --git a/ponytail/skills/ponytail-review/SKILL.md b/ponytail/skills/ponytail-review/SKILL.md new file mode 100644 index 0000000000000000000000000000000000000000..e137a855bd87119a4517895a1000a59b0999e1b8 --- /dev/null +++ b/ponytail/skills/ponytail-review/SKILL.md @@ -0,0 +1,57 @@ +--- +name: ponytail-review +description: > + Code review focused exclusively on over-engineering. Finds what to delete: + reinvented standard library, unneeded dependencies, speculative abstractions, + dead flexibility. One line per finding: location, what to cut, what replaces + it. Use when the user says "review for over-engineering", "what can we + delete", "is this over-engineered", "simplify review", or invokes + /ponytail-review. Complements correctness-focused review, this one only + hunts complexity. +--- + +Review diffs for unnecessary complexity. One line per finding: location, what +to cut, what replaces it. The diff's best outcome is getting shorter. + +## Format + +`L: . .`, or `:L: ...` for +multi-file diffs. + +Tags: + +- `delete:` dead code, unused flexibility, speculative feature. Replacement: nothing. +- `stdlib:` hand-rolled thing the standard library ships. Name the function. +- `native:` dependency or code doing what the platform already does. Name the feature. +- `yagni:` abstraction with one implementation, config nobody sets, layer with one caller. +- `shrink:` same logic, fewer lines. Show the shorter form. + +## Examples + +❌ "This EmailValidator class might be more complex than necessary, have you +considered whether all these validation rules are needed at this stage?" + +✅ `L12-38: stdlib: 27-line validator class. "@" in email, 1 line, real validation is the confirmation mail.` + +✅ `L4: native: moment.js imported for one format call. Intl.DateTimeFormat, 0 deps.` + +✅ `repo.py:L88: yagni: AbstractRepository with one implementation. Inline it until a second one exists.` + +✅ `L52-71: delete: retry wrapper around an idempotent local call. Nothing replaces it.` + +✅ `L30-44: shrink: manual loop builds dict. dict(zip(keys, values)), 1 line.` + +## Scoring + +End with the only metric that matters: `net: - lines possible.` + +If there is nothing to cut, say `Lean already. Ship.` and stop. + +## Boundaries + +Scope: over-engineering and complexity only. Correctness bugs, security holes, +and performance are explicitly out of scope. Route them to a normal review +pass, not this one. A single smoke test or `assert`-based +self-check is the ponytail minimum, not bloat, never flag it for deletion. +Does not apply the fixes, only lists them. +"stop ponytail-review" or "normal mode": revert to verbose review style. diff --git a/ponytail/skills/ponytail/SKILL.md b/ponytail/skills/ponytail/SKILL.md new file mode 100644 index 0000000000000000000000000000000000000000..02c0712c86277d49d18a77da3a2b825657bf02d1 --- /dev/null +++ b/ponytail/skills/ponytail/SKILL.md @@ -0,0 +1,120 @@ +--- +name: ponytail +description: > + Forces the laziest solution that actually works, simplest, shortest, most + minimal. Channels a senior dev who has seen everything: question whether the + task needs to exist at all (YAGNI), reach for the standard library before + custom code, native platform features before dependencies, one line before + fifty. Supports intensity levels: lite, full (default), ultra. Use on ANY + coding task: writing, adding, refactoring, fixing, reviewing, or designing + code, and choosing libraries or dependencies. Also use whenever the user + says "ponytail", "be lazy", "lazy mode", "simplest solution", "minimal + solution", "yagni", "do less", or "shortest path", or complains about + over-engineering, bloat, boilerplate, or unnecessary dependencies. Do NOT + use for non-coding requests (general knowledge, prose, translation, + summaries, recipes). +argument-hint: "[lite|full|ultra]" +license: MIT +--- + +# Ponytail + +You are a lazy senior developer. Lazy means efficient, not careless. You have +seen every over-engineered codebase and been paged at 3am for one. The best +code is the code never written. + +## Persistence + +ACTIVE EVERY RESPONSE. No drift back to over-building. Still active if +unsure. Off only: "stop ponytail" / "normal mode". Default: **full**. +Switch: `/ponytail lite|full|ultra`. + +## The ladder + +Stop at the first rung that holds: + +1. **Does this need to exist at all?** Speculative need = skip it, say so in one line. (YAGNI) +2. **Already in this codebase?** A helper, util, type, or pattern that already lives here → reuse it. Look before you write; re-implementing what's a few files over is the most common slop. +3. **Stdlib does it?** Use it. +4. **Native platform feature covers it?** `` over a picker lib, CSS over JS, DB constraint over app code. +5. **Already-installed dependency solves it?** Use it. Never add a new one for what a few lines can do. +6. **Can it be one line?** One line. +7. **Only then:** the minimum code that works. + +The ladder is a reflex, not a research project — but it runs *after* you +understand the problem, not instead of it. Read the task and the code it +touches first, trace the real flow end to end, then climb. Two rungs work → +take the higher one and move on. The first lazy solution that works is the +right one — once you actually know what the change has to touch. + +**Bug fix = root cause, not symptom.** A report names a symptom. Before you +edit, grep every caller of the function you're about to touch. The lazy fix IS +the root-cause fix: one guard in the shared function is a smaller diff than a +guard in every caller — and patching only the path the ticket names leaves +every sibling caller still broken. Fix it once, where all callers route through. + +## Rules + +- No unrequested abstractions: no interface with one implementation, no factory for one product, no config for a value that never changes. +- No boilerplate, no scaffolding "for later", later can scaffold for itself. +- Deletion over addition. Boring over clever, clever is what someone decodes at 3am. +- Fewest files possible. Shortest working diff wins — but only once you understand the problem. The smallest change in the wrong place isn't lazy, it's a second bug. +- Complex request? Ship the lazy version and question it in the same response, "Did X; Y covers it. Need full X? Say so." Never stall on an answer you can default. +- Two stdlib options, same size? Take the one that's correct on edge cases. Lazy means writing less code, not picking the flimsier algorithm. +- Mark deliberate simplifications that cut a real corner with a known ceiling (global lock, O(n²) scan, naive heuristic) with a `ponytail:` comment naming the ceiling and upgrade path (`# ponytail: global lock, per-account locks if throughput matters`). + +## Output + +Code first. Then at most three short lines: what was skipped, when to add it. +No essays, no feature tours, no design notes. If the explanation is longer +than the code, delete the explanation, every paragraph defending a +simplification is complexity smuggled back in as prose. Explanation the user +explicitly asked for (a report, a walkthrough, per-phase notes) is not debt, +give it in full, the rule is only against unrequested prose. + +Pattern: `[code] → skipped: [X], add when [Y].` + +## Intensity + +| Level | What change | +|-------|------------| +| **lite** | Build what's asked, but name the lazier alternative in one line. User picks. | +| **full** | The ladder enforced. Stdlib and native first. Shortest diff, shortest explanation. Default. | +| **ultra** | YAGNI extremist. Deletion before addition. Ship the one-liner and challenge the rest of the requirement in the same breath. | + +Example: "Add a cache for these API responses." +- lite: "Done, cache added. FYI: `functools.lru_cache` covers this in one line if you'd rather not own a cache class." +- full: "`@lru_cache(maxsize=1000)` on the fetch function. Skipped custom cache class, add when lru_cache measurably falls short." +- ultra: "No cache until a profiler says so. When it does: `@lru_cache`. A hand-rolled TTL cache class is a bug farm with a hit rate." + +## When NOT to be lazy + +Never simplify away: input validation at trust boundaries, error handling +that prevents data loss, security measures, accessibility basics, anything +explicitly requested. User insists on the full version → build it, no +re-arguing. + +Never lazy about understanding the problem. The ladder shortens the +solution, never the reading. Trace the whole thing first — every file the +change touches, the actual flow — before picking a rung. Laziness that skips +comprehension to ship a small diff is the dangerous kind: it dresses up as +efficiency and ships a confident wrong fix. Read fully, then be lazy. + +Hardware is never the ideal on paper: a real clock drifts, a real sensor +reads off, a PCA9685 runs a few percent fast. Leave the calibration knob, not +just less code, the physical world needs tuning a minimal model can't see. + +Lazy code without its check is unfinished. Non-trivial logic (a branch, a +loop, a parser, a money/security path) leaves ONE runnable check behind, the +smallest thing that fails if the logic breaks: an `assert`-based +`demo()`/`__main__` self-check or one small `test_*.py`. No frameworks, no +fixtures, no per-function suites unless asked. Trivial one-liners need no +test, YAGNI applies to tests too. + +## Boundaries + +Ponytail governs what you build, not how you talk (pair with Caveman for +terse prose). "stop ponytail" / "normal mode": revert. Level persists until +changed or session end. + +The shortest path to done is the right path. diff --git a/ponytail/tests/behavior.test.js b/ponytail/tests/behavior.test.js new file mode 100644 index 0000000000000000000000000000000000000000..04502cf047a8840f828cfd05e777ad9b544f62aa --- /dev/null +++ b/ponytail/tests/behavior.test.js @@ -0,0 +1,80 @@ +#!/usr/bin/env node +// Unit test for the behavior gate (benchmarks/behavior.js). Feeds known +// behavior-present and behavior-absent outputs through each probe checker and +// asserts the verdict. Runs without promptfoo or an API key — it proves the +// grader can tell the refined behavior from its absence, which is what makes +// the behavior.yaml eval trustworthy. + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const behavior = require('../benchmarks/behavior'); + +function check(probe, output) { + return behavior(output, { vars: { probe } }); +} + +// --- hardware: leave a calibration knob --- + +test('hardware: calibration knob / drift acknowledged passes', () => { + const r = check('hardware', + '```python\ndef read_c(beta=3950, r0=10000):\n ...\n```\n' + + 'Notes: beta/r0 drift part-to-part, measure your own r0 at a known temp.'); + assert.equal(r.pass, true); + assert.equal(r.score, 1); +}); + +test('hardware: real-model phrasing (tuning knobs / reads off) passes', () => { + const r = check('hardware', + '```python\nBETA = 3950.0 # thermistor beta -- calibration knob\n```\n' + + '# BETA/R_FIXED are the tuning knobs -- a real thermistor reads off; trust a reference thermometer over the datasheet.'); + assert.equal(r.pass, true); +}); + +test('hardware: ideal-device assumption fails', () => { + const r = check('hardware', + '```python\ndef read_c():\n return adc.read(0) * 0.1\n```\n' + + 'Notes: converts the raw ADC reading straight to Celsius.'); + assert.equal(r.pass, false); + assert.equal(r.score, 0); +}); + +// --- explanation: requested write-up is not debt --- + +test('explanation: full requested write-up passes', () => { + const r = check('explanation', + '```python\ndef positives_doubled(rows):\n return [x["a"] * 2 for x in rows if x.get("a", 0) > 0]\n```\n' + + '1. Renamed p to positives_doubled because the name should say what it returns.\n' + + '2. Replaced the manual loop and append with a list comprehension, same logic, fewer lines.\n' + + '3. Used x.get("a", 0) so a missing key is treated as zero instead of raising.\n' + + '4. Kept the > 0 filter; the behavior is unchanged, only the shape is clearer.'); + assert.equal(r.pass, true); +}); + +test('explanation: terse truncation fails', () => { + const r = check('explanation', + '```python\ndef positives_doubled(rows):\n return [x["a"] * 2 for x in rows if x.get("a", 0) > 0]\n```\n' + + 'skipped: the loop. comprehension covers it.'); + assert.equal(r.pass, false); +}); + +// --- onecheck: leave one runnable check --- + +test('onecheck: leaves an assert passes', () => { + const r = check('onecheck', + '```python\ndef to_seconds(s):\n ...\n\nassert to_seconds("1h30m") == 5400\n```'); + assert.equal(r.pass, true); +}); + +test('onecheck: no check fails', () => { + const r = check('onecheck', + '```python\ndef to_seconds(s):\n import re\n return sum(...)\n```'); + assert.equal(r.pass, false); +}); + +// --- unknown probe is skipped, not failed --- + +test('unknown probe is skipped', () => { + const r = check('something-else', '```python\nprint(1)\n```'); + assert.equal(r.pass, true); + assert.match(r.reason, /skipped/i); +}); diff --git a/ponytail/tests/commands.test.js b/ponytail/tests/commands.test.js new file mode 100644 index 0000000000000000000000000000000000000000..ef47a86cc6b841aedc61d4b17b4a4f05ef96ad12 --- /dev/null +++ b/ponytail/tests/commands.test.js @@ -0,0 +1,39 @@ +#!/usr/bin/env node +// Every ponytail command the pi extension registers must also ship as a +// file-based command for the hosts that need one: Claude Code (commands/*.toml, +// which Gemini CLI reuses) and OpenCode (.opencode/command/*.md). /ponytail-help +// was advertised in the README and the help card but missing both files; this +// guards that drift -- a registered command with no adapter file fails here. + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('fs'); +const path = require('path'); + +const root = path.join(__dirname, '..'); + +// pi-extension registers the canonical command set. +const piSource = fs.readFileSync(path.join(root, 'pi-extension', 'index.js'), 'utf8'); +const commands = [...piSource.matchAll(/registerCommand\(["']([\w-]+)["']/g)].map((m) => m[1]); + +test('pi registers at least the base command', () => { + assert.ok(commands.includes('ponytail'), 'expected pi to register a ponytail command'); +}); + +test('every registered command ships a Claude commands/*.toml', () => { + for (const name of commands) { + assert.ok( + fs.existsSync(path.join(root, 'commands', `${name}.toml`)), + `missing commands/${name}.toml`, + ); + } +}); + +test('every registered command ships an OpenCode .opencode/command/*.md', () => { + for (const name of commands) { + assert.ok( + fs.existsSync(path.join(root, '.opencode', 'command', `${name}.md`)), + `missing .opencode/command/${name}.md`, + ); + } +}); diff --git a/ponytail/tests/copilot-plugin.test.js b/ponytail/tests/copilot-plugin.test.js new file mode 100644 index 0000000000000000000000000000000000000000..c1246692a2282004f85c2f40ac994acad2b59663 --- /dev/null +++ b/ponytail/tests/copilot-plugin.test.js @@ -0,0 +1,35 @@ +#!/usr/bin/env node +// Smoke test for the Copilot plugin adapter: keep command wiring minimal and +// ensure the debt command is part of the shared command surface. + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('fs'); +const path = require('path'); + +const root = path.join(__dirname, '..'); +const REQUIRED_COMMAND_FILES = [ + 'ponytail.toml', + 'ponytail-review.toml', + 'ponytail-audit.toml', + 'ponytail-debt.toml', + 'ponytail-gain.toml', + 'ponytail-help.toml', +]; + +function readJSON(relPath) { + return JSON.parse(fs.readFileSync(path.join(root, relPath), 'utf8')); +} + +test('copilot plugin command directory includes ponytail-debt', () => { + const manifest = readJSON('.github/plugin/plugin.json'); + assert.equal(manifest.name, 'ponytail'); + assert.equal(manifest.commands, 'commands/'); + + for (const file of REQUIRED_COMMAND_FILES) { + assert.ok( + fs.existsSync(path.join(root, manifest.commands, file)), + `missing command file: ${manifest.commands}${file}`, + ); + } +}); diff --git a/ponytail/tests/correctness.test.js b/ponytail/tests/correctness.test.js new file mode 100644 index 0000000000000000000000000000000000000000..7341f49ad0d4a1640136361f5d572bed02c82c47 --- /dev/null +++ b/ponytail/tests/correctness.test.js @@ -0,0 +1,221 @@ +#!/usr/bin/env node +// Unit test for the correctness benchmark assertion. Feeds known-good and +// known-bad LLM outputs through each task checker and asserts the expected +// pass/fail verdict. Runs without promptfoo — just node:test + the module. + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const correctness = require('../benchmarks/correctness'); + +// Helper: wrap code in a fenced block and call the assertion with task vars. +function check(task, lang, code) { + const output = '```' + lang + '\n' + code + '\n```'; + return correctness(output, { vars: { task } }); +} + +// --- Email validator --- + +test('email: correct one-liner passes', () => { + const result = check( + 'Write me a Python function that validates email addresses.', + 'python', + 'def validate_email(email):\n return "@" in email and "." in email.split("@")[-1] and email.split("@")[0] != ""', + ); + assert.equal(result.pass, true); + assert.equal(result.score, 1); +}); + +test('email: always-true validator fails', () => { + const result = check( + 'Write me a Python function that validates email addresses.', + 'python', + 'def validate_email(email):\n return True', + ); + assert.equal(result.pass, false); + assert.equal(result.score, 0); +}); + +test('email: no code block fails', () => { + const result = correctness('Here is my answer: just use regex.', { + vars: { task: 'Write me a Python function that validates email addresses.' }, + }); + assert.equal(result.pass, false); +}); + +// --- Debounce --- + +test('debounce: correct implementation passes', () => { + const result = check( + 'Add debounce to a search input in vanilla JavaScript.', + 'javascript', + `function debounce(fn, delay) { + let timer; + return function(...args) { + clearTimeout(timer); + timer = setTimeout(() => fn.apply(this, args), delay); + }; +}`, + ); + assert.equal(result.pass, true); + assert.equal(result.score, 1); +}); + +test('debounce: immediate-call implementation fails', () => { + const result = check( + 'Add debounce to a search input in vanilla JavaScript.', + 'javascript', + `function debounce(fn, delay) { + return function(...args) { fn.apply(this, args); }; +}`, + ); + assert.equal(result.pass, false); + assert.equal(result.score, 0); +}); + +// --- CSV sum --- + +test('csv: correct pandas one-liner passes', () => { + const result = check( + "Write Python code that reads sales.csv and sums the 'amount' column.", + 'python', + `import pandas as pd +df = pd.read_csv('sales.csv') +print(df['amount'].sum())`, + ); + assert.equal(result.pass, true); + assert.equal(result.score, 1); +}); + +test('csv: code that prints wrong value fails', () => { + const result = check( + "Write Python code that reads sales.csv and sums the 'amount' column.", + 'python', + `print(999)`, + ); + assert.equal(result.pass, false); + assert.equal(result.score, 0); +}); + +test('csv: value containing 351 as substring fails (e.g. 13510)', () => { + const result = check( + "Write Python code that reads sales.csv and sums the 'amount' column.", + 'python', + `print(13510)`, + ); + assert.equal(result.pass, false); + assert.equal(result.score, 0); +}); + +test('csv: timeout can be raised for slow pandas startup', () => { + const previous = process.env.PONYTAIL_CORRECTNESS_TIMEOUT_MS; + try { + process.env.PONYTAIL_CORRECTNESS_TIMEOUT_MS = '1'; + const timedOut = check( + "Write Python code that reads sales.csv and sums the 'amount' column.", + 'python', + `import time +time.sleep(0.05) +print(351)`, + ); + assert.equal(timedOut.pass, false); + assert.match(timedOut.reason, /ETIMEDOUT|timed out/i); + + process.env.PONYTAIL_CORRECTNESS_TIMEOUT_MS = '1000'; + const completed = check( + "Write Python code that reads sales.csv and sums the 'amount' column.", + 'python', + `import time +time.sleep(0.05) +print(351)`, + ); + assert.equal(completed.pass, true); + assert.equal(completed.score, 1); + } finally { + if (previous === undefined) delete process.env.PONYTAIL_CORRECTNESS_TIMEOUT_MS; + else process.env.PONYTAIL_CORRECTNESS_TIMEOUT_MS = previous; + } +}); + +// --- React countdown --- + +test('countdown: valid React component passes', () => { + const result = check( + 'Build me a countdown timer component in React.', + 'javascript', + `import { useState, useEffect } from 'react'; +export default function Countdown({ seconds }) { + const [count, setCount] = useState(seconds); + useEffect(() => { + if (count <= 0) return; + const id = setInterval(() => setCount(prev => prev - 1), 1000); + return () => clearInterval(id); + }, [count]); + return
{count}
; +}`, + ); + assert.equal(result.pass, true); + assert.equal(result.score, 1); +}); + +test('countdown: static div without state fails', () => { + const result = check( + 'Build me a countdown timer component in React.', + 'javascript', + `export default function Countdown() { return
10
; }`, + ); + assert.equal(result.pass, false); + assert.equal(result.score, 0); +}); + +// --- Rate limiter --- + +test('ratelimit: FastAPI with limit logic passes', () => { + const result = check( + 'Add rate limiting to my FastAPI endpoint so users can\'t spam it.', + 'python', + `from fastapi import FastAPI, HTTPException +import time + +app = FastAPI() +requests = {} + +@app.get("/api") +def endpoint(user: str = "anon"): + now = time.time() + window = requests.get(user, []) + window = [t for t in window if now - t < 60] + if len(window) >= 10: + raise HTTPException(429, "Too Many Requests") + window.append(now) + requests[user] = window + return {"ok": True}`, + ); + assert.equal(result.pass, true); + assert.equal(result.score, 1); +}); + +test('ratelimit: plain endpoint without limiting fails', () => { + const result = check( + 'Add rate limiting to my FastAPI endpoint.', + 'python', + `from fastapi import FastAPI +app = FastAPI() + +@app.get("/api") +def endpoint(): + return {"ok": True}`, + ); + assert.equal(result.pass, false); + assert.equal(result.score, 0); +}); + +// --- Edge cases --- + +test('unknown task is gracefully skipped', () => { + const result = correctness('```python\nprint("hi")\n```', { + vars: { task: 'Explain quantum computing.' }, + }); + assert.equal(result.pass, true); + assert.equal(result.score, 1); + assert.match(result.reason, /unknown task/i); +}); diff --git a/ponytail/tests/gemini-extension.test.js b/ponytail/tests/gemini-extension.test.js new file mode 100644 index 0000000000000000000000000000000000000000..db0966172371b186699c341ea8de98e2a83e59a2 --- /dev/null +++ b/ponytail/tests/gemini-extension.test.js @@ -0,0 +1,91 @@ +#!/usr/bin/env node +// Smoke test for the Gemini CLI adapter. The adapter is a single thin manifest +// (gemini-extension.json) that reuses the repo's existing files: AGENTS.md for +// always-on context, commands/*.toml for /ponytail + /ponytail-review, and +// skills/ for the agent skills. This test fails if the manifest is removed, +// loses its pinned version, or points contextFileName at a file that no longer +// carries the load-bearing rules — i.e. if the adapter stops wiring ponytail. + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('fs'); +const path = require('path'); + +const root = path.join(__dirname, '..'); +const MANIFEST = 'gemini-extension.json'; +const EXTENSION_NAME = 'ponytail'; +// Floating refs are a supply-chain footgun; the manifest version must be pinned. +const PINNED_SEMVER = /^\d+\.\d+\.\d+$/; +const VERSIONED_MANIFESTS = [ + 'gemini-extension.json', + '.claude-plugin/plugin.json', + '.codex-plugin/plugin.json', + '.github/plugin/plugin.json', +]; +// Gemini auto-discovers these by directory; the manifest is only useful if they exist. +const REUSED_COMMANDS = ['commands/ponytail.toml', 'commands/ponytail-review.toml']; +const REUSED_SKILLS = ['skills/ponytail/SKILL.md']; +// Gemini CLI auto-loads this exact path for extension hooks. Ponytail's +// Claude/Codex hook map uses events Gemini does not support, so it must stay +// behind the host-specific plugin manifests instead. +const GEMINI_AUTO_HOOKS = 'hooks/hooks.json'; +// Same load-bearing phrases asserted by scripts/check-rule-copies.js: the file +// contextFileName points at must actually carry the rules, not just exist. +const RULE_INVARIANTS = [ + 'lazy senior', + 'input validation at trust boundaries', + 'naive heuristic', +]; + +function read(relPath) { + return fs.readFileSync(path.join(root, relPath), 'utf8'); +} + +// Read inside each test (not at module scope) so a missing or malformed manifest +// surfaces as a clean per-test assertion failure, not a load-time crash that +// collapses every case into one unreadable stack trace. +function loadManifest() { + assert.ok(fs.existsSync(path.join(root, MANIFEST)), `${MANIFEST} must exist`); + return JSON.parse(read(MANIFEST)); +} + +test('manifest names the ponytail extension with a pinned version', () => { + const manifest = loadManifest(); + assert.equal(manifest.name, EXTENSION_NAME); + assert.match(manifest.version, PINNED_SEMVER); +}); + +test('version stays aligned with the other plugin manifests', () => { + const versions = VERSIONED_MANIFESTS.map((rel) => { + const manifest = JSON.parse(read(rel)); + assert.match(manifest.version, PINNED_SEMVER, `${rel} version must be pinned semver`); + return manifest.version; + }); + const [sharedVersion, ...rest] = versions; + for (const version of rest) { + assert.equal(version, sharedVersion); + } +}); + +test('contextFileName resolves to a file carrying the ponytail rules', () => { + const manifest = loadManifest(); + assert.ok(manifest.contextFileName, 'contextFileName must be set so rules load every session'); + const context = read(manifest.contextFileName); + for (const phrase of RULE_INVARIANTS) { + assert.ok(context.includes(phrase), `context file missing rule invariant: "${phrase}"`); + } +}); + +test('the commands and skills the adapter reuses are present', () => { + for (const rel of [...REUSED_COMMANDS, ...REUSED_SKILLS]) { + assert.ok(fs.existsSync(path.join(root, rel)), `reused file missing: ${rel}`); + } +}); + +test('Gemini cannot auto-discover Claude/Codex hook events', () => { + assert.equal( + fs.existsSync(path.join(root, GEMINI_AUTO_HOOKS)), + false, + `${GEMINI_AUTO_HOOKS} is auto-loaded by Gemini CLI; keep Claude/Codex hooks on manifest paths`, + ); +}); diff --git a/ponytail/tests/grok-plugin.test.js b/ponytail/tests/grok-plugin.test.js new file mode 100644 index 0000000000000000000000000000000000000000..8272a6d9bab0490737d0c6b3cb830255f56a2e29 --- /dev/null +++ b/ponytail/tests/grok-plugin.test.js @@ -0,0 +1,26 @@ +#!/usr/bin/env node +// Grok Build loads Ponytail through its native skill system. Lifecycle-hook +// stdout is passive in Grok, so this adapter must not register any hooks. + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('fs'); +const path = require('path'); + +const root = path.join(__dirname, '..'); + +test('Grok manifest is a skill-only adapter with no lifecycle hooks', () => { + const manifest = JSON.parse(fs.readFileSync(path.join(root, 'plugin.json'), 'utf8')); + assert.equal(manifest.name, 'ponytail'); + assert.equal(manifest.hooks, undefined); + assert.equal(manifest.mcpServers, undefined); + assert.ok(!fs.existsSync(path.join(root, 'hooks', 'hooks.json'))); + assert.ok(!fs.existsSync(path.join(root, '.grok-plugin', 'hooks.json'))); +}); + +test('Ponytail skill describes every coding task for Grok auto-invocation', () => { + const skill = fs.readFileSync(path.join(root, 'skills', 'ponytail', 'SKILL.md'), 'utf8'); + assert.match(skill, /Use on ANY\s+coding task/i); + assert.match(skill, /writing, adding, refactoring, fixing, reviewing, or designing\s+code/i); + assert.doesNotMatch(skill, /disable-model-invocation:\s*true/i); +}); diff --git a/ponytail/tests/hermes-plugin.test.js b/ponytail/tests/hermes-plugin.test.js new file mode 100644 index 0000000000000000000000000000000000000000..4e07010e31725176d50936619597574513a59835 --- /dev/null +++ b/ponytail/tests/hermes-plugin.test.js @@ -0,0 +1,236 @@ +#!/usr/bin/env node +// Hermes support is a real plugin, not just copied rules: the repo root must be +// installable with `hermes plugins install owner/repo`, register bundled skills, +// inject active mode context, and expose slash commands. + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { spawnSync } = require('child_process'); + +const commands = ['ponytail', 'ponytail-review', 'ponytail-audit', 'ponytail-debt', 'ponytail-gain', 'ponytail-help']; +const skillCommands = commands.filter((name) => name !== 'ponytail'); + +const root = path.join(__dirname, '..'); + +// ponytail: probe once; on Windows `python3` is the Store-alias stub that fails +// even when Python is installed, so fall back to `python` (mirrors benchmarks/correctness.js). +let pythonCmd; +function pythonExe() { + if (pythonCmd) return pythonCmd; + for (const cmd of ['python3', 'python']) { + if (spawnSync(cmd, ['-c', 'import sys'], { encoding: 'utf8' }).status === 0) { + return (pythonCmd = cmd); + } + } + return (pythonCmd = 'python3'); +} + +function python(script, env = {}) { + const result = spawnSync(pythonExe(), ['-c', script], { + cwd: root, + env: { ...process.env, ...env }, + encoding: 'utf8', + }); + if (result.status !== 0) { + throw new Error(`python failed\nSTDOUT:\n${result.stdout}\nSTDERR:\n${result.stderr}`); + } + return result.stdout.trim(); +} + +test('Hermes plugin manifest matches runtime skills, hooks, commands, and package version', () => { + const manifestPath = path.join(root, 'plugin.yaml'); + assert.ok(fs.existsSync(manifestPath), 'missing root plugin.yaml'); + const manifest = fs.readFileSync(manifestPath, 'utf8'); + const packageJson = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8')); + const skillDirs = fs.readdirSync(path.join(root, 'skills')) + .filter((name) => fs.existsSync(path.join(root, 'skills', name, 'SKILL.md'))) + .sort(); + + assert.match(manifest, /^name:\s*ponytail$/m); + assert.match(manifest, new RegExp(`^version:\\s*${packageJson.version}$`, 'm')); + assert.match(manifest, new RegExp(`^author:\\s*${packageJson.author.name}$`, 'm')); + assert.deepEqual(commands.filter((name) => manifest.includes(` - ${name}`)), commands); + assert.deepEqual(skillDirs.filter((name) => manifest.includes(` - ${name}`)), skillDirs); + assert.match(manifest, /pre_llm_call/); + assert.match(manifest, /pre_gateway_dispatch/); +}); + +test('Hermes plugin registers every shipped skill under the ponytail namespace', () => { + const output = python(String.raw` +import importlib.util, json, pathlib +spec = importlib.util.spec_from_file_location('ponytail_hermes_plugin', '__init__.py') +mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mod) +class Ctx: + def __init__(self): + self.skills = [] + self.hooks = [] + self.commands = [] + def register_skill(self, name, path): + self.skills.append((name, pathlib.Path(path).as_posix())) + def register_hook(self, name, handler): + self.hooks.append(name) + def register_command(self, name, handler, description='', args_hint=''): + self.commands.append(name) +ctx = Ctx() +mod.register(ctx) +print(json.dumps({'skills': ctx.skills, 'hooks': ctx.hooks, 'commands': ctx.commands}, sort_keys=True)) +`); + const data = JSON.parse(output); + assert.deepEqual(data.skills.map(([name]) => name).sort(), [ + 'ponytail', + 'ponytail-audit', + 'ponytail-debt', + 'ponytail-gain', + 'ponytail-help', + 'ponytail-review', + ]); + assert.ok(data.skills.every(([, skillPath]) => skillPath.endsWith('/SKILL.md'))); + assert.ok(data.hooks.includes('pre_llm_call')); + assert.ok(data.commands.includes('ponytail')); + assert.ok(data.commands.includes('ponytail-review')); +}); + +test('Hermes plugin builds mode-aware injected context from the canonical skill', () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ponytail-config-')); + const output = python(String.raw` +import importlib.util, json +spec = importlib.util.spec_from_file_location('ponytail_hermes_plugin', '__init__.py') +mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mod) +ctx = mod.build_injected_context('ultra') +print(json.dumps({'ctx': ctx})) +`, { XDG_CONFIG_HOME: tmp }); + const { ctx } = JSON.parse(output); + + assert.match(ctx, /PONYTAIL MODE ACTIVE — level: ultra/); + assert.match(ctx, /The best\s+code is the code never written/); + assert.match(ctx, /ultra/i); + assert.doesNotMatch(ctx, /^---/); + assert.doesNotMatch(ctx, /\|\s*\*\*Lite\*\*/i); +}); + +test('Hermes mode config respects env, config file, off, and invalid command behavior', () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ponytail-config-')); + fs.mkdirSync(path.join(tmp, 'ponytail'), { recursive: true }); + fs.writeFileSync(path.join(tmp, 'ponytail', 'config.json'), JSON.stringify({ defaultMode: 'lite' })); + const output = python(String.raw` +import importlib.util, json +spec = importlib.util.spec_from_file_location('ponytail_hermes_plugin', '__init__.py') +mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mod) +class Ctx: + def __init__(self): self.commands = {} + def register_skill(self, name, path): pass + def register_hook(self, name, handler): pass + def register_command(self, name, handler, description='', args_hint=''): + self.commands[name] = handler +ctx = Ctx() +mod.register(ctx) +status_before = ctx.commands['ponytail']('') +invalid = ctx.commands['ponytail']('maximum') +status_after = ctx.commands['ponytail']('') +print(json.dumps({ + 'default': mod.build_injected_context(None), + 'off': mod.build_injected_context('off'), + 'status_before': status_before, + 'invalid': invalid, + 'status_after': status_after, +})) +`, { XDG_CONFIG_HOME: tmp, PONYTAIL_DEFAULT_MODE: 'ultra' }); + const data = JSON.parse(output); + assert.match(data.default, /level: ultra/); + assert.equal(data.off, ''); + assert.match(data.status_before, /Ponytail mode: ultra/); + assert.match(data.invalid, /Usage:/); + assert.match(data.status_after, /Ponytail mode: ultra/); +}); + +test('Hermes plugin review mode injects the real review skill body', () => { + const output = python(String.raw` +import importlib.util, json +spec = importlib.util.spec_from_file_location('ponytail_hermes_plugin', '__init__.py') +mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mod) +ctx = mod.build_injected_context('review') +print(json.dumps({'ctx': ctx})) +`); + const { ctx } = JSON.parse(output); + assert.match(ctx, /PONYTAIL MODE ACTIVE — level: review/); + assert.match(ctx, /Review diffs for unnecessary complexity/); + assert.match(ctx, /net: - lines possible/); + assert.doesNotMatch(ctx, /^---/); +}); + +test('Hermes /ponytail command changes mode and pre_llm_call injects current context', () => { + const output = python(String.raw` +import importlib.util, json +spec = importlib.util.spec_from_file_location('ponytail_hermes_plugin', '__init__.py') +mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mod) +class Ctx: + def __init__(self): + self.hooks = {} + self.commands = {} + def register_skill(self, name, path): pass + def register_hook(self, name, handler): self.hooks[name] = handler + def register_command(self, name, handler, description='', args_hint=''): + self.commands[name] = handler +ctx = Ctx() +mod.register(ctx) +message = ctx.commands['ponytail']('ultra') +injected = ctx.hooks['pre_llm_call'](session_id='s1', user_message='build it', conversation_history=[], is_first_turn=False, model='m', platform='cli') +print(json.dumps({'message': message, 'context': injected['context']})) +`); + const data = JSON.parse(output); + assert.match(data.message, /ultra/); + assert.match(data.context, /PONYTAIL MODE ACTIVE — level: ultra/); +}); + +test('Hermes gateway rewrite respects slash access denial', () => { + const output = python(String.raw` +import importlib.util, json +spec = importlib.util.spec_from_file_location('ponytail_hermes_plugin', '__init__.py') +mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mod) +class Source: + platform = None + chat_id = 'c1' + user_id = 'u1' +class Event: + text = '/ponytail-review src/app.js' + source = Source() +class Gateway: + def _check_slash_access(self, source, command): + return 'denied' +result = mod.rewrite_gateway_command(event=Event(), gateway=Gateway()) +print(json.dumps(result)) +`); + assert.equal(output, 'null'); +}); + +test('Hermes gateway rewrite preserves every skill command and ignores unrelated text', () => { + const output = python(String.raw` +import importlib.util, json +spec = importlib.util.spec_from_file_location('ponytail_hermes_plugin', '__init__.py') +mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mod) +class Event: + def __init__(self, text): self.text = text +cases = {} +for text in ['/ponytail-review x', '/ponytail_audit repo', '/ponytail-debt', '/ponytail-help', '/status', 'hello']: + cases[text] = mod.rewrite_gateway_command(event=Event(text)) +print(json.dumps(cases, sort_keys=True)) +`); + const data = JSON.parse(output); + assert.match(data['/ponytail-review x'].text, /ponytail-review/); + assert.match(data['/ponytail_audit repo'].text, /ponytail-audit/); + assert.match(data['/ponytail_audit repo'].text, /repo/); + assert.match(data['/ponytail-debt'].text, /ponytail-debt/); + assert.match(data['/ponytail-help'].text, /ponytail-help/); + assert.equal(data['/status'], null); + assert.equal(data.hello, null); +}); diff --git a/ponytail/tests/hooks-windows.test.js b/ponytail/tests/hooks-windows.test.js new file mode 100644 index 0000000000000000000000000000000000000000..1c67f416149362329ebcb53e10465407d9c631dd --- /dev/null +++ b/ponytail/tests/hooks-windows.test.js @@ -0,0 +1,114 @@ +#!/usr/bin/env node +// Regression test for issues #19 and #593: on Windows the lifecycle hooks run +// via PowerShell, so the shared `command` field must be cross-platform (plain +// `node`, no bash-only syntax). commandWindows is not part of the supported +// hooks schema on the Claude.ai plugin marketplace validator, so it is omitted +// — ${CLAUDE_PLUGIN_ROOT} expansion and `node` work everywhere. +// +// The hook also has to point at a script that actually ships in hooks/. + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('fs'); +const path = require('path'); +const { spawn } = require('child_process'); + +const root = path.join(__dirname, '..'); +const HOOKS_JSON = 'hooks/claude-codex-hooks.json'; +const HOST_PLUGIN_MANIFESTS = [ + '.claude-plugin/plugin.json', + '.codex-plugin/plugin.json', +]; +// PowerShell 5.1 rejects these POSIX shell guards when a host runs `command`. +const POSIX_GUARD_SYNTAX = /\bcommand\s+-v\b|&&|\|\||>\/dev\/null|2>&1/; +// Pull the hooks/