scripts/test_library_members.py

"""Test every library member through the real stack.

For each member: a fresh conversation, a request carrying the deterministic
``[[demo-call:...]]`` directive, a real queue, a real worker executing the
real handler, and a real artifact back. Every member of the library is
exercised end to end, and one that cannot produce its artifact fails the run.

THIS USED TO FILM EACH MEMBER AND IT NO LONGER DOES.

The frames became a GIF on each library card. The cards now draw what their
member makes, which answers "what is this for" at a glance and on a phone,
where a hover-to-play recording answered nothing. The GIFs are gone and so is
the capture; the run they were evidence of is the part worth keeping, so this
is now a plain test.
"""

from __future__ import annotations

import json
import re
import subprocess
import sys
import threading
import time
from pathlib import Path

REPO_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO_ROOT))
sys.path.insert(0, str(REPO_ROOT / "scripts"))

from live_e2e import (  # noqa: E402
    CLIENT_ID,
    CLIENT_SECRET,
    enter_the_app,
    free_port,
    generate_pairing_code,
    redeem,
    sign_in,
    start_provider,
    wait_for,
)

#: ref -> (spoken request, directive arguments). The spoken half is what a
#: person would type; the directive half is what the deterministic runner
#: turns into the real tool call. Together they are the test.
CASES: dict[str, tuple[str, dict]] = {
    "create_pdf@1": (
        "Turn these notes into a PDF please",
        {"title": "Field notes", "body": "Open weights on lent machines.\n\nEnergy read off a meter, never guessed."},
    ),
    "make_plan@1": (
        "Write up a plan for the community launch",
        {"goal": "Launch the community worker network",
         "steps": ["Pair two volunteer workers", "Run the smoke suite", "Open sign-ups"]},
    ),
    "create_docx@1": (
        "I need that as a Word document",
        {"title": "Quarterly note", "body": "# Summary\nRevenue rose.\nCosts fell."},
    ),
    "create_xlsx@1": (
        "Make me a spreadsheet of the inventory",
        {"text": "item | qty | price\nWidget | 4 | 9.50\nBolt, large | 12 | 0.35"},
    ),
    "create_deck@1": (
        "Turn this outline into slides",
        {"slides": "distinct :: measured energy; open weights\nHow it works :: pair a worker; queue a run; read the meter"},
    ),
    "csv_table@1": (
        "Give me that table as a CSV",
        {"text": "name | role\nAda | analysis\nGrace | systems"},
    ),
    "status_update@1": (
        "Draft this week's status update",
        {"team": "Worker network", "period": "This week",
         "progress": "Two new volunteer machines paired.",
         "plans": "Add the GPU meter to the ladder.",
         "problems": "One worker reports no usable counter."},
    ),
    "review_checklist@1": (
        "Make a review checklist for the energy patch",
        {"subject": "Energy meter patch",
         "claims": ["The GPU figure is measured, not modelled",
                    "Scopes are never summed",
                    "A missing reading never counts as zero"]},
    ),
    "skill_scaffold@1": (
        "Scaffold a new skill for meeting notes",
        {"name": "meeting-notes",
         "description": "Turn raw meeting notes into a clean summary document.",
         "steps": ["Collect the raw notes", "Group them by decision", "Write the summary"]},
    ),
    "frontend_design@1": (
        "Build me a landing page for the project",
        {"title": "distinct", "sections": "What it is :: Small models on lent machines.\nWhy :: The meter is the product."},
    ),
    "web_artifacts_builder@1": (
        "Plan the interactive dashboard artifact",
        {"subject": "Energy dashboard artifact", "context": "Live joules per worker, one HTML file."},
    ),
    "theme_factory@1": (
        "Make a CSS theme in sage tones",
        {"name": "Sage", "colors": "primary: #2E5E43\nsurface: #FAF9F5\nink: #203127"},
    ),
    "brand_guidelines@1": (
        "Apply our brand to the new deck",
        {"subject": "Launch deck", "context": "Sage green on cream, Georgia headings."},
    ),
    "mcp_builder@1": (
        "Plan an MCP server for our metrics API",
        {"subject": "Metrics API MCP server", "context": "Read-only REST API, token auth."},
    ),
    "webapp_testing@1": (
        "Write the browser test plan for sign-in",
        {"subject": "Sign-in flow", "context": "OAuth redirect, then the app shell loads."},
    ),
    "claude_api@1": (
        "Plan the Claude API integration",
        {"subject": "Support inbox triage bot", "context": "Python, low volume, cost matters."},
    ),
    "algorithmic_art@1": (
        "Plan a generative art piece about rivers",
        {"subject": "River braiding", "context": "Blues and sand, seeded, printable."},
    ),
    "canvas_design@1": (
        "Design a poster for the meetup",
        {"subject": "Community compute meetup", "context": "A2 poster, evening light mood."},
    ),
    "academy_guide@1": (
        "Build me a learning path for local LLMs",
        {"subject": "Running local LLMs", "context": "Knows Python, has a gaming PC, 4 weeks."},
    ),
    "discernment_nudge@1": (
        "Give me a discernment nudge on this rollout",
        {"subject": "Shipping the worker auto-update", "context": "Team is eager; rollback story unclear."},
    ),
    "slack_gif_creator@1": (
        "Plan a celebration GIF for the release",
        {"subject": "v1 shipped", "context": "Confetti over the wordmark, two seconds."},
    ),
}


def record(page, ref: str, spoken: str, arguments: dict) -> None:
    """One member: request in, run through, artifact out.

    The assertion at the end is the point of the whole script. Every member of
    this library exists to produce something, so a run that completes without
    an artifact card in the transcript has not demonstrated the member — it has
    demonstrated the plumbing around it.
    """

    page.get_by_role("button", name=re.compile("^New conversation$", re.I)).click(timeout=15000)
    page.wait_for_timeout(900)

    prompt = f"{spoken}\n[[demo-call:{ref} {json.dumps(arguments)}]]"
    box = page.get_by_label(re.compile("Your request", re.I))
    box.fill(prompt, timeout=15000)

    # Exactly one member selected: the one under test. A job carries at most
    # sixteen selections, and a run that selects the whole shelf has not tested
    # anything in particular.
    wanted = ref.split("@", 1)[0].replace("_", " ")
    picker = page.locator(".c-composer-tools input[type=checkbox]")
    for index in range(picker.count()):
        member = picker.nth(index)
        if member.is_checked():
            member.uncheck(timeout=5000, force=True)
    page.wait_for_timeout(250)
    target = page.locator(".c-composer-tools label", has_text=f"{wanted} ·")
    if target.count() == 0:
        raise RuntimeError(f"{ref}: not offered in the composer")
    target.first.locator("input[type=checkbox]").check(timeout=5000, force=True)
    page.wait_for_timeout(400)
    page.get_by_role(
        "checkbox", name=re.compile("plaintext to a community-operated worker", re.I)
    ).check(timeout=15000)
    page.wait_for_timeout(200)
    page.get_by_role("button", name=re.compile("^Queue run$", re.I)).click(timeout=15000)

    deadline = time.monotonic() + 45
    done = False
    while time.monotonic() < deadline:
        if "requested library call" in page.inner_text("body"):
            done = True
            break
        page.wait_for_timeout(1000)
    if not done:
        raise RuntimeError(f"{ref}: the run did not complete in time")
    page.wait_for_timeout(1500)

    if "requested library call" not in page.inner_text("body"):
        raise RuntimeError(f"{ref}: the answer does not show the tool ran")
    made = page.locator("figure.c-madecard")
    if made.count() == 0:
        raise RuntimeError(f"{ref}: the run produced no artifact")
    summary = " | ".join(made.first.inner_text().split("\n"))[:70]
    print(f"  {ref:26s} PASS  {summary}")


def main() -> int:
    import os

    from playwright.sync_api import sync_playwright

    provider_url, provider = start_provider()
    port = free_port()
    server_url = f"http://127.0.0.1:{port}"
    environment = {
        **os.environ,
        "DISTINCT_OAUTH_CLIENT_ID": CLIENT_ID,
        "DISTINCT_OAUTH_CLIENT_SECRET": CLIENT_SECRET,
        "DISTINCT_OAUTH_REDIRECT_URI": f"{server_url}/auth/callback",
        "DISTINCT_OAUTH_PROVIDER_URL": provider_url,
        "DISTINCT_SESSION_SECRET": "a-fixed-secret-so-this-run-is-repeatable",
        "DISTINCT_BIND_HOST": "127.0.0.1",
        "PORT": str(port),
        "PYTHONPATH": str(REPO_ROOT),
    }
    environment.pop("DISTINCT_DEV_AUTH", None)
    server_log = open("/tmp/record-server.log", "w")
    server = subprocess.Popen(
        [sys.executable, str(REPO_ROOT / "app.py")],
        cwd=str(REPO_ROOT), env=environment,
        stdout=server_log, stderr=subprocess.STDOUT, text=True,
    )
    workers: list[subprocess.Popen] = []
    failures: list[str] = []
    try:
        wait_for(server_url)
        with sync_playwright() as playwright:
            browser = playwright.chromium.launch()
            context = browser.new_context(viewport={"width": 1280, "height": 860})
            page = context.new_page()
            page.goto(server_url, wait_until="load")
            sign_in(page, server_url, "alice")
            enter_the_app(page)
            pairing = generate_pairing_code(page)

            tool_refs = ",".join(sorted(CASES))
            worker_environment = {
                **environment,
                # The worker advertises what its operator policy names, not
                # what --tools asks for: without this the registry is empty
                # and the composer shows "no tools installed" forever.
                "DISTINCT_APPROVED_TOOLS": tool_refs,
            }
            process = subprocess.Popen(
                [
                    sys.executable, "-m", "distinct_agent",
                    "--server", server_url, "--pair", pairing,
                    "--name", "demo-recorder", "--approve", "--demo-runner",
                    "--tools", tool_refs,
                ],
                cwd=str(REPO_ROOT), env=worker_environment,
                stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
            )
            workers.append(process)
            access = None
            deadline = time.monotonic() + 120
            while time.monotonic() < deadline and access is None:
                line = process.stdout.readline()
                if not line:
                    if process.poll() is not None:
                        raise RuntimeError("worker exited before printing an access code")
                    continue
                found = re.search(r"\b([A-Z0-9]{5}(?:-[A-Z0-9]{4,5}){5,8})\b", line)
                if found:
                    access = found.group(1)
            if access is None:
                raise RuntimeError("no access code appeared")
            threading.Thread(
                target=lambda: [None for _ in process.stdout], daemon=True
            ).start()

            page.reload(wait_until="load")
            enter_the_app(page)
            page.wait_for_timeout(2500)
            redeem(page, access)
            page.wait_for_timeout(4000)

            # Wait for the worker's advertised members to reach the
            # composer picker: they enter snapshots only after approval, so
            # the first load can race them.
            for _ in range(10):
                if page.locator(".c-composer-tools input[type=checkbox]").count() > 0:
                    break
                page.wait_for_timeout(1500)
                page.reload(wait_until="load")
                enter_the_app(page)
                page.wait_for_timeout(2000)
            offered = page.locator(".c-composer-tools input[type=checkbox]").count()
            print(f"  composer offers {offered} member(s)")
            if offered == 0:
                raise RuntimeError("the composer picker never populated")

            for ref in sorted(CASES):
                spoken, arguments = CASES[ref]
                try:
                    record(page, ref, spoken, arguments)
                except Exception as exc:  # noqa: BLE001 - report and continue
                    failures.append(f"{ref}: {exc}")
                    print(f"  {ref:26s} FAIL  {exc}")
            browser.close()
    finally:
        for process in workers:
            if process.poll() is None:
                process.terminate()
                try:
                    process.wait(timeout=10)
                except subprocess.TimeoutExpired:
                    process.kill()
        server.terminate()
        try:
            server.wait(timeout=15)
        except subprocess.TimeoutExpired:
            server.kill()
        provider.shutdown()
    if failures:
        print(f"\n{len(failures)} member(s) failed; no GIF was written for them.")
        return 1
    print(f"\nAll {len(CASES)} members produced their artifact.")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())