debug.py

"""Local debug workflow for the GAIA Unit 4 agent.

Run the agent locally before deploying and inspect exactly where exact-match
formatting may be going wrong. Every run is saved as a JSON record capturing:

    task_id, question, raw_output, submitted_answer

so you can compare the model's raw text against the normalized submission.

Usage
-----
    # one random question
    python debug.py one --random

    # one specific question
    python debug.py one --task-id <task_id>

    # all questions (batch)
    python debug.py all

    # also print a character-level comparison of raw vs submitted
    python debug.py one --random --compare

Records are appended to ``runs/<timestamp>.jsonl`` (one JSON object per line)
and the comparison is printed to the console. Nothing is submitted to the
scoring API — this is for inspection only.

Requires ANTHROPIC_API_KEY (or whichever provider you configured) in the
environment, the same as a real run.
"""

from __future__ import annotations

import argparse
import json
import sys
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional

from src import scoring_api
from src.agent import AgentResult, run_agent_detailed

RUNS_DIR = Path(__file__).resolve().parent / "runs"


# ---------------------------------------------------------------------------
# Record building + persistence
# ---------------------------------------------------------------------------

def _record(result: Optional[AgentResult], question: Dict[str, Any],
            error: Optional[str] = None) -> Dict[str, Any]:
    """Build the saved record. Raw output is stored verbatim, never normalized."""
    return {
        "timestamp": datetime.now(timezone.utc).isoformat(),
        "task_id": (result.task_id if result else str(question.get("task_id", ""))),
        "question": (result.question if result else question.get("question", "")),
        "file_name": question.get("file_name") or None,
        "file_kind": result.file_kind if result else None,
        "raw_output": result.raw_output if result else None,
        "submitted_answer": result.submitted_answer if result else None,
        "error": error,
    }


def _save(records: List[Dict[str, Any]]) -> Path:
    """Append records to a timestamped JSONL file under runs/."""
    RUNS_DIR.mkdir(exist_ok=True)
    stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
    path = RUNS_DIR / f"{stamp}.jsonl"
    with path.open("w", encoding="utf-8") as fh:
        for rec in records:
            fh.write(json.dumps(rec, ensure_ascii=False) + "\n")
    return path


# ---------------------------------------------------------------------------
# Safe comparison: reveal hidden formatting differences without guessing
# ---------------------------------------------------------------------------

def print_comparison(rec: Dict[str, Any]) -> None:
    """Print raw vs submitted using repr(), so invisible characters show up.

    repr() makes trailing spaces, newlines, tabs, and smart quotes visible —
    exactly the things that silently break an exact match. We never transform
    the strings here; we only display them.
    """
    raw = rec.get("raw_output")
    sub = rec.get("submitted_answer")
    print(f"\n=== task_id: {rec['task_id']} ===")
    print(f"question        : {rec['question'][:200]}")
    if rec.get("file_kind"):
        print(f"file            : {rec.get('file_name')} ({rec['file_kind']})")
    if rec.get("error"):
        print(f"ERROR           : {rec['error']}")
        return
    print(f"raw_output  repr: {raw!r}")
    print(f"submitted   repr: {sub!r}")
    if raw is not None and sub is not None:
        changed = raw.strip() != sub
        print(f"normalization   : {'changed the text' if changed else 'no change'}")
        if changed:
            # Show what was trimmed from each end, so you can see *why* it changed.
            print(f"  raw.strip() == submitted? {raw.strip() == sub}")
            print(f"  lengths: raw={len(raw)} stripped={len(raw.strip())} submitted={len(sub)}")


# ---------------------------------------------------------------------------
# Modes
# ---------------------------------------------------------------------------

def debug_one(task_id: Optional[str], use_random: bool, compare: bool) -> int:
    """Run a single question (random or by task_id)."""
    try:
        if use_random:
            question = scoring_api.get_random_question()
        else:
            question = _find_question(task_id)
    except scoring_api.ScoringAPIError as exc:
        print(f"Failed to fetch question: {exc}", file=sys.stderr)
        return 1
    if question is None:
        print(f"No question found with task_id={task_id!r}", file=sys.stderr)
        return 1

    rec = _run_and_record(question)
    path = _save([rec])
    print_comparison(rec)
    print(f"\nSaved 1 record to {path}")
    return 0


def debug_all(compare: bool) -> int:
    """Run every question in batch and save all records."""
    try:
        questions = scoring_api.get_questions()
    except scoring_api.ScoringAPIError as exc:
        print(f"Failed to fetch questions: {exc}", file=sys.stderr)
        return 1

    print(f"Running {len(questions)} questions...")
    records = []
    for i, question in enumerate(questions, 1):
        rec = _run_and_record(question)
        records.append(rec)
        flag = "ERR" if rec["error"] else "ok "
        print(f"[{i}/{len(questions)}] {flag} {rec['task_id']}: "
              f"{rec['submitted_answer']!r}")
        if compare:
            print_comparison(rec)

    path = _save(records)
    errors = sum(1 for r in records if r["error"])
    print(f"\nSaved {len(records)} records to {path} ({errors} error(s)).")
    return 0


def _run_and_record(question: Dict[str, Any]) -> Dict[str, Any]:
    """Run one question, capturing any error into the record (never raises)."""
    try:
        result = run_agent_detailed(question)
        return _record(result, question)
    except Exception as exc:  # noqa: BLE001 - debug tool must keep going
        return _record(None, question, error=f"{type(exc).__name__}: {exc}")


def _find_question(task_id: Optional[str]) -> Optional[Dict[str, Any]]:
    """Look up a specific question by task_id from the full list."""
    if not task_id:
        return None
    for question in scoring_api.get_questions():
        if str(question.get("task_id")) == str(task_id):
            return question
    return None


# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------

def main(argv: Optional[List[str]] = None) -> int:
    parser = argparse.ArgumentParser(description="Local debug runner for the GAIA agent.")
    sub = parser.add_subparsers(dest="mode", required=True)

    p_one = sub.add_parser("one", help="run a single question")
    group = p_one.add_mutually_exclusive_group(required=True)
    group.add_argument("--random", action="store_true", help="use a random question")
    group.add_argument("--task-id", help="run the question with this task_id")
    p_one.add_argument("--compare", action="store_true", help="print raw vs submitted")

    p_all = sub.add_parser("all", help="run every question (batch)")
    p_all.add_argument("--compare", action="store_true", help="print raw vs submitted")

    args = parser.parse_args(argv)

    if args.mode == "one":
        return debug_one(args.task_id, args.random, args.compare)
    if args.mode == "all":
        return debug_all(args.compare)
    parser.error("unknown mode")
    return 2


if __name__ == "__main__":
    raise SystemExit(main())