tools/annotate_dmhy_prefix_tree_agent.py

"""Agentic strict LLM annotator for DMHY prefix-tree terminals.

This runner is intentionally separate from ``annotate_dmhy_prefix_graph.py`` so
prefix-tree and DAG experiments can evolve independently. It reads the existing
prefix graph plus the full DMHY source list, emits terminal annotation patches,
and writes dmhy_weak-compatible dataset JSONL records.
"""

from __future__ import annotations

import argparse
import concurrent.futures
import json
import os
import re
import sys
import urllib.error
import urllib.request
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Iterable

from anifilebert.tokenizer import AnimeTokenizer
from tools.annotate_dmhy_prefix_graph import (
    DEFAULT_GRAPH,
    DEFAULT_MODEL,
    DEFAULT_OUTPUT,
    DEFAULT_PATCH_OUTPUT,
    DEFAULT_SOURCE_LIST,
    LLM_SOURCE,
    dataset_records,
    heuristic_patch,
    load_graph,
    selected_terminals,
    source_list_matches,
    string_list,
)


DEFAULT_BASE_URL = "http://10.137.32.209/v1"
DEFAULT_UNITS_OUTPUT = Path("datasets/AnimeName/dmhy_prefix_graph.annotation_units.prefix_sample.jsonl")
NON_TITLE_NUMERIC_FRAGMENT_RE = re.compile(
    r"(?ix)^\s*"
    r"(?:"
    r"[\[(【《]?\s*(?:EP?|\#)?\d{1,4}(?:\s*(?:v|ver|version|rev)\s*\d{1,3})?\s*[\])】》]?|"
    r"(?:v|ver|version|rev)\s*\d{1,3}|"
    r"vol(?:ume)?\.?\s*\d{1,3}|"
    r"S\d{1,2}E\d{1,4}|"
    r"\d{1,2}x\d{1,4}"
    r")"
    r"\s*$"
)
ANNOTATION_SCHEMA: dict[str, Any] = {
    "type": "object",
    "additionalProperties": False,
    "required": [
        "terminal_id",
        "episode_title_suffixes",
        "media_suffixes",
        "title_candidates",
        "llm_label",
        "notes",
    ],
    "properties": {
        "terminal_id": {"type": "string"},
        "episode_title_suffixes": {
            "type": "array",
            "items": {"type": "string"},
        },
        "media_suffixes": {
            "type": "array",
            "items": {"type": "string"},
        },
        "title_candidates": {
            "type": "array",
            "items": {"type": "string"},
        },
        "llm_label": {
            "type": ["string", "null"],
            "description": "Short classification such as accept, media_only, episode_title, mixed, or uncertain.",
        },
        "notes": {"type": "string"},
    },
}


@dataclass(frozen=True)
class Args:
    graph: Path
    source_list: Path
    output: Path
    patch_output: Path | None
    units_output: Path | None
    limit: int | None
    min_weight: int | None
    only_needs_review: bool
    llm: bool
    base_url: str
    api_key: str | None
    model: str
    max_requests: int | None
    http_timeout: int
    preserve_i_labels: bool
    examples_only: bool
    workers: int
    retries: int
    resume: bool
    failure_output: Path | None = None
    reasoning_effort: str = "medium"


def parse_args() -> Args:
    parser = argparse.ArgumentParser(
        description="Strict Responses API agent for annotating DMHY prefix-tree terminals"
    )
    parser.add_argument("--graph", type=Path, default=DEFAULT_GRAPH)
    parser.add_argument("--source-list", type=Path, default=DEFAULT_SOURCE_LIST)
    parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
    parser.add_argument("--patch-output", default=str(DEFAULT_PATCH_OUTPUT))
    parser.add_argument(
        "--units-output",
        default=str(DEFAULT_UNITS_OUTPUT),
        help="Optional selected annotation units JSONL; use empty string to disable",
    )
    parser.add_argument("--limit", type=int, default=None)
    parser.add_argument("--min-weight", type=int, default=None)
    parser.add_argument("--only-needs-review", action="store_true")
    parser.add_argument("--llm", action="store_true")
    parser.add_argument(
        "--base-url",
        default=os.environ.get("ANIFILEBERT_LLM_BASE_URL", DEFAULT_BASE_URL),
        help="OpenAI-compatible Responses API base URL",
    )
    parser.add_argument(
        "--api-key",
        default=os.environ.get("ANIFILEBERT_LLM_API_KEY"),
        help="API key; prefer ANIFILEBERT_LLM_API_KEY",
    )
    parser.add_argument("--model", default=DEFAULT_MODEL)
    parser.add_argument("--max-requests", type=int, default=None)
    parser.add_argument("--http-timeout", type=int, default=120)
    parser.add_argument("--preserve-i-labels", action="store_true")
    parser.add_argument("--examples-only", action="store_true")
    parser.add_argument("--workers", type=int, default=8, help="Concurrent LLM terminal patch workers")
    parser.add_argument("--retries", type=int, default=3, help="LLM validation retry attempts per terminal")
    parser.add_argument("--resume", action="store_true", help="Skip ok LLM terminal patches already in patch output")
    parser.add_argument(
        "--reasoning-effort",
        choices=("minimal", "low", "medium", "high", "xhigh"),
        default="medium",
        help="Responses API reasoning effort when the target model supports reasoning",
    )
    parser.add_argument(
        "--failure-output",
        default="",
        help="Optional JSONL path for attempted terminals that failed LLM validation/request retries",
    )
    ns = parser.parse_args()
    patch_arg = str(ns.patch_output).strip()
    units_arg = str(ns.units_output).strip()
    failure_arg = str(ns.failure_output).strip()
    return Args(
        graph=ns.graph,
        source_list=ns.source_list,
        output=ns.output,
        patch_output=Path(patch_arg) if patch_arg else None,
        units_output=Path(units_arg) if units_arg else None,
        limit=ns.limit,
        min_weight=ns.min_weight,
        only_needs_review=ns.only_needs_review,
        llm=ns.llm,
        base_url=ns.base_url,
        api_key=ns.api_key,
        model=ns.model,
        max_requests=max(0, ns.max_requests) if ns.max_requests is not None else None,
        http_timeout=ns.http_timeout,
        preserve_i_labels=ns.preserve_i_labels,
        examples_only=ns.examples_only,
        workers=max(1, ns.workers),
        retries=max(1, ns.retries),
        resume=ns.resume,
        failure_output=Path(failure_arg) if failure_arg else None,
        reasoning_effort=ns.reasoning_effort,
    )


def responses_url(base_url: str) -> str:
    return base_url.rstrip("/") + "/responses"


def redact_secrets(text: str) -> str:
    return re.sub(r"sk-[A-Za-z0-9._-]+", "sk-REDACTED", text)


def unique_strings(value: Any, *, field: str) -> list[str]:
    if not isinstance(value, list):
        raise ValueError(f"{field} must be a list")
    seen: set[str] = set()
    result: list[str] = []
    for item in value:
        if not isinstance(item, str):
            raise ValueError(f"{field} items must be strings")
        cleaned = " ".join(item.split()).strip()
        if cleaned and cleaned not in seen:
            seen.add(cleaned)
            result.append(cleaned)
    return result


def validate_strict_annotation(annotation: Any, *, expected_terminal_id: str) -> dict[str, Any]:
    if not isinstance(annotation, dict):
        raise ValueError("annotation must be a JSON object")
    terminal_id = annotation.get("terminal_id")
    if not isinstance(terminal_id, str):
        raise ValueError("terminal_id must be a string")
    if terminal_id != expected_terminal_id:
        raise ValueError(f"terminal_id mismatch: expected {expected_terminal_id}, got {terminal_id}")
    llm_label = annotation.get("llm_label")
    if llm_label is not None and not isinstance(llm_label, str):
        raise ValueError("llm_label must be a string or null")
    notes = annotation.get("notes")
    if not isinstance(notes, str):
        raise ValueError("notes must be a string")
    validated = {
        "terminal_id": terminal_id,
        "episode_title_suffixes": unique_strings(annotation.get("episode_title_suffixes"), field="episode_title_suffixes"),
        "media_suffixes": unique_strings(annotation.get("media_suffixes"), field="media_suffixes"),
        "title_candidates": unique_strings(annotation.get("title_candidates"), field="title_candidates"),
        "llm_label": llm_label,
        "notes": " ".join(notes.split()).strip(),
    }
    errors = annotation_quality_errors(validated)
    if errors:
        raise ValueError("; ".join(errors))
    return validated


def annotation_quality_errors(annotation: dict[str, Any]) -> list[str]:
    errors: list[str] = []
    for field in ("episode_title_suffixes", "title_candidates"):
        values = annotation.get(field)
        if not isinstance(values, list):
            continue
        bad_values = [
            str(value)
            for value in values
            if isinstance(value, str) and NON_TITLE_NUMERIC_FRAGMENT_RE.fullmatch(value)
        ]
        if bad_values:
            errors.append(
                f"{field} must not contain pure episode/version/volume fragments: {bad_values[:5]}"
            )
    return errors


def extract_response_annotation(data: dict[str, Any]) -> Any:
    output_text = data.get("output_text")
    if isinstance(output_text, str) and output_text.strip():
        return json.loads(output_text)

    for item in data.get("output") or []:
        if not isinstance(item, dict):
            continue
        if item.get("type") in {"function_call", "tool_call"}:
            arguments = item.get("arguments")
            if isinstance(arguments, str):
                return json.loads(arguments)
            if isinstance(arguments, dict):
                return arguments
        for content in item.get("content") or []:
            if not isinstance(content, dict):
                continue
            if content.get("type") in {"output_json", "json"} and "json" in content:
                return content["json"]
            text = content.get("text")
            if isinstance(text, str) and text.strip():
                return json.loads(text)
    raise ValueError("Responses API returned no annotation payload")


def format_error_hint(error: str | None) -> str:
    if not error:
        return ""
    return (
        "\nPrevious attempt failed validation. Correct the issue exactly and submit a new schema-valid "
        f"annotation. Error: {error}"
    )


def request_body(
    *,
    args: Args,
    terminal: dict[str, Any],
    patch: dict[str, Any],
    previous_error: str | None,
    mode: str,
) -> dict[str, Any]:
    task = {
        "terminal_id": patch["terminal_id"],
        "prefix": terminal.get("prefix"),
        "digit_skeleton": terminal.get("digit_skeleton"),
        "suffix_examples": terminal.get("suffix_examples") or [],
        "value_examples": terminal.get("value_examples") or [],
        "heuristic_patch": {
            "episode_title_suffixes": patch.get("episode_title_suffixes") or [],
            "media_suffixes": patch.get("media_suffixes") or [],
            "title_candidates": patch.get("title_candidates") or [],
            "notes": patch.get("notes") or "",
        },
    }
    instructions = (
        "Annotate one anime filename prefix-tree terminal. Distinguish human episode-title suffix text "
        "from media metadata such as resolution, source, codec, audio, subtitles, language, hashes, and "
        "release tags. Do not put pure episode numbers, episode versions like 01v2, version tokens like "
        "ver1, or volume markers like Vol.1 in episode_title_suffixes or title_candidates; those are "
        "episode/version structure, not human title text. Keep the same terminal_id. Return only the "
        "required strict schema data."
        + format_error_hint(previous_error)
    )
    body: dict[str, Any] = {
        "model": args.model,
        "input": [
            {"role": "developer", "content": instructions},
            {"role": "user", "content": json.dumps(task, ensure_ascii=False)},
        ],
    }
    if "reasoning" in mode:
        body["reasoning"] = {"effort": args.reasoning_effort}
    if "json_schema" in mode:
        body["text"] = {
            "format": {
                "type": "json_schema",
                "name": "dmhy_prefix_terminal_annotation",
                "strict": True,
                "schema": ANNOTATION_SCHEMA,
            }
        }
    else:
        body["tools"] = [
            {
                "type": "function",
                "name": "submit_annotation",
                "description": "Submit the validated DMHY terminal annotation.",
                "strict": True,
                "parameters": ANNOTATION_SCHEMA,
            }
        ]
        body["tool_choice"] = {"type": "function", "name": "submit_annotation"}
    return body


def post_responses(body: dict[str, Any], args: Args) -> dict[str, Any]:
    if not args.api_key:
        raise RuntimeError("--llm requires --api-key or ANIFILEBERT_LLM_API_KEY")
    request = urllib.request.Request(
        responses_url(args.base_url),
        data=json.dumps(body, ensure_ascii=False).encode("utf-8"),
        headers={
            "Authorization": f"Bearer {args.api_key}",
            "Content-Type": "application/json",
        },
        method="POST",
    )
    try:
        with urllib.request.urlopen(request, timeout=args.http_timeout) as response:
            return json.loads(response.read().decode("utf-8"))
    except urllib.error.HTTPError as exc:
        detail = exc.read().decode("utf-8", errors="replace")
        raise RuntimeError(f"Responses API HTTP {exc.code}: {redact_secrets(detail[:500])}") from exc
    except urllib.error.URLError as exc:
        raise RuntimeError(f"Responses API request failed: {redact_secrets(str(exc.reason))}") from exc
    except json.JSONDecodeError as exc:
        raise RuntimeError("Responses API returned invalid response JSON") from exc


def next_mode(mode: str, error: str) -> str:
    lowered = error.lower()
    if mode == "reasoning_json_schema" and "reasoning" in lowered:
        return "json_schema"
    if "json_schema" in mode and any(token in lowered for token in ("text.format", "json_schema", "schema")):
        return "function_tool"
    if mode == "reasoning_json_schema":
        return "json_schema"
    return mode


def merge_llm_annotation(patch: dict[str, Any], annotation: dict[str, Any]) -> dict[str, Any]:
    merged = dict(patch)
    merged.update(
        {
            "episode_title_suffixes": annotation["episode_title_suffixes"],
            "media_suffixes": annotation["media_suffixes"],
            "title_candidates": annotation["title_candidates"],
            "llm_label": annotation["llm_label"],
            "notes": annotation["notes"],
            "source": LLM_SOURCE,
            "status": "ok",
            "fallback": False,
            "errors": [],
        }
    )
    return merged


def annotate_terminal_with_llm(
    terminal: dict[str, Any],
    patch: dict[str, Any],
    args: Args,
) -> tuple[dict[str, Any], bool]:
    mode = "reasoning_json_schema"
    previous_error: str | None = None
    errors: list[str] = []
    for _attempt in range(args.retries):
        try:
            body = request_body(args=args, terminal=terminal, patch=patch, previous_error=previous_error, mode=mode)
            data = post_responses(body, args)
            raw_annotation = extract_response_annotation(data)
            annotation = validate_strict_annotation(raw_annotation, expected_terminal_id=str(patch["terminal_id"]))
            return merge_llm_annotation(patch, annotation), False
        except (RuntimeError, ValueError, json.JSONDecodeError) as exc:
            error = redact_secrets(str(exc))
            errors.append(error)
            mode = next_mode(mode, error)
            previous_error = error
    fallback = dict(patch)
    fallback["notes"] = f"{fallback.get('notes') or ''}; llm_error={' | '.join(errors)}"
    fallback["status"] = "fallback"
    fallback["fallback"] = True
    fallback["errors"] = errors[-3:]
    return fallback, True


def write_jsonl(path: Path, rows: Iterable[dict[str, Any]]) -> int:
    path.parent.mkdir(parents=True, exist_ok=True)
    count = 0
    with path.open("w", encoding="utf-8", newline="\n") as handle:
        for row in rows:
            handle.write(json.dumps(row, ensure_ascii=False, separators=(",", ":")) + "\n")
            count += 1
    return count


def is_completed_resume_row(row: dict[str, Any]) -> bool:
    status = row.get("status")
    source = str(row.get("source") or "")
    if status == "ok" and source.startswith("responses"):
        return not annotation_quality_errors(row)
    return status is None and source == LLM_SOURCE


def load_resume_patches(path: Path | None) -> dict[str, dict[str, Any]]:
    if path is None or not path.exists():
        return {}
    completed: dict[str, dict[str, Any]] = {}
    for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
        if not line.strip():
            continue
        try:
            row = json.loads(line)
        except json.JSONDecodeError as exc:
            raise SystemExit(f"invalid resume patch JSON in {path}:{line_number}: {exc}") from exc
        if not isinstance(row, dict) or not is_completed_resume_row(row):
            continue
        keys: list[str] = []
        for key in ("terminal_id", "unit_id"):
            value = row.get(key)
            if isinstance(value, (str, int)) and not isinstance(value, bool):
                keys.append(str(value))
        terminal_ids = row.get("terminal_ids")
        if isinstance(terminal_ids, list):
            keys.extend(str(item) for item in terminal_ids if isinstance(item, (str, int)) and not isinstance(item, bool))
        for key in keys:
            completed[key] = row
    return completed


def patch_from_resume_row(patch: dict[str, Any], row: dict[str, Any]) -> dict[str, Any]:
    merged = dict(patch)
    for key in ("episode_title_suffixes", "media_suffixes", "title_candidates"):
        merged[key] = unique_strings(row.get(key) or [], field=key)
    if "llm_label" in row:
        merged["llm_label"] = row.get("llm_label")
    if "notes" in row:
        merged["notes"] = str(row.get("notes") or "")
    merged["source"] = str(row.get("source") or patch.get("source") or LLM_SOURCE)
    merged["fallback"] = False
    merged["status"] = "ok"
    errors = row.get("errors")
    merged["errors"] = errors if isinstance(errors, list) else []
    return enrich_patch(merged)


def resume_row_for_patch(
    resume_rows: dict[str, dict[str, Any]],
    patch: dict[str, Any],
) -> dict[str, Any] | None:
    for key in (patch.get("terminal_id"), patch.get("unit_id")):
        if key is None:
            continue
        row = resume_rows.get(str(key))
        if row is not None:
            return row
    return None


def annotate_selected(
    selected: list[tuple[int, dict[str, Any], dict[str, Any]]],
    args: Args,
) -> tuple[list[tuple[int, int, dict[str, Any], dict[str, Any]]], int, int, int, int]:
    resume_rows = load_resume_patches(args.patch_output) if args.resume else {}
    results: list[dict[str, Any] | None] = [None] * len(selected)
    pending_llm: list[tuple[int, dict[str, Any], dict[str, Any]]] = []
    resume_skipped = 0

    for ordinal, (_index, _terminal, patch) in enumerate(selected):
        row = resume_row_for_patch(resume_rows, patch)
        if row is not None:
            results[ordinal] = patch_from_resume_row(patch, row)
            resume_skipped += 1
            continue
        if not args.llm:
            results[ordinal] = patch
            continue
        pending_llm.append((ordinal, selected[ordinal][1], patch))

    candidates = pending_llm
    if args.max_requests is not None:
        candidates = candidates[: args.max_requests]

    fallback_count = 0
    failure_rows: list[dict[str, Any]] = []
    if candidates:
        with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as executor:
            future_to_ordinal = {
                executor.submit(annotate_terminal_with_llm, terminal, patch, args): ordinal
                for ordinal, terminal, patch in candidates
            }
            for future in concurrent.futures.as_completed(future_to_ordinal):
                ordinal = future_to_ordinal[future]
                try:
                    patch, used_fallback = future.result()
                except Exception as exc:  # defensive; worker should already convert errors to fallback
                    patch = dict(selected[ordinal][2])
                    patch["errors"] = [redact_secrets(str(exc))]
                    used_fallback = True
                if used_fallback:
                    fallback_count += 1
                    terminal_id = selected[ordinal][2]["terminal_id"]
                    failure_rows.append(llm_failure_row(selected[ordinal][1], patch, args))
                    print(f"warning: terminal {terminal_id}: LLM failed; leaving pending", file=sys.stderr)
                    continue
                results[ordinal] = patch

    if args.failure_output is not None:
        write_jsonl(args.failure_output, failure_rows)

    completed = [
        (ordinal, selected[ordinal][0], selected[ordinal][1], patch)
        for ordinal, patch in enumerate(results)
        if patch is not None
    ]
    pending_unprocessed = len(selected) - len(completed)
    return completed, resume_skipped, len(candidates), fallback_count, pending_unprocessed


def llm_failure_row(terminal: dict[str, Any], patch: dict[str, Any], args: Args) -> dict[str, Any]:
    terminal_id = str(patch.get("terminal_id") or terminal.get("terminal_id") or "")
    errors = patch.get("errors")
    return {
        "terminal_id": terminal_id,
        "unit_id": str(patch.get("unit_id") or terminal_id),
        "model": args.model,
        "status": "pending",
        "source": LLM_SOURCE,
        "errors": errors if isinstance(errors, list) else [],
        "prefix": terminal.get("prefix") if isinstance(terminal.get("prefix"), str) else None,
        "digit_skeleton": terminal.get("digit_skeleton") if isinstance(terminal.get("digit_skeleton"), str) else None,
        "suffix_examples": string_list(terminal.get("suffix_examples")),
        "value_examples": string_list(terminal.get("value_examples")),
    }


def source_id(terminal: dict[str, Any], index: int, terminal_id: str) -> str | int:
    for key in ("node_id", "id"):
        value = terminal.get(key)
        if isinstance(value, (str, int)) and not isinstance(value, bool):
            return value
    return terminal_id or index


def int_weight(terminal: dict[str, Any]) -> int:
    try:
        return int(terminal.get("weight") or terminal.get("count") or 0)
    except (TypeError, ValueError):
        return 0


def enrich_patch(patch: dict[str, Any]) -> dict[str, Any]:
    terminal_id = str(patch["terminal_id"])
    enriched = dict(patch)
    enriched.setdefault("unit_id", terminal_id)
    enriched.setdefault("terminal_ids", [terminal_id])
    enriched.setdefault("status", "ok")
    enriched.setdefault("fallback", False)
    enriched.setdefault("errors", [])
    return enriched


def unit_public_row(index: int, terminal: dict[str, Any], patch: dict[str, Any]) -> dict[str, Any]:
    terminal_id = str(patch["terminal_id"])
    return {
        "unit_id": str(patch.get("unit_id") or terminal_id),
        "source_kind": "prefix_tree",
        "source_id": source_id(terminal, index, terminal_id),
        "terminal_ids": [terminal_id],
        "weight": int_weight(terminal),
        "context": {
            "prefixes": string_list([terminal.get("prefix")] if terminal.get("prefix") is not None else []),
            "digit_skeletons": string_list(
                [terminal.get("digit_skeleton")] if terminal.get("digit_skeleton") is not None else []
            ),
            "edge_labels": [],
            "notes": patch.get("notes") if isinstance(patch.get("notes"), str) else None,
        },
        "examples": {
            "values": string_list(terminal.get("value_examples")),
            "suffixes": string_list(terminal.get("suffix_examples")),
        },
        "expected_output": {"schema_version": "dmhy-annotation-v1"},
    }


def main() -> None:
    args = parse_args()
    graph = load_graph(args.graph)
    selected = selected_terminals(graph, args)
    if not selected:
        raise SystemExit("no terminals selected; adjust --limit/--min-weight/--only-needs-review")

    selected = [(index, terminal, enrich_patch(patch)) for index, terminal, patch in selected]
    completed, resume_skipped, llm_requests, llm_fallbacks, pending_unprocessed = annotate_selected(selected, args)
    tokenizer = AnimeTokenizer()
    source_matches = None if args.examples_only else source_list_matches(args.source_list, selected)

    records: list[dict[str, Any]] = []
    patches: list[dict[str, Any]] = []
    completed_by_ordinal = {ordinal: patch for ordinal, _index, _terminal, patch in completed}
    for ordinal, index, terminal, patch in completed:
        patches.append(patch)
        records.extend(
            dataset_records(
                terminal,
                index,
                patch,
                tokenizer,
                filenames=None if args.examples_only else source_matches.get(ordinal, []),
                preserve_i_labels=args.preserve_i_labels,
            )
        )

    record_count = write_jsonl(args.output, records)
    patch_count = 0
    if args.patch_output is not None:
        patch_count = write_jsonl(args.patch_output, patches)
    unit_count = 0
    if args.units_output is not None:
        unit_count = write_jsonl(
            args.units_output,
            (
                unit_public_row(index, terminal, completed_by_ordinal.get(ordinal, patch))
                for ordinal, (index, terminal, patch) in enumerate(selected)
            ),
        )

    summary = {
        "graph": str(args.graph),
        "source_list": None if args.examples_only else str(args.source_list),
        "output": str(args.output),
        "patch_output": str(args.patch_output) if args.patch_output is not None else None,
        "units_output": str(args.units_output) if args.units_output is not None else None,
        "selected_terminals": len(selected),
        "examples_only": args.examples_only,
        "dataset_records": record_count,
        "patches": patch_count,
        "units": unit_count,
        "llm_enabled": args.llm,
        "llm_requests": llm_requests,
        "llm_fallbacks": llm_fallbacks,
        "resume": args.resume,
        "resume_skipped": resume_skipped,
        "pending_unprocessed": pending_unprocessed,
        "workers": args.workers,
    }
    print(json.dumps(summary, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()