File size: 6,042 Bytes
1d0f191
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
"""Run the benchmark protocol and generate the report.

Refuses by default when the environment would make a run incomparable. Pass
`--allow-unfavourable` to take a labelled measurement anyway: the refusal
reasons are recorded in the report, which is more useful than nothing when a
machine cannot be put on AC power.
"""

from __future__ import annotations

import argparse
import sys
from datetime import UTC, datetime
from pathlib import Path

from forgesight.bench import protocol as proto
from forgesight.bench import report as report_mod
from forgesight.bench.runner import Refused, run
from forgesight.db.migrate import migrate
from forgesight.db.pool import get_pool
from forgesight.settings import get_settings


def main(argv: list[str] | None = None) -> int:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--tier", default="1", choices=["1", "2"])
    ap.add_argument("--models", nargs="*", default=["heron", "egret-medium"])
    ap.add_argument("--runtimes", nargs="*", default=["torch", "onnxruntime"])
    ap.add_argument("--pages", type=int, default=proto.MEASURED_PAGES)
    ap.add_argument("--trials", type=int, default=proto.TRIALS)
    ap.add_argument("--cooldown", type=float, default=proto.COOLDOWN_S)
    ap.add_argument("--allow-unfavourable", action="store_true",
                    help="measure anyway and record why the protocol would refuse")
    ap.add_argument("--quality", action="store_true", help="also run the quality pass")
    ap.add_argument("--out", default=None)
    args = ap.parse_args(argv)

    s = get_settings()
    s.ensure_dirs()
    pool = get_pool()
    migrate(pool, verbose=False)
    from forgesight.storage.object_store import build_store

    store = build_store(s)

    if args.tier == "1":
        configs = proto.tier1_matrix(args.models, args.runtimes)
    else:
        configs = proto.tier1_matrix(args.models, args.runtimes) + proto.tier2_matrix(
            args.models, args.runtimes, (2, 8), (8,), ("cv2_linear", "cv2_area")
        )
    print(f"matrix: {len(configs)} configurations x {args.trials} trials "
          f"x {args.pages} pages")

    try:
        outcome = run(
            configs, s, store, pool, trials=args.trials, pages=args.pages,
            allow_unfavourable=args.allow_unfavourable, cooldown_s=args.cooldown,
        )
    except Refused as exc:
        print(f"\nrefused to measure: {exc}", file=sys.stderr)
        print("Pass --allow-unfavourable to take a labelled measurement anyway.",
              file=sys.stderr)
        return 2
    except ValueError as exc:
        print(f"cannot run: {exc}", file=sys.stderr)
        return 1

    data = report_mod.ReportInput(
        env=outcome.env.to_dict(),
        trials=outcome.trials,
        raw_files=outcome.raw_files,
        refusals=outcome.refusals,
        memory_model=outcome.memory,
    )
    if args.quality:
        data.quality = _quality(args.models, s, store, pool)
        data.gating = _gating(args.models, s, store, pool)
    with pool.connection() as conn:
        data.dataset_hashes = [
            {"name": r["name"], "semver": r["semver"],
             "manifest_hash": r["manifest_hash"], "synthetic": bool(r["synthetic"])}
            for r in conn.fetchall(
                "SELECT name, semver, manifest_hash, synthetic FROM dataset_version"
            )
        ]

    stamp = datetime.now(UTC).strftime("%Y-%m-%d")
    out = Path(args.out) if args.out else s.reports_dir / f"report-{stamp}.md"
    # Plot first: the markdown references the file by name, so it has to exist
    # and be recorded on the input before `write` renders.
    fig = report_mod.write_plot(data, out.with_suffix(".svg"))
    report_mod.write(data, out)
    report_mod.write_json(data, out.with_suffix(".json"))
    print(f"\nwrote {out}")
    print(f"wrote {out.with_suffix('.json')}")
    if fig:
        print(f"wrote {fig}")
    for r in outcome.refusals:
        print(f"NOTE: protocol would have refused -- {r}")
    return 0


def _quality(models: list[str], s, store, pool) -> list[dict]:
    """Score each model on synth-clean/test and synth-scan via the reference."""
    from forgesight.eval import dataset as ds
    from forgesight.eval.coco_eval import evaluate
    from forgesight.eval.runner import predict_pages
    from forgesight.vision.registry import registry

    out: list[dict] = []
    ref = next(bc for bc in registry(s) if bc.role == "reference").candidate
    with pool.connection() as conn:
        ws_row = conn.fetchone("SELECT id FROM workspace ORDER BY created_at LIMIT 1")
    if ws_row is None:
        return out
    ws = ws_row["id"]
    for name, split in (("synth-clean", ds.TEST), ("synth-scan", ds.TEST)):
        truths = ds.load_truth(pool, ws, name, split)
        if not truths:
            continue
        from forgesight.eval.runner import _load_model

        cand = _load_model(ref, s)
        m = evaluate(truths, predict_pages(cand, ref, store, truths, pool),
                     ds.category_ids())
        out.append({"candidate": ref.name, "dataset": name, **m})
    return out


def _gating(models: list[str], s, store, pool) -> list[dict]:
    from forgesight.api.repo import WorkspaceRepo
    from forgesight.eval.runner import run_evaluation

    repo = WorkspaceRepo(pool)
    with pool.connection() as conn:
        row = conn.fetchone("SELECT id FROM workspace ORDER BY created_at LIMIT 1")
    if row is None:
        return []
    out = []
    for c in repo.list_candidates(row["id"]):
        try:
            result = run_evaluation(c, s, pool=pool, store=store)
        except Exception as exc:
            out.append({"candidate": c["name"], "gate": "evaluation", "passed": False,
                        "observed": str(exc)[:200], "threshold": None})
            continue
        for v in result.verdicts:
            out.append({"candidate": c["name"], **v})
    return out


if __name__ == "__main__":
    sys.path.insert(0, str(Path(__file__).resolve().parent))
    raise SystemExit(main())