File size: 4,154 Bytes
32c0c6c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 | #!/usr/bin/env python
"""Reproduce the documented dev metric (first-token top-1 / top-5 / CE) from packaged files.
Protocol (identical to the project's scripts/04_diag_checkpoint.py):
rows = first 1024 rows of data/dev/dev_ids.npy with dev_ids[k,1] == <|FORWARD|>
logits = model(ids)[k, mask_start-1] (the position right before the first tactic token)
target = ids[k, mask_start]
This is the ONLY comparable dev protocol: the numbers printed by the trainer use a
length-bucketed reshuffle of the dev split and must not be compared across datasets.
python eval_dev.py # checkpoints/e6, 1024 rows
python eval_dev.py --ckpt checkpoints/stage3-e3 --rows 256
python eval_dev.py --all --compare # every variant + compare with metrics.json
"""
import argparse
import json
import os
import numpy as np
import torch
import torch.nn.functional as F
from common import ROOT, VARIANTS, load_config, load_model, pick_device, specials
CFG = load_config()
def dev_eval(net, device, rows=1024, batch=64, verbose=False):
sp = specials()
pad = sp['<|pad|>']
ids_all = np.load(os.path.join(ROOT, 'data/dev/dev_ids.npy'), mmap_mode='r')
ms_all = np.load(os.path.join(ROOT, 'data/dev/dev_mask_start.npy'))
ln_all = np.load(os.path.join(ROOT, 'data/dev/dev_len.npy'))
fwd = [k for k in range(len(ln_all)) if ids_all[k, 1] == sp['<|FORWARD|>']][:rows]
n1 = n5 = n = 0
ce_sum = 0.0
with torch.no_grad():
for s in range(0, len(fwd), batch):
chunk = fwd[s:s + batch]
seqs = [np.asarray(ids_all[k, :int(ln_all[k])], dtype=np.int64) for k in chunk]
m = max(len(x) for x in seqs)
arr = np.full((len(chunk), m), pad, dtype=np.int64)
for j, x in enumerate(seqs):
arr[j, :len(x)] = x
t = torch.from_numpy(arr).to(device)
ms = torch.tensor([min(int(ms_all[k]), m - 1) for k in chunk], device=device)
ar = torch.arange(len(chunk), device=device)
lg = net(t)['logits'][ar, (ms - 1).clamp(min=0)]
tgt = t[ar, ms.clamp(max=m - 1)]
p5 = lg.topk(5, -1).indices
n1 += int((p5[:, 0] == tgt).sum())
n5 += int((p5 == tgt[:, None]).any(1).sum())
ce_sum += float(F.cross_entropy(lg.float(), tgt, reduction='sum'))
n += len(chunk)
if verbose:
print(f' ... {n}/{len(fwd)} rows', flush=True)
return {'rows': n, 'top1': n1 / n, 'top5': n5 / n, 'ce': ce_sum / n}
def documented(variant):
m = json.load(open(os.path.join(ROOT, 'metrics.json')))['dev']
key = {'checkpoints/e6': 'stage4_e6_full_best',
'checkpoints/stage3-e3': 'stage3_e3_best'}.get(variant)
return m.get(key) if key else None
if __name__ == '__main__':
ap = argparse.ArgumentParser()
ap.add_argument('--ckpt', default=VARIANTS[0])
ap.add_argument('--all', action='store_true', help='evaluate every packaged variant')
ap.add_argument('--rows', type=int, default=1024)
ap.add_argument('--batch', type=int, default=64)
ap.add_argument('--device', default=None)
ap.add_argument('--compare', action='store_true', help='compare with metrics.json')
a = ap.parse_args()
device = pick_device(a.device)
todo = VARIANTS if a.all else (a.ckpt,)
worst = 0.0
for ck in todo:
net, _cfg, device = load_model(ck, device)
r = dev_eval(net, device, rows=a.rows, batch=a.batch)
line = (f'{ck:22s} rows={r["rows"]:4d} top1={r["top1"]:.4f} '
f'top5={r["top5"]:.4f} ce={r["ce"]:.4f}')
doc = documented(ck)
if a.compare and doc and r['rows'] == 1024:
d = max(abs(r['top1'] - doc['top1']), abs(r['top5'] - doc['top5']),
abs(r['ce'] - doc['ce']))
worst = max(worst, d)
line += f' | 文档值 {doc["top1"]:.4f}/{doc["top5"]:.4f}/{doc["ce"]:.4f} 偏差 {d:.5f}'
line += ' ✅' if d < 5e-4 else ' ❌'
print(line, flush=True)
if a.compare:
print(f'\n最大偏差 {worst:.5f} → ' + ('PASS' if worst < 5e-4 else 'FAIL'))
|