| """Preregistered confirmation scoring of Standard v2 blind and the homophone challenge. |
| |
| `plan` writes configs/ime-confirmation-v2.json from the constants below: hypotheses, |
| model snapshots (SHA-256, step), candidate pool file hashes and code hashes. It reads |
| no benchmark text. The plan must be committed before `run`. |
| |
| `run` refuses unless the plan and this script are committed and unchanged, every hash |
| still matches, and no completed consumption receipt exists. It first rescores the exposed |
| v1 development n20 pool with every model and requires the recorded rankings to be |
| reproduced, and only then opens the blind and homophone pools. Scores are written once; |
| the report follows configs/ime-metrics-v2.json and the plan's decision rules. |
| |
| `report` recomputes summary.json from a completed run without scoring again. |
| """ |
| import argparse |
| import copy |
| import importlib.metadata |
| import json |
| import subprocess |
| import sys |
| from collections import Counter |
| from datetime import datetime, timezone |
| from pathlib import Path |
|
|
| ROOT = Path(__file__).resolve().parents[2] |
| sys.path.insert(0, str(ROOT / 'src')) |
| from vimeml.training.data import file_sha, write_json |
| from vimeml.benchmarks.comparison import rows as read_rows |
|
|
| PLAN = ROOT / 'configs/ime-confirmation-v2.json' |
| IMPORTED = 'artifacts/benchmarks/ime-candidates-v2-imported' |
| RECEIPT = ROOT / IMPORTED / 'consumption.json' |
| POOLS = { |
| 'standard-v2-blind': {'n20': 'standard-v2-blind-n20', 'n50': 'standard-v2-blind-n50'}, |
| 'homophone-v1': {'n20': 'homophone-v1-n20', 'n50': 'homophone-v1-n50'}, |
| } |
| PREFLIGHT = 'standard-v1-development-n20' |
| MODELS = { |
| 'v21': {'checkpoint': 'artifacts/models/tiny-ja-v2.1-e16k-d320-l6-extend5/best.pt', |
| 'tokenizer': 'artifacts/tokenizers/ja-unigram-16k-v2', 'loader': 'v2', |
| 'role': 'Deployed model (FP32 weights of the INT8 deployment)', |
| 'recorded_development': 'outputs/evaluation-v4-20261015/v21/standard-development/scores.jsonl'}, |
| 'v42-e20': {'checkpoint': 'outputs/training-v42-20261020/remote/checkpoints/best-lm.pt', |
| 'tokenizer': 'artifacts/tokenizers/v4-a/ja-unigram-8192', 'loader': 'v4', |
| 'role': 'Optional replacement; device-measured; selected by LM BPC before IME evaluation', |
| 'recorded_development': 'outputs/evaluation-v42-20261020/standard-development/scores.jsonl'}, |
| 'v43-e30': {'checkpoint': 'outputs/training-v43-20261020/remote/checkpoints/epoch-30.pt', |
| 'tokenizer': 'artifacts/tokenizers/v4-a/ja-unigram-8192', 'loader': 'v4', |
| 'role': 'V4.2 recipe plus web data; best LM BPC checkpoint', |
| 'recorded_development': 'outputs/evaluation-v43-20261020/epoch30/standard-development/scores.jsonl'}, |
| 'v5-e30': {'checkpoint': 'outputs/training-v5-20261021/remote/checkpoints/epoch-30.pt', |
| 'tokenizer': 'artifacts/tokenizers/v4-a/ja-unigram-8192', 'loader': 'v5', |
| 'role': 'V4.3 data and budget at d512/L8; best LM BPC checkpoint', |
| 'recorded_development': 'outputs/evaluation-v5-20261021/epoch30/standard-development/scores.jsonl'}, |
| } |
| PRIMARY = {'id': 'H1', 'before': 'v21', 'after': 'v42-e20', 'track': 'standard-v2-blind', 'pool': 'n20', |
| 'question': 'Does V4.2 epoch 20 improve exact Top-1 over the deployed V2.1?'} |
| SECONDARY = [ |
| {'id': 'S1', 'before': 'v21', 'after': 'v42-e20', 'track': 'homophone-v1', 'pool': 'n20', |
| 'question': 'V2.1 -> V4.2 on written-text homophones'}, |
| {'id': 'S2', 'before': 'v43-e30', 'after': 'v5-e30', 'track': 'standard-v2-blind', 'pool': 'n20', |
| 'question': 'Model size only (same data and budget), conversational'}, |
| {'id': 'S3', 'before': 'v43-e30', 'after': 'v5-e30', 'track': 'homophone-v1', 'pool': 'n20', |
| 'question': 'Model size only (same data and budget), written-text homophones'}, |
| {'id': 'S4', 'before': 'v42-e20', 'after': 'v43-e30', 'track': 'standard-v2-blind', 'pool': 'n20', |
| 'question': 'Added web data, conversational'}, |
| ] |
| CODE = ['configs/ime-metrics-v2.json', 'scripts/benchmarks/confirm_ime_v2.py', |
| 'src/vimeml/benchmarks/notation.py', 'src/vimeml/benchmarks/statistics.py', |
| 'src/vimeml/benchmarks/evaluate_ajimee.py', 'src/vimeml/benchmarks/standard_ime.py', |
| 'src/vimeml/benchmarks/expanded_ime.py', 'src/vimeml/benchmarks/comparison.py', |
| 'src/vimeml/training/evaluate_ime.py', 'src/vimeml/training/infer.py', |
| 'src/vimeml/training/infer_v4.py', 'src/vimeml/training/infer_v5.py'] |
| PACKAGES = ('torch', 'fugashi', 'unidic-lite', 'sentencepiece') |
| POOL_FILES = ('manifest.json', 'case-map.json', 'ajimee-results/ajimee-input.json', |
| 'ajimee-results/azookey-candidates.json') |
| OUTCOMES = {'exact': ('correct',), 'notation_lenient': ('correct', 'notation')} |
| ALPHA = 0.05 |
|
|
|
|
| def git(*args): |
| return subprocess.run(['git', *args], cwd=ROOT, check=True, capture_output=True, text=True).stdout |
|
|
|
|
| def versions(): |
| return {name: importlib.metadata.version(name) for name in PACKAGES} |
|
|
|
|
| def pool_hashes(name): |
| folder = ROOT / IMPORTED / name |
| return {f: file_sha(folder / f) for f in POOL_FILES} |
|
|
|
|
| def make_plan(args): |
| if PLAN.exists(): |
| raise ValueError('The plan already exists; a changed plan needs a new version and a new blind release.') |
| if RECEIPT.exists(): |
| raise ValueError('These pools have already been opened.') |
| import torch |
| models = {} |
| for name, spec in MODELS.items(): |
| path = ROOT / spec['checkpoint'] |
| saved = torch.load(path, map_location='cpu', weights_only=True) |
| models[name] = {**spec, 'sha256': file_sha(path), 'bytes': path.stat().st_size, 'step': saved['step']} |
| del saved |
| release_ids = {track: json.loads((ROOT / IMPORTED / pools['n20'] / 'manifest.json').read_text(encoding='utf-8'))['release_id'] |
| for track, pools in POOLS.items()} |
| write_json(PLAN, { |
| 'version': 'ime-confirmation-v2', |
| 'created_utc': datetime.now(timezone.utc).isoformat(), |
| 'metric_policy': 'configs/ime-metrics-v2.json', |
| 'tracks': {track: {'release_id': release_ids[track], 'cases': {'standard-v2-blind': 729, 'homophone-v1': 619}[track], |
| 'pools': {k: {'directory': f'{IMPORTED}/{v}', 'sha256': pool_hashes(v)} for k, v in pools.items()}} |
| for track, pools in POOLS.items()}, |
| 'preflight': {'pool': f'{IMPORTED}/{PREFLIGHT}', 'sha256': pool_hashes(PREFLIGHT), |
| 'requirement': 'Every model reproduces its recorded development lm_context_sum order on all 728 cases before any blind or homophone file is read'}, |
| 'models': models, |
| 'scoring': {'semantics': 'FP32 full-vocabulary suffix logP sum; joint context+candidate tokenization; no final EOS, truncation or tuning; stable ties', |
| 'precision': 'FP32 on CUDA; TF32 disabled', 'packages': versions()}, |
| 'baseline': 'AzooKey first candidate (no language model), descriptive only', |
| 'primary': {**PRIMARY, 'metric': 'exact Top-1', |
| 'claim_rule': 'net > 0 and author-cluster sign-flip p < 0.05; and no notation-lenient loss with cluster p < 0.05 for the same pair on either n20 track', |
| 'if_confirmed': 'V4.2 is a confirmed exact Top-1 improvement over the deployed V2.1', |
| 'if_not_confirmed': 'No confirmed difference; V2.1 stays deployed and V4.2 stays an optional variant'}, |
| 'secondary': {'tests': SECONDARY, 'metric': 'exact Top-1', |
| 'multiplicity': 'Holm over the four cluster sign-flip p-values at 0.05', |
| 'claim_rule': 'Holm-adjusted p < 0.05 (two-sided; direction from net) and the same notation-lenient guard as H1'}, |
| 'descriptive': ['notation-lenient and pool-conditional Top-1 for every model and track', |
| 'pool coverage at n20 and n50, opened with scoring', |
| 'n50 pools: every metric and pair, descriptive only (metrics v2 keeps n20 primary)', |
| 'every model pair on both tracks, unadjusted, descriptive only', |
| 'template and union cluster sensitivity', |
| 'Standard by source (WRIME, JMultiWOZ)', |
| 'homophone: exact Top-1 when the reference is the more frequent vs the second lemma of its group; groups with both items correct (complete groups only)', |
| 'error classes'], |
| 'clusters': 'Author (WRIME user, JMultiWOZ user) on Standard; homophone group on the challenge track', |
| 'execution': 'One run. A run that stops for a technical failure before writing summary.json may be repeated into a new directory under the same plan, recorded in the receipt. Results are never rerun to change an outcome', |
| 'deviations': 'Any change after the first run (bug fix, metric correction) is reported as a deviation next to the original output, which is kept', |
| 'publication': 'Aggregates and case IDs only; no benchmark text, candidates or homophone group lists', |
| 'code_sha256': {path: file_sha(ROOT / path) for path in CODE}, |
| }) |
| print(f'Wrote {PLAN.relative_to(ROOT)}; commit it with this script before `run`.') |
|
|
|
|
| def check_committed(plan): |
| paths = ['configs/ime-confirmation-v2.json', *plan['code_sha256']] |
| git('ls-files', '--error-unmatch', *paths) |
| dirty = git('status', '--porcelain', '--', *paths) |
| if dirty: |
| raise ValueError(f'Commit the plan and code first:\n{dirty}') |
| for path, digest in plan['code_sha256'].items(): |
| if file_sha(ROOT / path) != digest: |
| raise ValueError(f'{path} differs from the plan') |
| if versions() != plan['scoring']['packages']: |
| raise ValueError(f'Package versions differ from the plan: {versions()}') |
|
|
|
|
| def load_model(spec): |
| checkpoint, tokenizer = ROOT / spec['checkpoint'], ROOT / spec['tokenizer'] |
| if spec['loader'] == 'v2': |
| from vimeml.training.infer import JapaneseLM as Model |
| elif spec['loader'] == 'v4': |
| from vimeml.training.infer_v4 import JapaneseLMV4 as Model |
| else: |
| from vimeml.training.infer_v5 import JapaneseLMV5 as Model |
| lm = Model(checkpoint, tokenizer, 'cuda') |
| if lm.metadata['checkpoint_step'] != spec['step'] or file_sha(checkpoint) != spec['sha256']: |
| raise ValueError(f'{checkpoint} differs from the planned snapshot') |
| return lm |
|
|
|
|
| def load_pool(entry): |
| from vimeml.benchmarks.standard_ime import load_standard_export |
| folder = ROOT / entry['directory'] |
| if {f: file_sha(folder / f) for f in POOL_FILES} != entry['sha256']: |
| raise ValueError(f'{folder} differs from the plan') |
| return load_standard_export(folder)[2] |
|
|
|
|
| def score(lm, rows): |
| from vimeml.benchmarks.evaluate_ajimee import rerank |
| import torch |
| rows = copy.deepcopy(rows) |
| with torch.inference_mode(): |
| rerank(lm, rows) |
| return rows |
|
|
|
|
| def write_rows(path, rows): |
| path.write_text(''.join(json.dumps(r, ensure_ascii=False, allow_nan=False) + '\n' for r in rows), |
| encoding='utf-8', newline='\n') |
|
|
|
|
| def run(args): |
| import torch |
| plan = json.loads(PLAN.read_text(encoding='utf-8')) |
| check_committed(plan) |
| receipt = json.loads(RECEIPT.read_text(encoding='utf-8')) if RECEIPT.exists() else {'attempts': []} |
| if receipt.get('plan_sha256', file_sha(PLAN)) != file_sha(PLAN): |
| raise ValueError('These pools were opened under a different plan.') |
| if any(a['status'] == 'complete' for a in receipt['attempts']): |
| raise ValueError('This plan has already been scored; use `report` on the completed output.') |
| out = args.output.resolve() |
| if out.exists() and any(out.iterdir()): |
| raise ValueError('Use an empty output directory') |
| if not torch.cuda.is_available(): |
| raise ValueError('FP32 CUDA scoring is required') |
| torch.backends.cuda.matmul.allow_tf32 = False |
| torch.backends.cudnn.allow_tf32 = False |
| torch.set_float32_matmul_precision('highest') |
| torch.set_num_threads(4) |
| commit = git('rev-parse', 'HEAD').strip() |
| plan_sha = file_sha(PLAN) |
|
|
| |
| development = load_pool({'directory': plan['preflight']['pool'], 'sha256': plan['preflight']['sha256']}) |
| models = {} |
| for name, spec in plan['models'].items(): |
| lm = load_model(spec) |
| scored = score(lm, development) |
| recorded = read_rows(ROOT / spec['recorded_development']) |
| same = sum(a['id'] == b['id'] and a['orders']['lm_context_sum'] == b['orders']['lm_context_sum'] |
| for a, b in zip(recorded, scored, strict=True)) |
| print(f'preflight {name}: {same}/{len(scored)} recorded development orders reproduced', flush=True) |
| if same != len(scored): |
| raise ValueError(f'{name} does not reproduce its recorded development scores; blind pools stay closed.') |
| models[name] = lm |
|
|
| out.mkdir(parents=True) |
| receipt['plan_sha256'] = plan_sha |
| receipt['attempts'].append({'output': str(out.relative_to(ROOT)), 'commit': commit, 'status': 'started', |
| 'started_utc': datetime.now(timezone.utc).isoformat()}) |
| write_json(RECEIPT, receipt) |
| for track, info in plan['tracks'].items(): |
| for pool, entry in info['pools'].items(): |
| rows = load_pool(entry) |
| if len(rows) != info['cases']: |
| raise ValueError(f'{track} {pool}: {len(rows)} cases') |
| for name, lm in models.items(): |
| dest = out / track / pool |
| dest.mkdir(parents=True, exist_ok=True) |
| write_rows(dest / f'{name}.jsonl', score(lm, rows)) |
| print(f'scored {track} {pool} {name}', flush=True) |
| write_json(out / 'run.json', {'plan_sha256': plan_sha, 'commit': commit, 'gpu': torch.cuda.get_device_name(), |
| 'cuda': torch.version.cuda, 'packages': versions(), |
| 'completed_utc': datetime.now(timezone.utc).isoformat()}) |
| receipt['attempts'][-1].update(status='complete', completed_utc=datetime.now(timezone.utc).isoformat()) |
| write_json(RECEIPT, receipt) |
| report(argparse.Namespace(output=out)) |
|
|
|
|
| def holm(pvalues): |
| order = sorted(pvalues, key=pvalues.get) |
| adjusted, running = {}, 0.0 |
| for i, key in enumerate(order): |
| running = max(running, min(1.0, (len(order) - i) * pvalues[key])) |
| adjusted[key] = running |
| return adjusted |
|
|
|
|
| def track_report(rows): |
| from vimeml.benchmarks.notation import error_class |
| from vimeml.benchmarks import statistics |
| base = next(iter(rows.values())) |
| for data in rows.values(): |
| assert [r['id'] for r in data] == [r['id'] for r in base] |
| assert all(r['answers'] == b['answers'] and r['orders']['azookey'] == b['orders']['azookey'] |
| for r, b in zip(data, base)) |
| reachable = [any(a in {c['text'] for c in r['candidates']} for a in r['answers']) for r in base] |
| labels = {'author': statistics.clusters(base, template=None), |
| 'template': statistics.clusters([{'answers': r['answers']} for r in base]), |
| 'union': statistics.clusters(base)} |
| classes = {'azookey': [error_class(r, r['orders']['azookey'][0])[0] for r in base]} |
| classes.update({name: [error_class(r, r['orders']['lm_context_sum'][0])[0] for r in data] for name, data in rows.items()}) |
| sources = sorted({r['source'] for r in base}) |
| homophone = all(str(r['provenance'].get('group_id', '')).startswith('homophone:') for r in base) |
| entry = {'cases': len(base), 'pool_reachable': sum(reachable), |
| 'clusters': {k: len(set(v)) for k, v in labels.items()}, 'models': {}, 'paired': {}} |
| for name, items in classes.items(): |
| exact = sum(c == 'correct' for c in items) |
| model = {'exact': exact, 'notation_lenient': exact + items.count('notation'), |
| 'pool_conditional': exact / sum(reachable), 'error_classes': dict(Counter(items))} |
| if len(sources) > 1: |
| model['by_source'] = {s: sum(c == 'correct' for c, r in zip(items, base) if r['source'] == s) for s in sources} |
| if homophone: |
| ranks = [r['provenance']['group_rank'] for r in base] |
| model['by_group_rank'] = {str(k): sum(c == 'correct' for c, g in zip(items, ranks) if g == k) for k in (0, 1)} |
| groups = Counter(r['provenance']['group_id'] for r in base) |
| correct = Counter(r['provenance']['group_id'] for c, r in zip(items, base) if c == 'correct') |
| model['complete_groups_both_correct'] = sum(correct[g] == 2 for g, n in groups.items() if n == 2) |
| entry['models'][name] = model |
| if homophone: |
| groups = Counter(r['provenance']['group_id'] for r in base) |
| entry['groups'] = {'all': len(groups), 'complete': sum(n == 2 for n in groups.values()), |
| 'cases_by_group_rank': dict(Counter(str(r['provenance']['group_rank']) for r in base))} |
| names = list(classes) |
| for i, a in enumerate(names): |
| for b in names[i + 1:]: |
| entry['paired'][f'{a}->{b}'] = { |
| metric: {scheme: statistics.paired([c in ok for c in classes[a]], [c in ok for c in classes[b]], lab) |
| for scheme, lab in labels.items()} |
| for metric, ok in OUTCOMES.items()} |
| return entry, reachable |
|
|
|
|
| def report(args): |
| out = args.output.resolve() |
| plan = json.loads(PLAN.read_text(encoding='utf-8')) |
| run_info = json.loads((out / 'run.json').read_text(encoding='utf-8')) |
| tracks, reach = {}, {} |
| for track, info in plan['tracks'].items(): |
| tracks[track] = {} |
| for pool in info['pools']: |
| rows = {name: read_rows(out / track / pool / f'{name}.jsonl') for name in plan['models']} |
| tracks[track][pool], reach[track, pool] = track_report(rows) |
| n20, n50 = reach[track, 'n20'], reach[track, 'n50'] |
| tracks[track]['coverage'] = {'n20': sum(n20), 'n50': sum(n50), |
| 'gained': sum(b and not a for a, b in zip(n20, n50)), |
| 'lost': sum(a and not b for a, b in zip(n20, n50))} |
|
|
| def test(spec): |
| pair = tracks[spec['track']][spec['pool']]['paired'][f"{spec['before']}->{spec['after']}"] |
| guard = {t: tracks[t]['n20']['paired'][f"{spec['before']}->{spec['after']}"]['notation_lenient']['author'] |
| for t in plan['tracks']} |
| lenient_loss = [t for t, g in guard.items() if g['net'] < 0 and g['cluster_sign_flip_p'] < ALPHA] |
| return pair['exact']['author'], lenient_loss |
|
|
| exact, lenient_loss = test(plan['primary']) |
| primary = {**plan['primary'], 'exact': exact, 'notation_lenient_losses': lenient_loss, |
| 'confirmed': exact['net'] > 0 and exact['cluster_sign_flip_p'] < ALPHA and not lenient_loss} |
| results = {s['id']: test(s) for s in plan['secondary']['tests']} |
| adjusted = holm({k: v[0]['cluster_sign_flip_p'] for k, v in results.items()}) |
| secondary = [] |
| for spec in plan['secondary']['tests']: |
| exact, losses = results[spec['id']] |
| significant = adjusted[spec['id']] < ALPHA and not losses |
| secondary.append({**spec, 'exact': exact, 'holm_p': adjusted[spec['id']], 'notation_lenient_losses': losses, |
| 'claim': ('improvement' if exact['net'] > 0 else 'regression') if significant else 'none'}) |
| summary = {'plan_sha256': file_sha(PLAN), 'run': run_info, 'primary': primary, 'secondary': secondary, |
| 'tracks': tracks, 'report_script_sha256': file_sha(Path(__file__))} |
| write_json(out / 'summary.json', summary) |
| print(json.dumps({'primary': {k: primary[k] for k in ('id', 'exact', 'notation_lenient_losses', 'confirmed')}, |
| 'secondary': [{k: s[k] for k in ('id', 'holm_p', 'claim')} for s in secondary], |
| 'models': {t: {p: {m: {k: v[k] for k in ('exact', 'notation_lenient')} |
| for m, v in tracks[t][p]['models'].items()} for p in ('n20', 'n50')} |
| for t in tracks}, |
| 'coverage': {t: tracks[t]['coverage'] for t in tracks}}, ensure_ascii=False, indent=1)) |
|
|
|
|
| def main(): |
| parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) |
| subs = parser.add_subparsers(dest='command', required=True) |
| subs.add_parser('plan') |
| for name in ('run', 'report'): |
| subs.add_parser(name).add_argument('--output', type=Path, required=True) |
| args = parser.parse_args() |
| {'plan': make_plan, 'run': run, 'report': report}[args.command](args) |
|
|
|
|
| if __name__ == '__main__': |
| main() |
|
|
