| """Preregistered second use of Standard v2 blind and the homophone set: V4.2-recipe seed ensemble. |
| |
| The four V4.2-recipe seeds (V4.2 epoch 20 and V4.4 s1-s3) were combined only after the V4.4 |
| analysis on development (docs/zh-CN/training-v44.md). Both pools were consumed once by |
| confirmation v2, and no model or ensemble rule was tuned on them since, so this is a |
| replication: its outcome is descriptive evidence, not a deployment confirmation. |
| |
| `plan` writes configs/ime-ensemble-replication-v2.json (hypotheses, snapshots, the reused |
| V4.2 confirmation score files, pool and code hashes); it reads no benchmark text and must be |
| committed and pushed before `run`. `run` refuses unless the plan and code are committed and |
| unchanged, s1-s3 reproduce their recorded development orders, and no completed attempt |
| exists; it then scores s1-s3 once on the n20 and n50 pools, reuses the recorded V4.2 scores, |
| and builds the ensembles. `report` recomputes summary.json without scoring. Loading, |
| scoring and per-track statistics are those of `confirm_ime_v2.py`. |
| """ |
| import argparse |
| import copy |
| import json |
| import sys |
| from datetime import datetime, timezone |
| from itertools import combinations |
| from pathlib import Path |
|
|
| ROOT = Path(__file__).resolve().parents[2] |
| sys.path.insert(0, str(ROOT / 'src')) |
| sys.path.insert(0, str(ROOT / 'scripts/benchmarks')) |
| import confirm_ime_v2 as v2 |
| from vimeml.training.data import file_sha, write_json |
| from vimeml.benchmarks.comparison import rows as read_rows |
|
|
| PLAN = ROOT / 'configs/ime-ensemble-replication-v2.json' |
| RECEIPT = ROOT / v2.IMPORTED / 'ensemble-replication.json' |
| CONFIRMATION = 'outputs/confirmation-v2-20261021' |
| SEEDS = {arm: {'checkpoint': f'outputs/training-v44-20261105/remote/{arm}/checkpoints/epoch-20.pt', |
| 'tokenizer': 'artifacts/tokenizers/v4-a/ja-unigram-8192', 'loader': 'v4', |
| 'recorded_development': f'outputs/evaluation-v44-20261105/{arm}/standard-development/scores.jsonl'} |
| for arm in ('s1', 's2', 's3')} |
| MEMBERS = ['v42-e20', 's1', 's2', 's3'] |
| ENSEMBLE = 'ens4' |
| PRIMARY = {'id': 'R1', 'before': 'v42-e20', 'after': ENSEMBLE, 'track': 'standard-v2-blind', 'pool': 'n20', |
| 'question': 'Does the four-seed ensemble improve exact Top-1 over the deployed V4.2 epoch 20?'} |
| SECONDARY = [{'id': 'R2', 'before': 'v42-e20', 'after': ENSEMBLE, 'track': 'homophone-v1', 'pool': 'n20', |
| 'question': 'The same on written-text homophones'}] |
| CODE = ['scripts/benchmarks/replicate_ensemble_v2.py', *v2.CODE] |
|
|
|
|
| def make_plan(args): |
| if PLAN.exists(): |
| raise ValueError('The plan already exists.') |
| if RECEIPT.exists(): |
| raise ValueError('These pools have already been opened for this replication.') |
| import torch |
| confirmation = json.loads(v2.PLAN.read_text(encoding='utf-8')) |
| models = {} |
| for name, spec in SEEDS.items(): |
| path = ROOT / spec['checkpoint'] |
| saved = torch.load(path, map_location='cpu', weights_only=True) |
| models[name] = {**spec, 'sha256': file_sha(path), 'bytes': path.stat().st_size, 'step': saved['step']} |
| del saved |
| reused = {f'{track}/{pool}': {'file': f'{CONFIRMATION}/{track}/{pool}/v42-e20.jsonl', |
| 'sha256': file_sha(ROOT / CONFIRMATION / track / pool / 'v42-e20.jsonl')} |
| for track, info in confirmation['tracks'].items() for pool in info['pools']} |
| write_json(PLAN, { |
| 'version': 'ime-ensemble-replication-v2', |
| 'created_utc': datetime.now(timezone.utc).isoformat(), |
| 'metric_policy': 'configs/ime-metrics-v2.json', |
| 'status_of_sets': ('Second use. Both pools were consumed once by configs/ime-confirmation-v2.json; ' |
| 'the ensemble idea came from development after that. Outcomes are descriptive replication ' |
| 'evidence; any deployment claim needs a fresh blind release'), |
| 'tracks': confirmation['tracks'], |
| 'preflight': {**confirmation['preflight'], |
| 'requirement': 's1-s3 reproduce their recorded development lm_context_sum order on all 728 cases before any blind or homophone file is read'}, |
| 'models': models, |
| 'reused_scores': {'model': 'v42-e20', 'source': 'configs/ime-confirmation-v2.json run', 'files': reused}, |
| 'ensemble': {'name': ENSEMBLE, 'members': MEMBERS, |
| 'rule': 'Rank candidates by the equal-weight sum of the members\' contextual full-vocabulary suffix logP sums; ties keep AzooKey order'}, |
| 'scoring': {**confirmation['scoring'], 'packages': v2.versions()}, |
| 'primary': {**PRIMARY, 'metric': 'exact Top-1', |
| 'claim_rule': 'net > 0 and author-cluster sign-flip p < 0.05; and no notation-lenient loss with cluster p < 0.05 for the same pair on either n20 track', |
| 'if_replicated': 'The development ensemble gain replicates on the consumed blind; distill the ensemble (V4.5)', |
| 'if_not_replicated': 'No replicated gain; ensemble distillation is not pursued on this evidence'}, |
| 'secondary': {'tests': SECONDARY, 'metric': 'exact Top-1', |
| 'claim_rule': 'two-sided author/homophone-group cluster p < 0.05 (single test, no adjustment) and the same notation-lenient guard'}, |
| 'descriptive': ['seed band (mean, range, SD) of the four single seeds on each track and pool', |
| 'every two- and three-member ensemble', |
| 'ensemble against each single seed', |
| 'n50 pools, error classes, by source and homophone diagnostics as in confirmation v2'], |
| 'execution': 'One run; a run that stops for a technical failure before summary.json may be repeated into a new directory, recorded in the receipt. Never rerun to change an outcome', |
| 'publication': 'Aggregates and case IDs only', |
| 'code_sha256': {path: file_sha(ROOT / path) for path in CODE}, |
| }) |
| print(f'Wrote {PLAN.relative_to(ROOT)}; commit and push it with this script before `run`.') |
|
|
|
|
| def check_committed(plan): |
| paths = [str(PLAN.relative_to(ROOT)), *plan['code_sha256']] |
| v2.git('ls-files', '--error-unmatch', *paths) |
| if v2.git('status', '--porcelain', '--', *paths): |
| raise ValueError('Commit the plan and code first') |
| if v2.git('rev-list', '--count', '@{u}..HEAD').strip() != '0': |
| raise ValueError('Push the plan before opening the pools') |
| for path, digest in plan['code_sha256'].items(): |
| if file_sha(ROOT / path) != digest: |
| raise ValueError(f'{path} differs from the plan') |
| if v2.versions() != plan['scoring']['packages']: |
| raise ValueError(f'Package versions differ from the plan: {v2.versions()}') |
|
|
|
|
| def run(args): |
| import torch |
| plan = json.loads(PLAN.read_text(encoding='utf-8')) |
| check_committed(plan) |
| receipt = json.loads(RECEIPT.read_text(encoding='utf-8')) if RECEIPT.exists() else {'attempts': []} |
| if any(a['status'] == 'complete' for a in receipt['attempts']): |
| raise ValueError('This plan has already been scored; use `report`.') |
| out = args.output.resolve() |
| if out.exists() and any(out.iterdir()): |
| raise ValueError('Use an empty output directory') |
| for entry in plan['reused_scores']['files'].values(): |
| if file_sha(ROOT / entry['file']) != entry['sha256']: |
| raise ValueError(f"{entry['file']} differs from the plan") |
| torch.backends.cuda.matmul.allow_tf32 = False |
| torch.backends.cudnn.allow_tf32 = False |
| torch.set_float32_matmul_precision('highest') |
| torch.set_num_threads(4) |
| development = v2.load_pool({'directory': plan['preflight']['pool'], 'sha256': plan['preflight']['sha256']}) |
| models = {} |
| for name, spec in plan['models'].items(): |
| lm = v2.load_model(spec) |
| scored = v2.score(lm, development) |
| recorded = read_rows(ROOT / spec['recorded_development']) |
| same = sum(a['id'] == b['id'] and a['orders']['lm_context_sum'] == b['orders']['lm_context_sum'] |
| for a, b in zip(recorded, scored, strict=True)) |
| print(f'preflight {name}: {same}/{len(scored)} recorded development orders reproduced', flush=True) |
| if same != len(scored): |
| raise ValueError(f'{name} does not reproduce its development scores; pools stay closed.') |
| models[name] = lm |
| out.mkdir(parents=True) |
| commit = v2.git('rev-parse', 'HEAD').strip() |
| receipt['plan_sha256'] = file_sha(PLAN) |
| receipt['attempts'].append({'output': str(out.relative_to(ROOT)), 'commit': commit, 'status': 'started', |
| 'started_utc': datetime.now(timezone.utc).isoformat()}) |
| write_json(RECEIPT, receipt) |
| for track, info in plan['tracks'].items(): |
| for pool, entry in info['pools'].items(): |
| rows = v2.load_pool(entry) |
| if len(rows) != info['cases']: |
| raise ValueError(f'{track} {pool}: {len(rows)} cases') |
| dest = out / track / pool |
| dest.mkdir(parents=True, exist_ok=True) |
| for name, lm in models.items(): |
| v2.write_rows(dest / f'{name}.jsonl', v2.score(lm, rows)) |
| print(f'scored {track} {pool} {name}', flush=True) |
| write_json(out / 'run.json', {'plan_sha256': file_sha(PLAN), 'commit': commit, 'gpu': torch.cuda.get_device_name(), |
| 'cuda': torch.version.cuda, 'packages': v2.versions(), |
| 'completed_utc': datetime.now(timezone.utc).isoformat()}) |
| receipt['attempts'][-1].update(status='complete', completed_utc=datetime.now(timezone.utc).isoformat()) |
| write_json(RECEIPT, receipt) |
| report(argparse.Namespace(output=out)) |
|
|
|
|
| def ensemble_rows(rows, members): |
| """Rows whose lm_context_sum order is the members' summed contextual logP ranking. |
| |
| Where the context is not scorable (e.g. longer than the 128-token window) every |
| single model falls back to the AzooKey order; the shared tokenizer makes that |
| fallback identical across members, and the ensemble keeps it too. |
| """ |
| base = rows[members[0]] |
| result = [] |
| for i, row in enumerate(base): |
| order = row['orders']['azookey'] |
| fallback = {rows[name][i]['fallbacks'].get('lm_context_sum') for name in members} |
| if len(fallback) != 1: |
| raise ValueError(f"{row['id']}: members disagree on the contextual fallback") |
| if fallback != {None}: |
| merged = copy.copy(row) |
| merged['orders'] = {**row['orders'], 'lm_context_sum': list(order)} |
| result.append(merged) |
| continue |
| total = {t: 0.0 for t in order} |
| for name in members: |
| assert rows[name][i]['id'] == row['id'] |
| for c in rows[name][i]['lm_scores']['contextual']['candidates']: |
| total[c['text']] += c['log_probability_sum'] |
| merged = copy.copy(row) |
| merged['orders'] = {**row['orders'], 'lm_context_sum': sorted(order, key=lambda t: -total[t])} |
| result.append(merged) |
| return result |
|
|
|
|
| def report(args): |
| import statistics as stats |
| from vimeml.benchmarks.notation import error_class |
| out = args.output.resolve() |
| plan = json.loads(PLAN.read_text(encoding='utf-8')) |
| tracks = {} |
| for track, info in plan['tracks'].items(): |
| tracks[track] = {} |
| for pool in info['pools']: |
| rows = {'v42-e20': read_rows(ROOT / plan['reused_scores']['files'][f'{track}/{pool}']['file'])} |
| rows.update({name: read_rows(out / track / pool / f'{name}.jsonl') for name in plan['models']}) |
| rows = {name: rows[name] for name in MEMBERS} |
| rows[ENSEMBLE] = ensemble_rows(rows, MEMBERS) |
| entry, _ = v2.track_report(rows) |
| exact = [entry['models'][m]['exact'] for m in MEMBERS] |
| lenient = [entry['models'][m]['notation_lenient'] for m in MEMBERS] |
| entry['seed_band'] = {key: {'mean': stats.mean(v), 'min': min(v), 'max': max(v), 'sd': stats.stdev(v)} |
| for key, v in (('exact', exact), ('notation_lenient', lenient))} |
| subsets = {} |
| for size in (2, 3): |
| for members in combinations(MEMBERS, size): |
| merged = ensemble_rows(rows, list(members)) |
| items = [error_class(r, r['orders']['lm_context_sum'][0])[0] for r in merged] |
| subsets['+'.join(members)] = {'exact': items.count('correct'), |
| 'notation_lenient': items.count('correct') + items.count('notation')} |
| entry['subset_ensembles'] = subsets |
| entry['ensemble_size_exact'] = {'1': stats.mean(exact), **{ |
| str(size): stats.mean(v['exact'] for k, v in subsets.items() if k.count('+') == size - 1) |
| for size in (2, 3)}, '4': entry['models'][ENSEMBLE]['exact']} |
| tracks[track][pool] = entry |
|
|
| def test(spec): |
| key = f"{spec['before']}->{spec['after']}" |
| guard = {t: tracks[t]['n20']['paired'][key]['notation_lenient']['author'] for t in plan['tracks']} |
| losses = [t for t, g in guard.items() if g['net'] < 0 and g['cluster_sign_flip_p'] < v2.ALPHA] |
| exact = tracks[spec['track']][spec['pool']]['paired'][key]['exact']['author'] |
| return exact, losses |
|
|
| exact, losses = test(plan['primary']) |
| primary = {**plan['primary'], 'exact': exact, 'notation_lenient_losses': losses, |
| 'replicated': exact['net'] > 0 and exact['cluster_sign_flip_p'] < v2.ALPHA and not losses} |
| secondary = [] |
| for spec in plan['secondary']['tests']: |
| exact, losses = test(spec) |
| significant = exact['cluster_sign_flip_p'] < v2.ALPHA and not losses |
| secondary.append({**spec, 'exact': exact, 'notation_lenient_losses': losses, |
| 'claim': ('improvement' if exact['net'] > 0 else 'regression') if significant else 'none'}) |
| run_info = json.loads((out / 'run.json').read_text(encoding='utf-8')) |
| write_json(out / 'summary.json', {'plan_sha256': file_sha(PLAN), 'run': run_info, 'primary': primary, |
| 'secondary': secondary, 'tracks': tracks, |
| 'report_script_sha256': file_sha(Path(__file__))}) |
| print(json.dumps({'primary': {k: primary[k] for k in ('id', 'exact', 'notation_lenient_losses', 'replicated')}, |
| 'secondary': [{k: s[k] for k in ('id', 'exact', 'claim')} for s in secondary], |
| 'band_and_sizes': {t: {p: {'band': tracks[t][p]['seed_band']['exact'], |
| 'sizes': tracks[t][p]['ensemble_size_exact']} |
| for p in tracks[t]} for t in tracks}}, ensure_ascii=False, indent=1)) |
|
|
|
|
| def main(): |
| parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) |
| subs = parser.add_subparsers(dest='command', required=True) |
| subs.add_parser('plan') |
| for name in ('run', 'report'): |
| subs.add_parser(name).add_argument('--output', type=Path, required=True) |
| args = parser.parse_args() |
| {'plan': make_plan, 'run': run, 'report': report}[args.command](args) |
|
|
|
|
| if __name__ == '__main__': |
| main() |
|
|
