File size: 15,294 Bytes
81cffe7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 | """Preregistered second use of Standard v2 blind and the homophone set: V4.2-recipe seed ensemble.
The four V4.2-recipe seeds (V4.2 epoch 20 and V4.4 s1-s3) were combined only after the V4.4
analysis on development (docs/zh-CN/training-v44.md). Both pools were consumed once by
confirmation v2, and no model or ensemble rule was tuned on them since, so this is a
replication: its outcome is descriptive evidence, not a deployment confirmation.
`plan` writes configs/ime-ensemble-replication-v2.json (hypotheses, snapshots, the reused
V4.2 confirmation score files, pool and code hashes); it reads no benchmark text and must be
committed and pushed before `run`. `run` refuses unless the plan and code are committed and
unchanged, s1-s3 reproduce their recorded development orders, and no completed attempt
exists; it then scores s1-s3 once on the n20 and n50 pools, reuses the recorded V4.2 scores,
and builds the ensembles. `report` recomputes summary.json without scoring. Loading,
scoring and per-track statistics are those of `confirm_ime_v2.py`.
"""
import argparse
import copy
import json
import sys
from datetime import datetime, timezone
from itertools import combinations
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'src'))
sys.path.insert(0, str(ROOT / 'scripts/benchmarks'))
import confirm_ime_v2 as v2
from vimeml.training.data import file_sha, write_json
from vimeml.benchmarks.comparison import rows as read_rows
PLAN = ROOT / 'configs/ime-ensemble-replication-v2.json'
RECEIPT = ROOT / v2.IMPORTED / 'ensemble-replication.json'
CONFIRMATION = 'outputs/confirmation-v2-20261021'
SEEDS = {arm: {'checkpoint': f'outputs/training-v44-20261105/remote/{arm}/checkpoints/epoch-20.pt',
'tokenizer': 'artifacts/tokenizers/v4-a/ja-unigram-8192', 'loader': 'v4',
'recorded_development': f'outputs/evaluation-v44-20261105/{arm}/standard-development/scores.jsonl'}
for arm in ('s1', 's2', 's3')}
MEMBERS = ['v42-e20', 's1', 's2', 's3']
ENSEMBLE = 'ens4'
PRIMARY = {'id': 'R1', 'before': 'v42-e20', 'after': ENSEMBLE, 'track': 'standard-v2-blind', 'pool': 'n20',
'question': 'Does the four-seed ensemble improve exact Top-1 over the deployed V4.2 epoch 20?'}
SECONDARY = [{'id': 'R2', 'before': 'v42-e20', 'after': ENSEMBLE, 'track': 'homophone-v1', 'pool': 'n20',
'question': 'The same on written-text homophones'}]
CODE = ['scripts/benchmarks/replicate_ensemble_v2.py', *v2.CODE]
def make_plan(args):
if PLAN.exists():
raise ValueError('The plan already exists.')
if RECEIPT.exists():
raise ValueError('These pools have already been opened for this replication.')
import torch
confirmation = json.loads(v2.PLAN.read_text(encoding='utf-8'))
models = {}
for name, spec in SEEDS.items():
path = ROOT / spec['checkpoint']
saved = torch.load(path, map_location='cpu', weights_only=True)
models[name] = {**spec, 'sha256': file_sha(path), 'bytes': path.stat().st_size, 'step': saved['step']}
del saved
reused = {f'{track}/{pool}': {'file': f'{CONFIRMATION}/{track}/{pool}/v42-e20.jsonl',
'sha256': file_sha(ROOT / CONFIRMATION / track / pool / 'v42-e20.jsonl')}
for track, info in confirmation['tracks'].items() for pool in info['pools']}
write_json(PLAN, {
'version': 'ime-ensemble-replication-v2',
'created_utc': datetime.now(timezone.utc).isoformat(),
'metric_policy': 'configs/ime-metrics-v2.json',
'status_of_sets': ('Second use. Both pools were consumed once by configs/ime-confirmation-v2.json; '
'the ensemble idea came from development after that. Outcomes are descriptive replication '
'evidence; any deployment claim needs a fresh blind release'),
'tracks': confirmation['tracks'],
'preflight': {**confirmation['preflight'],
'requirement': 's1-s3 reproduce their recorded development lm_context_sum order on all 728 cases before any blind or homophone file is read'},
'models': models,
'reused_scores': {'model': 'v42-e20', 'source': 'configs/ime-confirmation-v2.json run', 'files': reused},
'ensemble': {'name': ENSEMBLE, 'members': MEMBERS,
'rule': 'Rank candidates by the equal-weight sum of the members\' contextual full-vocabulary suffix logP sums; ties keep AzooKey order'},
'scoring': {**confirmation['scoring'], 'packages': v2.versions()},
'primary': {**PRIMARY, 'metric': 'exact Top-1',
'claim_rule': 'net > 0 and author-cluster sign-flip p < 0.05; and no notation-lenient loss with cluster p < 0.05 for the same pair on either n20 track',
'if_replicated': 'The development ensemble gain replicates on the consumed blind; distill the ensemble (V4.5)',
'if_not_replicated': 'No replicated gain; ensemble distillation is not pursued on this evidence'},
'secondary': {'tests': SECONDARY, 'metric': 'exact Top-1',
'claim_rule': 'two-sided author/homophone-group cluster p < 0.05 (single test, no adjustment) and the same notation-lenient guard'},
'descriptive': ['seed band (mean, range, SD) of the four single seeds on each track and pool',
'every two- and three-member ensemble',
'ensemble against each single seed',
'n50 pools, error classes, by source and homophone diagnostics as in confirmation v2'],
'execution': 'One run; a run that stops for a technical failure before summary.json may be repeated into a new directory, recorded in the receipt. Never rerun to change an outcome',
'publication': 'Aggregates and case IDs only',
'code_sha256': {path: file_sha(ROOT / path) for path in CODE},
})
print(f'Wrote {PLAN.relative_to(ROOT)}; commit and push it with this script before `run`.')
def check_committed(plan):
paths = [str(PLAN.relative_to(ROOT)), *plan['code_sha256']]
v2.git('ls-files', '--error-unmatch', *paths)
if v2.git('status', '--porcelain', '--', *paths):
raise ValueError('Commit the plan and code first')
if v2.git('rev-list', '--count', '@{u}..HEAD').strip() != '0':
raise ValueError('Push the plan before opening the pools')
for path, digest in plan['code_sha256'].items():
if file_sha(ROOT / path) != digest:
raise ValueError(f'{path} differs from the plan')
if v2.versions() != plan['scoring']['packages']:
raise ValueError(f'Package versions differ from the plan: {v2.versions()}')
def run(args):
import torch
plan = json.loads(PLAN.read_text(encoding='utf-8'))
check_committed(plan)
receipt = json.loads(RECEIPT.read_text(encoding='utf-8')) if RECEIPT.exists() else {'attempts': []}
if any(a['status'] == 'complete' for a in receipt['attempts']):
raise ValueError('This plan has already been scored; use `report`.')
out = args.output.resolve()
if out.exists() and any(out.iterdir()):
raise ValueError('Use an empty output directory')
for entry in plan['reused_scores']['files'].values():
if file_sha(ROOT / entry['file']) != entry['sha256']:
raise ValueError(f"{entry['file']} differs from the plan")
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
torch.set_float32_matmul_precision('highest')
torch.set_num_threads(4)
development = v2.load_pool({'directory': plan['preflight']['pool'], 'sha256': plan['preflight']['sha256']})
models = {}
for name, spec in plan['models'].items():
lm = v2.load_model(spec)
scored = v2.score(lm, development)
recorded = read_rows(ROOT / spec['recorded_development'])
same = sum(a['id'] == b['id'] and a['orders']['lm_context_sum'] == b['orders']['lm_context_sum']
for a, b in zip(recorded, scored, strict=True))
print(f'preflight {name}: {same}/{len(scored)} recorded development orders reproduced', flush=True)
if same != len(scored):
raise ValueError(f'{name} does not reproduce its development scores; pools stay closed.')
models[name] = lm
out.mkdir(parents=True)
commit = v2.git('rev-parse', 'HEAD').strip()
receipt['plan_sha256'] = file_sha(PLAN)
receipt['attempts'].append({'output': str(out.relative_to(ROOT)), 'commit': commit, 'status': 'started',
'started_utc': datetime.now(timezone.utc).isoformat()})
write_json(RECEIPT, receipt)
for track, info in plan['tracks'].items():
for pool, entry in info['pools'].items():
rows = v2.load_pool(entry)
if len(rows) != info['cases']:
raise ValueError(f'{track} {pool}: {len(rows)} cases')
dest = out / track / pool
dest.mkdir(parents=True, exist_ok=True)
for name, lm in models.items():
v2.write_rows(dest / f'{name}.jsonl', v2.score(lm, rows))
print(f'scored {track} {pool} {name}', flush=True)
write_json(out / 'run.json', {'plan_sha256': file_sha(PLAN), 'commit': commit, 'gpu': torch.cuda.get_device_name(),
'cuda': torch.version.cuda, 'packages': v2.versions(),
'completed_utc': datetime.now(timezone.utc).isoformat()})
receipt['attempts'][-1].update(status='complete', completed_utc=datetime.now(timezone.utc).isoformat())
write_json(RECEIPT, receipt)
report(argparse.Namespace(output=out))
def ensemble_rows(rows, members):
"""Rows whose lm_context_sum order is the members' summed contextual logP ranking.
Where the context is not scorable (e.g. longer than the 128-token window) every
single model falls back to the AzooKey order; the shared tokenizer makes that
fallback identical across members, and the ensemble keeps it too.
"""
base = rows[members[0]]
result = []
for i, row in enumerate(base):
order = row['orders']['azookey']
fallback = {rows[name][i]['fallbacks'].get('lm_context_sum') for name in members}
if len(fallback) != 1:
raise ValueError(f"{row['id']}: members disagree on the contextual fallback")
if fallback != {None}:
merged = copy.copy(row)
merged['orders'] = {**row['orders'], 'lm_context_sum': list(order)}
result.append(merged)
continue
total = {t: 0.0 for t in order}
for name in members:
assert rows[name][i]['id'] == row['id']
for c in rows[name][i]['lm_scores']['contextual']['candidates']:
total[c['text']] += c['log_probability_sum']
merged = copy.copy(row)
merged['orders'] = {**row['orders'], 'lm_context_sum': sorted(order, key=lambda t: -total[t])}
result.append(merged)
return result
def report(args):
import statistics as stats
from vimeml.benchmarks.notation import error_class
out = args.output.resolve()
plan = json.loads(PLAN.read_text(encoding='utf-8'))
tracks = {}
for track, info in plan['tracks'].items():
tracks[track] = {}
for pool in info['pools']:
rows = {'v42-e20': read_rows(ROOT / plan['reused_scores']['files'][f'{track}/{pool}']['file'])}
rows.update({name: read_rows(out / track / pool / f'{name}.jsonl') for name in plan['models']})
rows = {name: rows[name] for name in MEMBERS}
rows[ENSEMBLE] = ensemble_rows(rows, MEMBERS)
entry, _ = v2.track_report(rows)
exact = [entry['models'][m]['exact'] for m in MEMBERS]
lenient = [entry['models'][m]['notation_lenient'] for m in MEMBERS]
entry['seed_band'] = {key: {'mean': stats.mean(v), 'min': min(v), 'max': max(v), 'sd': stats.stdev(v)}
for key, v in (('exact', exact), ('notation_lenient', lenient))}
subsets = {}
for size in (2, 3):
for members in combinations(MEMBERS, size):
merged = ensemble_rows(rows, list(members))
items = [error_class(r, r['orders']['lm_context_sum'][0])[0] for r in merged]
subsets['+'.join(members)] = {'exact': items.count('correct'),
'notation_lenient': items.count('correct') + items.count('notation')}
entry['subset_ensembles'] = subsets
entry['ensemble_size_exact'] = {'1': stats.mean(exact), **{
str(size): stats.mean(v['exact'] for k, v in subsets.items() if k.count('+') == size - 1)
for size in (2, 3)}, '4': entry['models'][ENSEMBLE]['exact']}
tracks[track][pool] = entry
def test(spec):
key = f"{spec['before']}->{spec['after']}"
guard = {t: tracks[t]['n20']['paired'][key]['notation_lenient']['author'] for t in plan['tracks']}
losses = [t for t, g in guard.items() if g['net'] < 0 and g['cluster_sign_flip_p'] < v2.ALPHA]
exact = tracks[spec['track']][spec['pool']]['paired'][key]['exact']['author']
return exact, losses
exact, losses = test(plan['primary'])
primary = {**plan['primary'], 'exact': exact, 'notation_lenient_losses': losses,
'replicated': exact['net'] > 0 and exact['cluster_sign_flip_p'] < v2.ALPHA and not losses}
secondary = []
for spec in plan['secondary']['tests']:
exact, losses = test(spec)
significant = exact['cluster_sign_flip_p'] < v2.ALPHA and not losses
secondary.append({**spec, 'exact': exact, 'notation_lenient_losses': losses,
'claim': ('improvement' if exact['net'] > 0 else 'regression') if significant else 'none'})
run_info = json.loads((out / 'run.json').read_text(encoding='utf-8'))
write_json(out / 'summary.json', {'plan_sha256': file_sha(PLAN), 'run': run_info, 'primary': primary,
'secondary': secondary, 'tracks': tracks,
'report_script_sha256': file_sha(Path(__file__))})
print(json.dumps({'primary': {k: primary[k] for k in ('id', 'exact', 'notation_lenient_losses', 'replicated')},
'secondary': [{k: s[k] for k in ('id', 'exact', 'claim')} for s in secondary],
'band_and_sizes': {t: {p: {'band': tracks[t][p]['seed_band']['exact'],
'sizes': tracks[t][p]['ensemble_size_exact']}
for p in tracks[t]} for t in tracks}}, ensure_ascii=False, indent=1))
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
subs = parser.add_subparsers(dest='command', required=True)
subs.add_parser('plan')
for name in ('run', 'report'):
subs.add_parser(name).add_argument('--output', type=Path, required=True)
args = parser.parse_args()
{'plan': make_plan, 'run': run, 'report': report}[args.command](args)
if __name__ == '__main__':
main()
|