| """Matched full FP32 choice/IFEval/chat evaluation for a merged adapter export.""" |
| from pathlib import Path |
| import json |
| import os |
| import shutil |
| import subprocess |
| import sys |
|
|
| ROOT = Path(__file__).resolve().parents[2] |
| sys.path.insert(0, str(ROOT/'pretraining')) |
| from sml_v1.common import read_json, atomic_json, file_sha256 |
|
|
| BASELINE = ROOT/'reports/benchmarks/deep44-reliable-chat-167-fp32-20261106' |
|
|
|
|
| def verify_scoring(): |
| receipt = read_json(BASELINE/'readiness.json') |
| names = ['pretraining/scripts/benchmark_best.py', 'pretraining/scripts/benchmark_ifeval.py', |
| 'pretraining/sml_v1/model.py', 'pretraining/sml_v1/tokenization.py', |
| 'pretraining/sft/data.py', 'pretraining/sft/evaluate.py', |
| 'pretraining/chat_benchmark/panel.py', 'pretraining/chat_benchmark/scoring.py'] |
| for name in names: |
| if file_sha256(ROOT/name) != receipt['files'][name]: |
| raise ValueError('Scoring no longer matches 167: '+name) |
| return {n:receipt['files'][n] for n in names} |
|
|
|
|
| def chat(snapshot, output, training_data): |
| import mlx.core as mx |
| from mlx.utils import tree_flatten |
| from sml_v1.model import TransformerLM, TransformerConfig |
| from sml_v1.tokenization import Tokenizer |
| from sft.evaluate import generate |
| from chat_benchmark.panel import build |
| from chat_benchmark.scoring import score, summarize, normalize |
| cases = build() |
| expected = [json.loads(l) for l in (BASELINE/'chat/cases.jsonl').read_text().splitlines()] |
| if cases != expected: raise ValueError('Private chat cases differ from 167') |
| prompts = {normalize(m['content']) for l in Path(training_data).read_text().splitlines() |
| for m in json.loads(l)['messages'] if m['role']=='user'} |
| if any(normalize(c['messages'][-1]['content']) in prompts for c in cases): |
| raise ValueError('Private chat prompt overlaps training') |
| meta = read_json(snapshot/'model.safetensors.json') |
| model = TransformerLM(TransformerConfig(**dict(meta['recipe']['model'], attention_impl='vanilla', |
| ffn_impl='reference', ce_impl='reference'))) |
| model.load_weights(str(snapshot/'model.safetensors'), strict=True) |
| mx.eval(model.parameters()); model.eval() |
| if any(v.dtype != mx.float32 for _,v in tree_flatten(model.parameters())): |
| raise ValueError('Chat evaluation requires FP32 weights') |
| tok = Tokenizer(snapshot/'tokenizer') |
| output.mkdir(parents=True, exist_ok=True) |
| path = output/'responses.jsonl' |
| rows = [json.loads(l) for l in path.read_text().splitlines()] if path.exists() else [] |
| if len(rows)>len(cases) or any(r['case']!=c for r,c in zip(rows,cases)): |
| raise ValueError('Saved chat responses differ') |
| with path.open('a') as stream: |
| for case in cases[len(rows):]: |
| value=generate(model,tok,case['messages'],limit=case['max_tokens'],temperature=0) |
| row=dict(case=case,response=value,scores=score(case,value));rows.append(row) |
| stream.write(json.dumps(row,ensure_ascii=False)+'\n');stream.flush() |
| atomic_json(output/'progress.json',dict(completed=len(rows),total=len(cases))) |
| result=summarize(rows) |
| atomic_json(output/'results.json',result) |
| atomic_json(output/'protocol.json',dict(scoring=verify_scoring(),cases=file_sha256(BASELINE/'chat/cases.jsonl'), |
| precision='FP32 base masters plus FP32 trained deltas',model_sha256=file_sha256(snapshot/'model.safetensors'), |
| weight_provenance=meta['inference_export'],temperature=0,repetition_penalty=1, |
| attention='vanilla',ffn='reference',system_prompt=None,prompt_truncation=False)) |
| return result |
|
|
|
|
| def run(bundle, staging, output): |
| verify_scoring() |
| bundle,staging,output=map(Path,(bundle,staging,output)) |
| output.mkdir(parents=True,exist_ok=True) |
| snapshot=output/'choices/checkpoint' |
| if not snapshot.exists(): |
| snapshot.mkdir(parents=True) |
| source=bundle/'inference' |
| manifest=read_json(source/'manifest.json') |
| for name in ('model.safetensors','model.safetensors.json'): |
| if file_sha256(source/name)!=manifest['files'][name]['sha256']: |
| raise ValueError('Merged export checksum mismatch') |
| shutil.copyfile(source/name,snapshot/name) |
| shutil.copytree(staging/'tokenizer',snapshot/'tokenizer') |
| meta=read_json(snapshot/'model.safetensors.json') |
| atomic_json(snapshot/'manifest.json',dict(format='sml-benchmark-snapshot-v1',source_bundle=bundle.name, |
| step=meta['step'],tokens=meta['tokens'],phase='SFT',files=manifest['files'])) |
| cfg=read_json(ROOT/'configs/benchmark_reliable_chat_167_fp32_20261106.json') |
| cfg.update(run_dir=str(bundle.parent),output_dir=str(output/'choices'),explicit_checkpoint=str(bundle/'inference')) |
| config=output/'config.json';atomic_json(config,cfg) |
| if not (output/'choices/datasets').exists(): |
| shutil.copytree(BASELINE/'choices/datasets',output/'choices/datasets') |
| (output/'ifeval').mkdir(exist_ok=True) |
| for name in ('inputs.json','inputs.sha256.json'): |
| target=output/'ifeval'/name |
| if not target.exists():shutil.copyfile(BASELINE/'ifeval'/name,target) |
| env=dict(os.environ,HF_HUB_OFFLINE='1',HF_DATASETS_OFFLINE='1',HF_HUB_DISABLE_IMPLICIT_TOKEN='1',TOKENIZERS_PARALLELISM='false') |
| commands=[('choices-prepare',[sys.executable,str(ROOT/'pretraining/scripts/benchmark_best.py'),'--config',str(config),'--prepare']), |
| ('choices',[sys.executable,'-u',str(ROOT/'pretraining/scripts/benchmark_best.py'),'--config',str(config)]), |
| ('ifeval',[sys.executable,'-u',str(ROOT/'pretraining/scripts/benchmark_ifeval.py'),'--checkpoint',str(snapshot), |
| '--tokenizer',str(snapshot/'tokenizer'),'--output',str(output/'ifeval'),'--max-tokens','1024'])] |
| for label,command in commands: |
| with (output/(label+'.log')).open('a') as log: |
| subprocess.run(command,cwd=ROOT,env=env,stdout=log,stderr=subprocess.STDOUT,check=True) |
| choices=read_json(output/'choices/results.json');ifeval=read_json(output/'ifeval/results.json') |
| if choices['status']!='complete' or not ifeval['complete'] or ifeval['evaluated']!=541: |
| raise ValueError('Full benchmark did not complete') |
| for name,spec in cfg['tasks'].items(): |
| if not choices['results'][name]['complete'] or choices['results'][name]['evaluated']!=spec['expected_examples']: |
| raise ValueError('Incomplete choice task') |
| old=read_json(BASELINE/'ifeval/protocol.json');new=ifeval['protocol'] |
| for key in ('dataset','dataset_revision','google_research_commit','evaluator_files','runner_sha256', |
| 'temperature','repetition_penalty','max_tokens','system_prompt','serialization', |
| 'prompt_truncation','context_limit','compute_precision','model_implementation','mlx_enable_tf32'): |
| if old[key]!=new[key]:raise ValueError('IFEval differs from 167: '+key) |
| result=dict(choices=choices['results'],ifeval=ifeval['metrics'], |
| chat=chat(snapshot,output/'chat',staging/'corpus/train.jsonl'), |
| complete=True,model_sha256=file_sha256(snapshot/'model.safetensors')) |
| atomic_json(output/'summary.json',result) |
| return result |
|
|
|
|
| if __name__=='__main__': |
| if len(sys.argv)!=4:raise SystemExit('bundle staging output required') |
| run(*sys.argv[1:]) |
|
|
