Download scale/script/baseline_dashboard.py from Tinghong-Ye22/code-Movens-Zero_train: direct link, hf CLI and curl.
- Browser
- Download file 12.4 kB
-
https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/scale/script/baseline_dashboard.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Movens-Zero_train/scale/script/baseline_dashboard.py
-
curl -L -o baseline_dashboard.py https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/scale/script/baseline_dashboard.py
12.4 kB
| """Read-only fresh baseline / historical scale comparison on the existing server.""" | |
| import json | |
| import math | |
| from pathlib import Path | |
| import threading | |
| import time | |
| from urllib.parse import urlparse | |
| ROOT = Path(__file__).resolve().parents[2] | |
| LOCAL = Path('/mnt/workspace/Movens-Zero_train/scale/baseline_reference_20261008') | |
| GROUP = ROOT / 'scale/groups/sml_1m_224_a1e-4_o1e-5_nojerk' | |
| FLOW = 'train/loss_group/flow' | |
| def finite(v): | |
| return isinstance(v, (float, int)) and math.isfinite(v) | |
| class Prefix: | |
| """Incrementally read the needed early history, never scan GB logs per refresh.""" | |
| def __init__(self, path): | |
| self.path = path | |
| self.offset = 0 | |
| self.rows = {} | |
| def read(self, stop): | |
| if self.path.stat().st_size < self.offset: | |
| self.offset = 0 | |
| self.rows = {} | |
| with self.path.open('rb') as stream: | |
| stream.seek(self.offset) | |
| while True: | |
| position = stream.tell() | |
| line = stream.readline() | |
| if not line.endswith(b'\n'): | |
| break | |
| row = json.loads(line) | |
| step = int(row['step']) | |
| if step > stop: | |
| break | |
| self.rows[step] = row | |
| self.offset = stream.tell() | |
| return [r for s, r in sorted(self.rows.items()) if s <= stop] | |
| def average(rows, key): | |
| vals = [r['metrics'].get(key) for r in rows] | |
| vals = [v for v in vals if finite(v)] | |
| return sum(vals) / len(vals) if vals else None | |
| def window(rows, end, width): | |
| selected = [r for r in rows if end-width < r['step'] <= end] | |
| return dict(start=max(1, end-width+1), end=end, count=len(selected), | |
| expected=min(end, width), flow=average(selected, FLOW), | |
| tasks={task: average(selected, f'train/loss_task/{task}') | |
| for task in ('text', 'audio', 'video')}, | |
| task_counts={task: sum(finite(r['metrics'].get(f'train/loss_task/{task}')) | |
| for r in selected) for task in ('text', 'audio', 'video')}) | |
| def curve(rows, batch, end, key=FLOW): | |
| width = max(1, math.ceil(end / 150)) | |
| bins = {} | |
| for row in rows: | |
| if row['step'] > end: | |
| continue | |
| value = row['metrics'].get(key) | |
| if finite(value): | |
| bins.setdefault((row['step']-1)//width, []).append((row['step'], value)) | |
| return [[sum(x for x, y in points)/len(points), | |
| batch*sum(x for x, y in points)/len(points), | |
| sum(y for x, y in points)/len(points)] for points in bins.values()] | |
| class Comparison: | |
| def __init__(self): | |
| self.payload = {'loading': True} | |
| self.logs = {} | |
| def log(self, path, stop): | |
| return self.logs.setdefault(str(path), Prefix(path)).read(stop) | |
| def collect(self): | |
| cfg = json.loads((LOCAL/'fresh_config.json').read_text()) | |
| candidates = list((LOCAL/'runs/posttrain_v1_vlm').glob('*/metrics.jsonl')) | |
| if len(candidates) != 1: | |
| raise RuntimeError(f'Expected exactly one baseline run, found {len(candidates)}') | |
| base = self.log(candidates[0], cfg['train']['max_training_steps']) | |
| if not base: | |
| raise RuntimeError('Baseline has no metrics yet') | |
| end = base[-1]['step'] | |
| base_win = window(base, end, 100) | |
| now = time.time() | |
| age = now-base[-1]['time'] | |
| result = dict(loading=False, generated_at=now, baseline=dict( | |
| step=end, target=cfg['train']['max_training_steps'], age=age, | |
| status='日志持续更新' if age < 180 else '日志已停更,请检查训练进程', | |
| elapsed_s=base[-1]['time']-base[0]['time'], samples=end*384, | |
| loss=base_win['flow'], initial_loss=window(base,min(100,end),100)['flow'], | |
| seconds=average(base[-100:], 'train/seconds_per_step'), | |
| lr_action=base[-1]['metrics'].get('train/lr_action_header'), | |
| lr_omni=base[-1]['metrics'].get('train/lr_vlm_model'), | |
| metrics_path=str(candidates[0]), | |
| checkpoints=sorted(int(p.parent.name[5:]) for p in candidates[0].parent.glob('checkpoints/ckpt_*/checkpoint_complete.json'))), | |
| modes={}, curves={}, errors=[]) | |
| if (LOCAL/'user_stopped.json').exists():result['baseline']['status']='已按用户要求停止(保留历史)' | |
| elif (LOCAL/'user_stop_request.json').exists():result['baseline']['status']='等待2000步保存完成后停止' | |
| sources = {'Baseline': (base, 384, cfg, str(candidates[0]))} | |
| highlocal=LOCAL.with_name('baseline_reference_highlr_20261008') | |
| highpaths=list((highlocal/'runs/posttrain_v1_vlm').glob('*/metrics.jsonl')) | |
| result['highlr']=dict(step=0,status='启动中,等待第一步',gpu=1) | |
| paired_end=0 | |
| if len(highpaths)==1: | |
| highrows=self.log(highpaths[0],cfg['train']['max_training_steps']) | |
| if highrows: | |
| highcfg=json.loads((highlocal/'fresh_config.json').read_text()) | |
| sources['HighLR']=(highrows,384,highcfg,str(highpaths[0])) | |
| highage=now-highrows[-1]['time'] | |
| result['highlr']=dict(step=highrows[-1]['step'],loss=average(highrows[-100:],FLOW), | |
| age=highage,status='日志持续更新' if highage<180 else '日志停更',gpu=1, | |
| seconds=average(highrows[-100:],'train/seconds_per_step'), | |
| elapsed_s=highrows[-1]['time']-highrows[0]['time'],samples=highrows[-1]['step']*384, | |
| initial_loss=window(highrows,min(100,highrows[-1]['step']),100)['flow'], | |
| lr_action=highrows[-1]['metrics'].get('train/lr_action_header'), | |
| lr_omni=highrows[-1]['metrics'].get('train/lr_vlm_model'), | |
| checkpoints=sorted(int(p.parent.name[5:]) for p in highpaths[0].parent.glob('checkpoints/ckpt_*/checkpoint_complete.json'))) | |
| paired_end=min(end,highrows[-1]['step']) | |
| # The active experiment, not the stopped control, sets the dashboard horizon. | |
| end=highrows[-1]['step'] | |
| result['active_name']='HighLR' if 'HighLR' in sources else 'Baseline' | |
| if (highlocal/'user_stopped.json').exists(): | |
| result['highlr']['status']='已停止,为四卡条件稳定性实验腾卡;完整ckpt_2000保留' | |
| result['active']=result['highlr'] if 'HighLR' in sources else result['baseline'] | |
| active_rows=sources[result['active_name']][0] | |
| for size in ('S','M','L'): | |
| try: | |
| run = Path(json.loads((GROUP/f'runs/latest_{size}.json').read_text())['run_dir']) | |
| rows = self.log(run/'metrics.jsonl', end*8) | |
| config = json.loads((run/'run_config.json').read_text()) | |
| batch = config['train']['train_batch_size']*config.get('world_size_expected',1)*config['train'].get('gradient_accumulation_steps',1) | |
| if batch != 48: | |
| raise ValueError(f'{size} expected historical global batch 48, got {batch}') | |
| sources[size] = rows, batch, config, str(run/'metrics.jsonl') | |
| except Exception as exc: | |
| result['errors'].append(f'{size}: {exc}') | |
| result['paired_end']=paired_end | |
| for mode in ('steps', 'samples','paired'): | |
| table = [] | |
| selected={n:v for n,v in sources.items() if mode!='paired' or n in ('Baseline','HighLR')} | |
| for name, (rows, batch, config, path) in selected.items(): | |
| target = paired_end if mode=='paired' else end if mode == 'steps' else end*384//batch | |
| width = 100 if mode == 'steps' else 100*384//batch | |
| stats = window(rows,target,width) | |
| complete=bool(target and rows and rows[-1]['step']>=target and stats['count']==stats['expected']) | |
| if not complete: | |
| stats['flow']=None;stats['tasks']={t:None for t in ('text','audio','video')} | |
| reference=window(base,paired_end,100)['flow'] if mode=='paired' else window(active_rows,end,100)['flow'] | |
| delta = ((stats['flow']/reference-1)*100 | |
| if stats['flow'] is not None and reference else None) | |
| table.append(dict(name=name, batch=batch, samples=target*batch, | |
| delta_pct=delta, **stats)) | |
| result['modes'][mode] = table | |
| result['curves'][mode] = {name:curve(rows,batch,paired_end if mode=='paired' else end if mode=='steps' else end*384//batch) | |
| for name,(rows,batch,config,path) in selected.items()} | |
| result['sources'] = {name:dict(metrics=path,batch=batch, | |
| recorded_action_lr=rows[-1]['metrics'].get('train/lr_action_header') if rows else None, | |
| aux_weight=config['train'].get('absolute_height_loss_weight'), | |
| cfg_dropout=config['train'].get('cfg_condition_dropout_prob',0), | |
| hidden=config['model'].get('hidden_dim'),depth=config['model'].get('num_blocks')) | |
| for name,(rows,batch,config,path) in sources.items()} | |
| backup=highlocal/'backup_status.json' if 'HighLR' in sources else LOCAL/'backup_live_status.json' | |
| if not backup.exists():backup=LOCAL/'backup_status.json' | |
| result['backup']=json.loads(backup.read_text()) if backup.exists() else None | |
| preview=highlocal/'previews/web/status.json' | |
| result['preview']=json.loads(preview.read_text()) if preview.exists() else dict(status='waiting_checkpoint',models=[]) | |
| self.payload=result | |
| def poll(self): | |
| while True: | |
| try: | |
| self.collect() | |
| except Exception as exc: | |
| self.payload={**self.payload,'error':str(exc),'failed_at':time.time()} | |
| time.sleep(10) | |
| def handler_with_baseline(parent, comparison): | |
| import server | |
| from types import SimpleNamespace | |
| gallery=server.PreviewGallery('reference-highlr','HighLR',LOCAL.with_name('baseline_reference_highlr_20261008')/'previews',True) | |
| media_handler=server.handler_for(SimpleNamespace(previews={'reference-highlr':gallery})) | |
| study_gallery=server.PreviewGallery('condition-study','条件稳定性',Path('/mnt/workspace/Movens-Zero_train/scale/condition_stability_20261009/previews'),True) | |
| study_media=server.handler_for(SimpleNamespace(previews={'condition-study':study_gallery})) | |
| class Handler(parent): | |
| def do_GET(self): | |
| route=urlparse(self.path).path.rstrip('/') | |
| if '/media/condition-study/' in route: | |
| return study_media.serve_media(self,'condition-study',route.split('/media/condition-study/',1)[1]) | |
| if route.endswith('/condition-study') or route.endswith('/api/condition-study'): | |
| base=ROOT/'scale/experiments/condition_stability_20261009' | |
| is_api=route.endswith('/api/condition-study') | |
| file=base/('status.json' if is_api else 'dashboard.html') | |
| if not file.exists():return self.send_error(503,'Experiment snapshot preparing') | |
| data=file.read_bytes() | |
| self.send_response(200) | |
| self.send_header('Content-Type','application/json; charset=utf-8' if is_api else 'text/html; charset=utf-8') | |
| self.send_header('Content-Length',str(len(data)));self.send_header('Cache-Control','no-store') | |
| self.end_headers();self.wfile.write(data);return | |
| if '/media/reference-highlr/' in route: | |
| return media_handler.serve_media(self,'reference-highlr',route.split('/media/reference-highlr/',1)[1]) | |
| if route.endswith('/api/baseline'): | |
| data=json.dumps(comparison.payload,allow_nan=False).encode() | |
| content_type='application/json; charset=utf-8' | |
| elif route.endswith('/baseline'): | |
| data=(ROOT/'scale/dashboard/baseline.html').read_bytes() | |
| content_type='text/html; charset=utf-8' | |
| else: | |
| return super().do_GET() | |
| self.send_response(200) | |
| self.send_header('Content-Type',content_type) | |
| self.send_header('Cache-Control','no-store') | |
| self.send_header('Content-Length',str(len(data))) | |
| self.end_headers() | |
| self.wfile.write(data) | |
| return Handler | |