"""Read-only fresh baseline / historical scale comparison on the existing server.""" import json import math from pathlib import Path import threading import time from urllib.parse import urlparse ROOT = Path(__file__).resolve().parents[2] LOCAL = Path('/mnt/workspace/Movens-Zero_train/scale/baseline_reference_20261008') GROUP = ROOT / 'scale/groups/sml_1m_224_a1e-4_o1e-5_nojerk' FLOW = 'train/loss_group/flow' def finite(v): return isinstance(v, (float, int)) and math.isfinite(v) class Prefix: """Incrementally read the needed early history, never scan GB logs per refresh.""" def __init__(self, path): self.path = path self.offset = 0 self.rows = {} def read(self, stop): if self.path.stat().st_size < self.offset: self.offset = 0 self.rows = {} with self.path.open('rb') as stream: stream.seek(self.offset) while True: position = stream.tell() line = stream.readline() if not line.endswith(b'\n'): break row = json.loads(line) step = int(row['step']) if step > stop: break self.rows[step] = row self.offset = stream.tell() return [r for s, r in sorted(self.rows.items()) if s <= stop] def average(rows, key): vals = [r['metrics'].get(key) for r in rows] vals = [v for v in vals if finite(v)] return sum(vals) / len(vals) if vals else None def window(rows, end, width): selected = [r for r in rows if end-width < r['step'] <= end] return dict(start=max(1, end-width+1), end=end, count=len(selected), expected=min(end, width), flow=average(selected, FLOW), tasks={task: average(selected, f'train/loss_task/{task}') for task in ('text', 'audio', 'video')}, task_counts={task: sum(finite(r['metrics'].get(f'train/loss_task/{task}')) for r in selected) for task in ('text', 'audio', 'video')}) def curve(rows, batch, end, key=FLOW): width = max(1, math.ceil(end / 150)) bins = {} for row in rows: if row['step'] > end: continue value = row['metrics'].get(key) if finite(value): bins.setdefault((row['step']-1)//width, []).append((row['step'], value)) return [[sum(x for x, y in points)/len(points), batch*sum(x for x, y in points)/len(points), sum(y for x, y in points)/len(points)] for points in bins.values()] class Comparison: def __init__(self): self.payload = {'loading': True} self.logs = {} def log(self, path, stop): return self.logs.setdefault(str(path), Prefix(path)).read(stop) def collect(self): cfg = json.loads((LOCAL/'fresh_config.json').read_text()) candidates = list((LOCAL/'runs/posttrain_v1_vlm').glob('*/metrics.jsonl')) if len(candidates) != 1: raise RuntimeError(f'Expected exactly one baseline run, found {len(candidates)}') base = self.log(candidates[0], cfg['train']['max_training_steps']) if not base: raise RuntimeError('Baseline has no metrics yet') end = base[-1]['step'] base_win = window(base, end, 100) now = time.time() age = now-base[-1]['time'] result = dict(loading=False, generated_at=now, baseline=dict( step=end, target=cfg['train']['max_training_steps'], age=age, status='日志持续更新' if age < 180 else '日志已停更,请检查训练进程', elapsed_s=base[-1]['time']-base[0]['time'], samples=end*384, loss=base_win['flow'], initial_loss=window(base,min(100,end),100)['flow'], seconds=average(base[-100:], 'train/seconds_per_step'), lr_action=base[-1]['metrics'].get('train/lr_action_header'), lr_omni=base[-1]['metrics'].get('train/lr_vlm_model'), metrics_path=str(candidates[0]), checkpoints=sorted(int(p.parent.name[5:]) for p in candidates[0].parent.glob('checkpoints/ckpt_*/checkpoint_complete.json'))), modes={}, curves={}, errors=[]) if (LOCAL/'user_stopped.json').exists():result['baseline']['status']='已按用户要求停止(保留历史)' elif (LOCAL/'user_stop_request.json').exists():result['baseline']['status']='等待2000步保存完成后停止' sources = {'Baseline': (base, 384, cfg, str(candidates[0]))} highlocal=LOCAL.with_name('baseline_reference_highlr_20261008') highpaths=list((highlocal/'runs/posttrain_v1_vlm').glob('*/metrics.jsonl')) result['highlr']=dict(step=0,status='启动中,等待第一步',gpu=1) paired_end=0 if len(highpaths)==1: highrows=self.log(highpaths[0],cfg['train']['max_training_steps']) if highrows: highcfg=json.loads((highlocal/'fresh_config.json').read_text()) sources['HighLR']=(highrows,384,highcfg,str(highpaths[0])) highage=now-highrows[-1]['time'] result['highlr']=dict(step=highrows[-1]['step'],loss=average(highrows[-100:],FLOW), age=highage,status='日志持续更新' if highage<180 else '日志停更',gpu=1, seconds=average(highrows[-100:],'train/seconds_per_step'), elapsed_s=highrows[-1]['time']-highrows[0]['time'],samples=highrows[-1]['step']*384, initial_loss=window(highrows,min(100,highrows[-1]['step']),100)['flow'], lr_action=highrows[-1]['metrics'].get('train/lr_action_header'), lr_omni=highrows[-1]['metrics'].get('train/lr_vlm_model'), checkpoints=sorted(int(p.parent.name[5:]) for p in highpaths[0].parent.glob('checkpoints/ckpt_*/checkpoint_complete.json'))) paired_end=min(end,highrows[-1]['step']) # The active experiment, not the stopped control, sets the dashboard horizon. end=highrows[-1]['step'] result['active_name']='HighLR' if 'HighLR' in sources else 'Baseline' if (highlocal/'user_stopped.json').exists(): result['highlr']['status']='已停止,为四卡条件稳定性实验腾卡;完整ckpt_2000保留' result['active']=result['highlr'] if 'HighLR' in sources else result['baseline'] active_rows=sources[result['active_name']][0] for size in ('S','M','L'): try: run = Path(json.loads((GROUP/f'runs/latest_{size}.json').read_text())['run_dir']) rows = self.log(run/'metrics.jsonl', end*8) config = json.loads((run/'run_config.json').read_text()) batch = config['train']['train_batch_size']*config.get('world_size_expected',1)*config['train'].get('gradient_accumulation_steps',1) if batch != 48: raise ValueError(f'{size} expected historical global batch 48, got {batch}') sources[size] = rows, batch, config, str(run/'metrics.jsonl') except Exception as exc: result['errors'].append(f'{size}: {exc}') result['paired_end']=paired_end for mode in ('steps', 'samples','paired'): table = [] selected={n:v for n,v in sources.items() if mode!='paired' or n in ('Baseline','HighLR')} for name, (rows, batch, config, path) in selected.items(): target = paired_end if mode=='paired' else end if mode == 'steps' else end*384//batch width = 100 if mode == 'steps' else 100*384//batch stats = window(rows,target,width) complete=bool(target and rows and rows[-1]['step']>=target and stats['count']==stats['expected']) if not complete: stats['flow']=None;stats['tasks']={t:None for t in ('text','audio','video')} reference=window(base,paired_end,100)['flow'] if mode=='paired' else window(active_rows,end,100)['flow'] delta = ((stats['flow']/reference-1)*100 if stats['flow'] is not None and reference else None) table.append(dict(name=name, batch=batch, samples=target*batch, delta_pct=delta, **stats)) result['modes'][mode] = table result['curves'][mode] = {name:curve(rows,batch,paired_end if mode=='paired' else end if mode=='steps' else end*384//batch) for name,(rows,batch,config,path) in selected.items()} result['sources'] = {name:dict(metrics=path,batch=batch, recorded_action_lr=rows[-1]['metrics'].get('train/lr_action_header') if rows else None, aux_weight=config['train'].get('absolute_height_loss_weight'), cfg_dropout=config['train'].get('cfg_condition_dropout_prob',0), hidden=config['model'].get('hidden_dim'),depth=config['model'].get('num_blocks')) for name,(rows,batch,config,path) in sources.items()} backup=highlocal/'backup_status.json' if 'HighLR' in sources else LOCAL/'backup_live_status.json' if not backup.exists():backup=LOCAL/'backup_status.json' result['backup']=json.loads(backup.read_text()) if backup.exists() else None preview=highlocal/'previews/web/status.json' result['preview']=json.loads(preview.read_text()) if preview.exists() else dict(status='waiting_checkpoint',models=[]) self.payload=result def poll(self): while True: try: self.collect() except Exception as exc: self.payload={**self.payload,'error':str(exc),'failed_at':time.time()} time.sleep(10) def handler_with_baseline(parent, comparison): import server from types import SimpleNamespace gallery=server.PreviewGallery('reference-highlr','HighLR',LOCAL.with_name('baseline_reference_highlr_20261008')/'previews',True) media_handler=server.handler_for(SimpleNamespace(previews={'reference-highlr':gallery})) study_gallery=server.PreviewGallery('condition-study','条件稳定性',Path('/mnt/workspace/Movens-Zero_train/scale/condition_stability_20261009/previews'),True) study_media=server.handler_for(SimpleNamespace(previews={'condition-study':study_gallery})) class Handler(parent): def do_GET(self): route=urlparse(self.path).path.rstrip('/') if '/media/condition-study/' in route: return study_media.serve_media(self,'condition-study',route.split('/media/condition-study/',1)[1]) if route.endswith('/condition-study') or route.endswith('/api/condition-study'): base=ROOT/'scale/experiments/condition_stability_20261009' is_api=route.endswith('/api/condition-study') file=base/('status.json' if is_api else 'dashboard.html') if not file.exists():return self.send_error(503,'Experiment snapshot preparing') data=file.read_bytes() self.send_response(200) self.send_header('Content-Type','application/json; charset=utf-8' if is_api else 'text/html; charset=utf-8') self.send_header('Content-Length',str(len(data)));self.send_header('Cache-Control','no-store') self.end_headers();self.wfile.write(data);return if '/media/reference-highlr/' in route: return media_handler.serve_media(self,'reference-highlr',route.split('/media/reference-highlr/',1)[1]) if route.endswith('/api/baseline'): data=json.dumps(comparison.payload,allow_nan=False).encode() content_type='application/json; charset=utf-8' elif route.endswith('/baseline'): data=(ROOT/'scale/dashboard/baseline.html').read_bytes() content_type='text/html; charset=utf-8' else: return super().do_GET() self.send_response(200) self.send_header('Content-Type',content_type) self.send_header('Cache-Control','no-store') self.send_header('Content-Length',str(len(data))) self.end_headers() self.wfile.write(data) return Handler