code-Movens-Zero_train / scale /script /baseline_dashboard.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
13cc860 verified
Raw History Blame Contribute Delete
12.4 kB
"""Read-only fresh baseline / historical scale comparison on the existing server."""
import json
import math
from pathlib import Path
import threading
import time
from urllib.parse import urlparse
ROOT = Path(__file__).resolve().parents[2]
LOCAL = Path('/mnt/workspace/Movens-Zero_train/scale/baseline_reference_20261008')
GROUP = ROOT / 'scale/groups/sml_1m_224_a1e-4_o1e-5_nojerk'
FLOW = 'train/loss_group/flow'
def finite(v):
return isinstance(v, (float, int)) and math.isfinite(v)
class Prefix:
"""Incrementally read the needed early history, never scan GB logs per refresh."""
def __init__(self, path):
self.path = path
self.offset = 0
self.rows = {}
def read(self, stop):
if self.path.stat().st_size < self.offset:
self.offset = 0
self.rows = {}
with self.path.open('rb') as stream:
stream.seek(self.offset)
while True:
position = stream.tell()
line = stream.readline()
if not line.endswith(b'\n'):
break
row = json.loads(line)
step = int(row['step'])
if step > stop:
break
self.rows[step] = row
self.offset = stream.tell()
return [r for s, r in sorted(self.rows.items()) if s <= stop]
def average(rows, key):
vals = [r['metrics'].get(key) for r in rows]
vals = [v for v in vals if finite(v)]
return sum(vals) / len(vals) if vals else None
def window(rows, end, width):
selected = [r for r in rows if end-width < r['step'] <= end]
return dict(start=max(1, end-width+1), end=end, count=len(selected),
expected=min(end, width), flow=average(selected, FLOW),
tasks={task: average(selected, f'train/loss_task/{task}')
for task in ('text', 'audio', 'video')},
task_counts={task: sum(finite(r['metrics'].get(f'train/loss_task/{task}'))
for r in selected) for task in ('text', 'audio', 'video')})
def curve(rows, batch, end, key=FLOW):
width = max(1, math.ceil(end / 150))
bins = {}
for row in rows:
if row['step'] > end:
continue
value = row['metrics'].get(key)
if finite(value):
bins.setdefault((row['step']-1)//width, []).append((row['step'], value))
return [[sum(x for x, y in points)/len(points),
batch*sum(x for x, y in points)/len(points),
sum(y for x, y in points)/len(points)] for points in bins.values()]
class Comparison:
def __init__(self):
self.payload = {'loading': True}
self.logs = {}
def log(self, path, stop):
return self.logs.setdefault(str(path), Prefix(path)).read(stop)
def collect(self):
cfg = json.loads((LOCAL/'fresh_config.json').read_text())
candidates = list((LOCAL/'runs/posttrain_v1_vlm').glob('*/metrics.jsonl'))
if len(candidates) != 1:
raise RuntimeError(f'Expected exactly one baseline run, found {len(candidates)}')
base = self.log(candidates[0], cfg['train']['max_training_steps'])
if not base:
raise RuntimeError('Baseline has no metrics yet')
end = base[-1]['step']
base_win = window(base, end, 100)
now = time.time()
age = now-base[-1]['time']
result = dict(loading=False, generated_at=now, baseline=dict(
step=end, target=cfg['train']['max_training_steps'], age=age,
status='日志持续更新' if age < 180 else '日志已停更,请检查训练进程',
elapsed_s=base[-1]['time']-base[0]['time'], samples=end*384,
loss=base_win['flow'], initial_loss=window(base,min(100,end),100)['flow'],
seconds=average(base[-100:], 'train/seconds_per_step'),
lr_action=base[-1]['metrics'].get('train/lr_action_header'),
lr_omni=base[-1]['metrics'].get('train/lr_vlm_model'),
metrics_path=str(candidates[0]),
checkpoints=sorted(int(p.parent.name[5:]) for p in candidates[0].parent.glob('checkpoints/ckpt_*/checkpoint_complete.json'))),
modes={}, curves={}, errors=[])
if (LOCAL/'user_stopped.json').exists():result['baseline']['status']='已按用户要求停止(保留历史)'
elif (LOCAL/'user_stop_request.json').exists():result['baseline']['status']='等待2000步保存完成后停止'
sources = {'Baseline': (base, 384, cfg, str(candidates[0]))}
highlocal=LOCAL.with_name('baseline_reference_highlr_20261008')
highpaths=list((highlocal/'runs/posttrain_v1_vlm').glob('*/metrics.jsonl'))
result['highlr']=dict(step=0,status='启动中,等待第一步',gpu=1)
paired_end=0
if len(highpaths)==1:
highrows=self.log(highpaths[0],cfg['train']['max_training_steps'])
if highrows:
highcfg=json.loads((highlocal/'fresh_config.json').read_text())
sources['HighLR']=(highrows,384,highcfg,str(highpaths[0]))
highage=now-highrows[-1]['time']
result['highlr']=dict(step=highrows[-1]['step'],loss=average(highrows[-100:],FLOW),
age=highage,status='日志持续更新' if highage<180 else '日志停更',gpu=1,
seconds=average(highrows[-100:],'train/seconds_per_step'),
elapsed_s=highrows[-1]['time']-highrows[0]['time'],samples=highrows[-1]['step']*384,
initial_loss=window(highrows,min(100,highrows[-1]['step']),100)['flow'],
lr_action=highrows[-1]['metrics'].get('train/lr_action_header'),
lr_omni=highrows[-1]['metrics'].get('train/lr_vlm_model'),
checkpoints=sorted(int(p.parent.name[5:]) for p in highpaths[0].parent.glob('checkpoints/ckpt_*/checkpoint_complete.json')))
paired_end=min(end,highrows[-1]['step'])
# The active experiment, not the stopped control, sets the dashboard horizon.
end=highrows[-1]['step']
result['active_name']='HighLR' if 'HighLR' in sources else 'Baseline'
if (highlocal/'user_stopped.json').exists():
result['highlr']['status']='已停止,为四卡条件稳定性实验腾卡;完整ckpt_2000保留'
result['active']=result['highlr'] if 'HighLR' in sources else result['baseline']
active_rows=sources[result['active_name']][0]
for size in ('S','M','L'):
try:
run = Path(json.loads((GROUP/f'runs/latest_{size}.json').read_text())['run_dir'])
rows = self.log(run/'metrics.jsonl', end*8)
config = json.loads((run/'run_config.json').read_text())
batch = config['train']['train_batch_size']*config.get('world_size_expected',1)*config['train'].get('gradient_accumulation_steps',1)
if batch != 48:
raise ValueError(f'{size} expected historical global batch 48, got {batch}')
sources[size] = rows, batch, config, str(run/'metrics.jsonl')
except Exception as exc:
result['errors'].append(f'{size}: {exc}')
result['paired_end']=paired_end
for mode in ('steps', 'samples','paired'):
table = []
selected={n:v for n,v in sources.items() if mode!='paired' or n in ('Baseline','HighLR')}
for name, (rows, batch, config, path) in selected.items():
target = paired_end if mode=='paired' else end if mode == 'steps' else end*384//batch
width = 100 if mode == 'steps' else 100*384//batch
stats = window(rows,target,width)
complete=bool(target and rows and rows[-1]['step']>=target and stats['count']==stats['expected'])
if not complete:
stats['flow']=None;stats['tasks']={t:None for t in ('text','audio','video')}
reference=window(base,paired_end,100)['flow'] if mode=='paired' else window(active_rows,end,100)['flow']
delta = ((stats['flow']/reference-1)*100
if stats['flow'] is not None and reference else None)
table.append(dict(name=name, batch=batch, samples=target*batch,
delta_pct=delta, **stats))
result['modes'][mode] = table
result['curves'][mode] = {name:curve(rows,batch,paired_end if mode=='paired' else end if mode=='steps' else end*384//batch)
for name,(rows,batch,config,path) in selected.items()}
result['sources'] = {name:dict(metrics=path,batch=batch,
recorded_action_lr=rows[-1]['metrics'].get('train/lr_action_header') if rows else None,
aux_weight=config['train'].get('absolute_height_loss_weight'),
cfg_dropout=config['train'].get('cfg_condition_dropout_prob',0),
hidden=config['model'].get('hidden_dim'),depth=config['model'].get('num_blocks'))
for name,(rows,batch,config,path) in sources.items()}
backup=highlocal/'backup_status.json' if 'HighLR' in sources else LOCAL/'backup_live_status.json'
if not backup.exists():backup=LOCAL/'backup_status.json'
result['backup']=json.loads(backup.read_text()) if backup.exists() else None
preview=highlocal/'previews/web/status.json'
result['preview']=json.loads(preview.read_text()) if preview.exists() else dict(status='waiting_checkpoint',models=[])
self.payload=result
def poll(self):
while True:
try:
self.collect()
except Exception as exc:
self.payload={**self.payload,'error':str(exc),'failed_at':time.time()}
time.sleep(10)
def handler_with_baseline(parent, comparison):
import server
from types import SimpleNamespace
gallery=server.PreviewGallery('reference-highlr','HighLR',LOCAL.with_name('baseline_reference_highlr_20261008')/'previews',True)
media_handler=server.handler_for(SimpleNamespace(previews={'reference-highlr':gallery}))
study_gallery=server.PreviewGallery('condition-study','条件稳定性',Path('/mnt/workspace/Movens-Zero_train/scale/condition_stability_20261009/previews'),True)
study_media=server.handler_for(SimpleNamespace(previews={'condition-study':study_gallery}))
class Handler(parent):
def do_GET(self):
route=urlparse(self.path).path.rstrip('/')
if '/media/condition-study/' in route:
return study_media.serve_media(self,'condition-study',route.split('/media/condition-study/',1)[1])
if route.endswith('/condition-study') or route.endswith('/api/condition-study'):
base=ROOT/'scale/experiments/condition_stability_20261009'
is_api=route.endswith('/api/condition-study')
file=base/('status.json' if is_api else 'dashboard.html')
if not file.exists():return self.send_error(503,'Experiment snapshot preparing')
data=file.read_bytes()
self.send_response(200)
self.send_header('Content-Type','application/json; charset=utf-8' if is_api else 'text/html; charset=utf-8')
self.send_header('Content-Length',str(len(data)));self.send_header('Cache-Control','no-store')
self.end_headers();self.wfile.write(data);return
if '/media/reference-highlr/' in route:
return media_handler.serve_media(self,'reference-highlr',route.split('/media/reference-highlr/',1)[1])
if route.endswith('/api/baseline'):
data=json.dumps(comparison.payload,allow_nan=False).encode()
content_type='application/json; charset=utf-8'
elif route.endswith('/baseline'):
data=(ROOT/'scale/dashboard/baseline.html').read_bytes()
content_type='text/html; charset=utf-8'
else:
return super().do_GET()
self.send_response(200)
self.send_header('Content-Type',content_type)
self.send_header('Cache-Control','no-store')
self.send_header('Content-Length',str(len(data)))
self.end_headers()
self.wfile.write(data)
return Handler