Download scripts/training_dashboard.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 44.4 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/training_dashboard.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/training_dashboard.py
-
curl -L -o training_dashboard.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/training_dashboard.py
44.4 kB
| #!/usr/bin/env python3 | |
| """Read-only, run-scoped curves and rendered videos; no external JS/CDN.""" | |
| import argparse | |
| from functools import lru_cache | |
| from collections import Counter | |
| from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer | |
| import json | |
| import math | |
| import mimetypes | |
| from pathlib import Path | |
| import re | |
| import subprocess | |
| import threading | |
| import time | |
| import unicodedata | |
| from urllib.parse import parse_qs, unquote, urlsplit | |
| WANDB_ROWS = [] | |
| WANDB_ERROR = "" | |
| CURVE_KEYS = ("train/loss", "train/lr_act", "train/loss_action", | |
| "train/control_accuracy", "train/swa_qa_token_accuracy", | |
| "train/swa_qa_loss", | |
| "eval/loss", "eval/control_accuracy", | |
| "train/token_control_macro_f1", "train/token_false_transition_rate", | |
| "train/token_flicker_rate", "eval/token_control_macro_f1") | |
| def answer_tokens(text): | |
| # A transparent lexical proxy, not a semantic or speech-quality judge. | |
| text = unicodedata.normalize("NFKC", str(text)).lower() | |
| return re.findall(r"[\u4e00-\u9fff]|[^\W_]+", re.sub(r"([\u4e00-\u9fff])", r" \1 ", text)) | |
| def qa_case(metric, directory): | |
| """Score actual answer triggers against labeled reply windows, never GT audio. | |
| One matched trigger per contiguous positive interval; further triggers are | |
| extras. Content uses the FIRST on-time answer, not best-of-many. Multiple | |
| questions without per-event references get timing scores only. | |
| """ | |
| if not metric.get("events") or "overlay" not in metric: | |
| return None | |
| events = sorted(metric["events"], key=lambda event: event["seconds"]) | |
| if any("reply_gt" not in event for event in events): | |
| return None | |
| intervals = [] | |
| previous = False | |
| for event in events: | |
| positive = bool(event["reply_gt"]) | |
| if positive: | |
| if not previous: | |
| intervals.append([event["seconds"], event["seconds"]]) | |
| else: | |
| intervals[-1][1] = event["seconds"] | |
| previous = positive | |
| matched, extras, audio = {}, 0, 0 | |
| answers = sorted(metric.get("answers", []), key=lambda answer: answer.get("seconds", float("inf"))) | |
| for answer in answers: | |
| seconds = answer.get("seconds") | |
| index = next((i for i, (start, end) in enumerate(intervals) | |
| if isinstance(seconds, (int, float)) and start-1e-6 <= seconds <= end+1e-6), None) | |
| if index is not None and index not in matched: | |
| matched[index] = answer | |
| else: | |
| extras += 1 | |
| path = (directory / answer.get("audio_path", "")).resolve() | |
| if path.is_relative_to(directory.resolve()) and path.is_file() and path.stat().st_size > 44: | |
| audio += 1 | |
| overlay = metric["overlay"] | |
| result = dict(question=overlay.get("question", ""), reference=overlay.get("answer", ""), | |
| speech_failures=len(metric.get('speech_failures',[])), | |
| predictions=[dict(seconds=a.get("seconds"), text=a.get("text", "")) for a in answers], | |
| questions=len(intervals), on_time=len(matched), extra_answers=extras, | |
| answers=len(answers), audio_files=audio, source_id=metric.get("source_id", ""), | |
| asset_id=overlay.get("asset_id", "")) | |
| # Missing/late answer is a failure, not omitted from content denominator. | |
| target = answer_tokens(result["reference"]) | |
| if len(intervals) == 1 and target: | |
| prediction = answer_tokens(matched.get(0, {}).get("text", "")) | |
| overlap = sum((Counter(prediction) & Counter(target)).values()) | |
| result.update(text_em=float(prediction == target), | |
| text_f1=2*overlap/(len(prediction)+len(target))) | |
| return result | |
| def qa_summary(cases): | |
| grouped = {} | |
| for case in cases: | |
| grouped.setdefault((case["step"], case["split"]), []).append(case) | |
| rows = [] | |
| for (step, split), items in sorted(grouped.items()): | |
| total = sum(item["questions"] for item in items) | |
| row = dict(step=step, case=split, sessions=len(items), questions=total, | |
| extra_answers_per_session=sum(item["extra_answers"] for item in items)/len(items)) | |
| if total: | |
| row["reply_recall"] = sum(item["on_time"] for item in items)/total | |
| for key in ("text_em", "text_f1"): | |
| values = [item[key] for item in items if key in item] | |
| if values: | |
| row[key] = sum(values)/len(values) | |
| row["text_samples"] = len(values) | |
| rows.append(row) | |
| return rows | |
| class MetricHistory: | |
| """Read the complete history once, then only appended complete JSONL lines.""" | |
| def __init__(self, path): | |
| self.path, self.identity, self.offset, self.rows = path, None, 0, [] | |
| def read(self): | |
| try: | |
| with self.path.open("rb") as stream: | |
| import os | |
| stat = os.fstat(stream.fileno()) | |
| identity = (stat.st_dev, stat.st_ino) | |
| if identity != self.identity or stat.st_size < self.offset: | |
| self.rows, self.offset = [], 0 | |
| self.identity = identity | |
| stream.seek(self.offset) | |
| while True: | |
| line = stream.readline() | |
| if not line.endswith(b"\n"): | |
| break # Retry incomplete final line next refresh. | |
| self.offset = stream.tell() | |
| try: | |
| row = json.loads(line) | |
| if not isinstance(row, dict) or not isinstance(row.get("step"), (int, float)): | |
| continue | |
| if not math.isfinite(row["step"]): | |
| continue | |
| self.rows.append({key: value for key, value in row.items() | |
| if key in (*CURVE_KEYS, "step", "time", "train/swa_qa_target_tokens") | |
| and isinstance(value, (int, float)) and math.isfinite(value)}) | |
| if self.rows[-1].get("train/swa_qa_target_tokens", 0) <= 0: | |
| for key in ("train/swa_qa_loss", "train/swa_qa_token_accuracy"): | |
| self.rows[-1].pop(key, None) | |
| except (ValueError, UnicodeDecodeError): | |
| continue | |
| except OSError: | |
| pass | |
| return self.rows | |
| def compact_curves(rows, bins=400): | |
| """Bound display size per metric, retaining endpoints and each bin's extrema. | |
| Sample metrics independently so sparse eval/control points are not discarded | |
| by the much denser loss stream. Does not alter the training log on disk. | |
| """ | |
| selected = set() | |
| for key in CURVE_KEYS: | |
| indices = [i for i, row in enumerate(rows) if key in row] | |
| if len(indices) <= bins * 2: | |
| selected.update(indices) | |
| continue | |
| selected.update((indices[0], indices[-1])) | |
| for start in range(bins): | |
| bucket = indices[len(indices)*start//bins:len(indices)*(start+1)//bins] | |
| selected.add(min(bucket, key=lambda i: rows[i][key])) | |
| selected.add(max(bucket, key=lambda i: rows[i][key])) | |
| if rows: | |
| selected.update((0, len(rows)-1)) | |
| return [rows[i] for i in sorted(selected)] | |
| def curve_view(data, mode="global", window=50000): | |
| latest = max((r["step"] for name in ("rows", "semantic_rows", "validation_rows") | |
| for r in data[name]), default=0) | |
| first = min((r["step"] for r in data["rows"]), default=0) | |
| start = max(0, latest-window) if mode == "local" else first if mode == "current" else 0 | |
| result = dict(data, step_range=[start, max(start+1, latest)], view=mode, | |
| first_step=first, latest=data["rows"][-1] if data["rows"] else {}) | |
| for name in ("rows", "semantic_rows", "validation_rows"): | |
| rows = sorted((r for r in data[name] if start <= r["step"] <= latest), key=lambda r: r["step"]) | |
| result[name] = rows if name == "validation_rows" else compact_curves(rows) | |
| result["qa_rows"] = [r for r in data.get("qa_rows", []) if start <= r["step"] <= latest] | |
| cases = [r for r in data.get("qa_cases", []) if start <= r["step"] <= latest] | |
| result["qa_cases"] = [r for r in cases if r["step"] == max( | |
| x["step"] for x in cases if x["split"] == r["split"])] | |
| return result | |
| def poll_wandb(path): | |
| """Read sparse semantic metrics off-thread; never delays the training job.""" | |
| global WANDB_ROWS, WANDB_ERROR | |
| import wandb | |
| while True: | |
| try: | |
| run = wandb.Api(timeout=15).run(path) | |
| collected = {} | |
| for keys in (["train/control_accuracy"], | |
| ["train/swa_qa_loss", "train/swa_qa_token_accuracy", "train/swa_qa_target_tokens"]): | |
| for row in run.history(samples=1000, keys=keys, pandas=False): | |
| if "_step" not in row: | |
| continue | |
| if "train/swa_qa_target_tokens" in row and row["train/swa_qa_target_tokens"] <= 0: | |
| continue | |
| step = int(row.pop("_step")) | |
| collected.setdefault(step, {"step": step}).update(row) | |
| WANDB_ROWS = list(collected.values()) | |
| WANDB_ERROR = "" | |
| except Exception as error: | |
| WANDB_ERROR = type(error).__name__ | |
| time.sleep(60) | |
| PAGE = r"""<!doctype html><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>HFD 训练与推理</title> | |
| <style> | |
| :root{--ink:#192c43;--muted:#66778c;--blue:#2563eb;--line:#e3eaf2;--bg:#f4f7fb} | |
| *{box-sizing:border-box}body{max-width:1540px;margin:auto;padding:32px 40px;background:var(--bg);color:var(--ink);font:15px/1.6 system-ui,sans-serif} | |
| h1,h2,p{margin:0}h1{font-size:30px;letter-spacing:-1px}h2{font-size:17px;font-weight:650}a{color:var(--blue)} | |
| .eyebrow{font-size:11px;font-weight:700;letter-spacing:2px;color:var(--blue);margin-bottom:8px}.hero{display:flex;justify-content:space-between;align-items:center;gap:24px;margin-bottom:22px} | |
| #health{margin-top:8px;font-size:13px;color:var(--muted)}#name{display:block;font:12px/1.6 ui-monospace,monospace;color:var(--muted);overflow-wrap:anywhere} | |
| .kpis{display:grid;grid-template-columns:2fr 1fr 1fr 1fr;gap:16px;margin:22px 0}.kpi{background:white;border:1px solid var(--line);border-radius:14px;padding:20px 24px} | |
| .kpi label{display:block;font-size:12px;color:var(--muted);font-weight:600}.kpi strong{font-size:29px;letter-spacing:-1px;font-variant-numeric:tabular-nums;display:block;margin:6px 0}.kpi small{font-size:12px;color:var(--muted)} | |
| progress{appearance:none;display:block;width:100%;height:5px;border:0;border-radius:4px;overflow:hidden;margin:10px 0;background:#e8eef8}progress::-webkit-progress-bar{background:#e8eef8}progress::-webkit-progress-value{background:var(--blue)}progress::-moz-progress-bar{background:var(--blue)} | |
| nav{display:flex;gap:6px;border-bottom:1px solid #d8e2ee;margin:26px 0 20px;position:sticky;top:0;z-index:4;background:var(--bg);padding-top:8px} | |
| button,select{font:inherit;border:1px solid #d4deeb;border-radius:8px;padding:8px 12px;background:white;color:var(--ink)}button{cursor:pointer}button:hover{border-color:#7fa5eb}button:disabled{cursor:default;opacity:.5}button[aria-pressed="true"]{background:#eaf1ff;color:#1d4ed8;border-color:#aac5ff} | |
| nav button{border:0;border-bottom:3px solid transparent;border-radius:0;background:transparent;padding:12px 20px;color:var(--muted);font-weight:600}nav button[aria-pressed="true"]{color:var(--blue);background:transparent;border-color:var(--blue)} | |
| button:focus-visible,select:focus-visible,summary:focus-visible{outline:3px solid #9dc2ff;outline-offset:2px}.panel[hidden]{display:none!important} | |
| .grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:20px;margin:18px 0 24px}article,section{background:white;border:1px solid var(--line);border-radius:14px;padding:22px 24px;min-width:0}article{position:relative} | |
| canvas{display:block;width:100%;height:270px;touch-action:pan-y}.chart-stat{font-size:25px;font-weight:650;letter-spacing:-.6px;font-variant-numeric:tabular-nums;margin:8px 0;color:#244a80}.chart-stat small{font-size:12px;font-weight:400;letter-spacing:0;color:var(--muted);margin-left:10px} | |
| .tooltip{display:none;position:absolute;z-index:3;pointer-events:none;background:#14283e;color:white;padding:9px 12px;border-radius:8px;font-size:13px;white-space:pre-line;max-width:90%} | |
| .toolbar{display:flex;gap:12px;flex-wrap:wrap;align-items:center}.toolbar label{display:grid;gap:4px;font-size:12px;color:var(--muted)}.toolbar select{max-width:310px}#range{font-size:12px;color:var(--muted);margin-left:auto}#curve-tools{padding:14px 18px} | |
| small{color:var(--muted);font-size:13px;line-height:1.7}details{color:var(--muted);font-size:13px;margin-top:14px}summary{cursor:pointer;font-weight:550}details small{display:block;margin-top:10px}pre{white-space:pre-wrap;overflow-wrap:anywhere;font:inherit}#gpu{font:12px/1.8 ui-monospace,monospace;margin-top:12px}#stats{font-size:12px;color:var(--muted);margin-top:10px} | |
| .badge{display:inline-block;background:#edf3ff;color:#345685;border-radius:5px;padding:3px 8px;margin:0 6px 6px 0;font-size:12px}.video-card{padding:0;overflow:hidden}.video-card header,.video-card .description{padding:18px 22px}.video-card h2{margin-bottom:10px}video{display:block;width:100%;aspect-ratio:16/9;background:#102035}.video-card p{margin:8px 0;line-height:1.7}.video-card details{border-top:1px solid var(--line);padding-top:12px}.empty{padding:32px;color:var(--muted);text-align:center} | |
| #video-status,#qa-status{font-size:13px;color:var(--muted);margin:10px 0}.panel>h2{margin-top:24px}.legend{line-height:1.8} | |
| @media(max-width:900px){body{padding:22px}.kpis{grid-template-columns:1fr 1fr}.grid{gap:14px}article,section{padding:18px}} | |
| @media(max-width:650px){body{padding:16px}h1{font-size:25px}.hero{display:block}.kpis{gap:10px}.kpi{padding:15px}.kpi strong{font-size:23px}nav{gap:0}nav button{padding:12px 10px;font-size:13px}.grid{grid-template-columns:1fr}#range{margin:0;width:100%}.toolbar label{width:100%}.toolbar select{max-width:none}.toolbar{gap:8px}canvas{height:260px}} | |
| </style> | |
| <header class="hero"><div><div class="eyebrow">HFD / EXPERIMENT MONITOR</div><h1>训练与推理</h1><b id="name"></b><div id="health">读取最新指标…</div></div><button id="manual-refresh">刷新数据 ↻</button></header> | |
| <div class="kpis"> | |
| <div class="kpi"><label>训练进度 · GLOBAL STEP</label><strong id="k-step">—</strong><progress id="progress" max="100" value="0"></progress><small id="k-budget">等待指标</small></div> | |
| <div class="kpi"><label>训练 Loss · 最新记录</label><strong id="k-loss">—</strong><small id="k-loss-step">越低越好</small></div> | |
| <div class="kpi"><label>控制准确率 · 最新记录</label><strong id="k-control">—</strong><small id="k-control-step">越高越好</small></div> | |
| <div class="kpi"><label>已渲染视频</label><strong id="k-videos">—</strong><small>在「推理视频」中按任务筛选</small></div> | |
| </div> | |
| <nav aria-label="指标分组"><button data-panel="training-panel" aria-pressed="true">训练概览</button><button data-panel="validation-panel" aria-pressed="false">验证与闭环</button><button data-panel="qa-panel" aria-pressed="false">视觉问答</button><button data-panel="gallery-panel" aria-pressed="false">推理视频</button></nav> | |
| <section id="curve-tools"><div class="toolbar"><button id="current" aria-pressed="true">本次续训</button><button id="global" aria-pressed="false">全程 · 从 0 开始</button><button id="local" aria-pressed="false">最近区间</button> | |
| <select id="window" aria-label="曲线时间范围"><option value="1000">最近 1 千步</option><option value="5000">最近 5 千步</option><option value="10000" selected>最近 1 万步</option><option value="50000">最近 5 万步</option><option value="100000">最近 10 万步</option></select><span id="range"></span></div> | |
| <details><summary>显示口径与运行配置</summary><p id="stats"></p><pre id="gpu"></pre><small>每15秒刷新。浅线为保留极值的展示点,深线是20个展示点均值,不是全量统计均值。悬停查看原始展示点。缺失指标不填0;「本次续训」从本地首条记录开始,不把之前没有的曲线补出来。</small></details></section> | |
| <div id="training-panel" class="panel"> | |
| <div id="training" class="grid"><article><h2>训练 Loss</h2><canvas id="loss" width="850" height="220"></canvas></article> | |
| <article><h2>控制准确率</h2><canvas id="accuracy" width="850" height="220"></canvas></article> | |
| <article><h2>动作 Loss</h2><canvas id="action" width="850" height="220"></canvas></article> | |
| <article><h2>学习率</h2><canvas id="lr" width="850" height="220"></canvas></article></div> | |
| <details id="control-details"><summary>更多控制诊断 · 状态 F1 / 误切换 / 闪烁</summary><div class="grid"><article><h2>状态分类宏平均 F1 ↑</h2><canvas id="tokenf1"></canvas></article> | |
| <article><h2>状态误切换率 ↓</h2><canvas id="tokenfalse"></canvas></article> | |
| <article><h2>状态往返闪烁率 ↓</h2><canvas id="tokenflicker"></canvas></article></div></details> | |
| </div><div id="qa-panel" class="panel" hidden> | |
| <h2 id="visual-qa">视觉问答 → 语音回答</h2> | |
| <section><p id="qa-status"></p><details><summary>这些指标衡量什么?</summary><small>训练答词准确率:给定正确前文后预测下一个 token,不等于整句回答正确率。无问答监督的 step 不画成0。推理图:train / val 分开,每个点来自少量固定样例,非全量测试集。按标注允许回答的窗口匹配,每个问题只取首次按时触发的回答;迟答、抢答和重复回答计入额外回答。时机指触发生成的时刻,不是声音实际播放时刻。文字匹配仅忽略大小写、标点和空格,不判语义,也不衡量声音自然度。当前没有独立的视觉问答验证 loss,不补造曲线。</small></details></section> | |
| <div class="grid"> | |
| <article><h2>训练:回答 Loss ↓</h2><canvas id="qaloss"></canvas></article> | |
| <article><h2>训练:回答 token 准确率 ↑</h2><canvas id="qa"></canvas></article> | |
| <article><h2>推理:按时触发回答的比例 ↑</h2><canvas id="replyrecall"></canvas></article> | |
| <article><h2>推理:每条 session 额外触发回答次数 ↓</h2><canvas id="replyextra"></canvas></article> | |
| <article><h2>推理:参考文字完全匹配率 ↑(非语义评分)</h2><canvas id="textem"></canvas></article> | |
| <article><h2>推理:参考词重合 F1 ↑(非语义评分)</h2><canvas id="textf1"></canvas></article></div> | |
| <details><summary>展开区间最新问答明细:问题 / 参考答案 / 实际回答</summary><div id="qa-cases" class="grid"></div></details> | |
| </div><div id="validation-panel" class="panel" hidden> | |
| <h2 id="validation">验证集 · 逐窗口指标与连续推理</h2><div class="grid"> | |
| <article><h2>验证 Loss</h2><canvas id="eval" width="850" height="220"></canvas></article> | |
| <article><h2>验证控制准确率</h2><canvas id="valcontrol"></canvas></article> | |
| <article><h2>闭环控制准确率(每条样例一条线)</h2><canvas id="closedcontrol"></canvas></article> | |
| <article><h2>闭环关节误差(弧度,越低越好)</h2><canvas id="closedjoint"></canvas></article> | |
| <article><h2>闭环关节速度误差(弧度/秒)</h2><canvas id="closedvelocity"></canvas></article></div> | |
| </div><div id="gallery-panel" class="panel" hidden> | |
| <h2 id="gallery">推理视频</h2> | |
| <section><div class="toolbar"> | |
| <label>模型版本<select id="video-checkpoint"></select></label> | |
| <button id="video-latest">查看最新已渲染版本</button> | |
| <label>数据划分<select id="filter"><option value="val" selected>验证集 val</option><option value="train">训练集 train</option><option value="">两者都看</option></select></label> | |
| <label>交互任务<select id="video-task"><option value="">全部任务</option></select></label> | |
| <label>视觉输入<select id="video-kind"><option value="">图片和视频</option><option value="video">视频</option><option value="image">图片</option><option value="none">无视觉</option></select></label> | |
| </div><p id="video-status"></p><small>一次只展示一个 checkpoint,默认最新已渲染版本的验证集。新结果不会自动切走正在看的版本;点击按钮再跳到最新。视频筛选独立于上方曲线的 step 范围。</small></section> | |
| <div id="videos" class="grid"></div></div> | |
| <script> | |
| let lastVideos=''; | |
| let videoSnapshot=null,galleryInitialized=false; | |
| let view=['global','local','current'].includes(localStorage.getItem('hfd-step-view-v2'))?localStorage.getItem('hfd-step-view-v2'):'current',stepRange=[0,1],requestId=0,chartSpecs=new Map(); | |
| const el=id=>document.getElementById(id); | |
| const TASK_NAMES={'01_turn-taking':'轮流交互 · Turn-taking','02_interruption/command':'新指令打断 · Interruption','02_interruption/stop':'急停 · Stop','03_backchannel':'简短反馈 · Backchannel','04_hesitation':'犹豫修正 · Hesitation','05_pause':'语音停顿 · Pause','06_third-party':'旁人说话 · Third-party','07_third-party':'旁人说话 · Third-party'}; | |
| function videoSelection(videos,checkpoint,split,task,kind){return videos.filter(v=>String(v.step)===String(checkpoint)&&(!split||v.split===split)&&(!task||v.task===task)&&(!kind||v.visual_kind===kind)).sort((a,b)=>a.task.localeCompare(b.task)||a.split.localeCompare(b.split)||a.path.localeCompare(b.path))} | |
| function options(id,values){let select=el(id),hash=JSON.stringify(values);if(select.dataset.options===hash)return;let old=select.value;select.replaceChildren();for(let [value,label] of values){let o=document.createElement('option');o.value=value;o.textContent=label;select.append(o)}if(values.some(v=>String(v[0])===old))select.value=old;select.dataset.options=hash} | |
| function renderVideos(s){ | |
| videoSnapshot=s; | |
| let catalog=s.video_catalog||[],latest=catalog.find(c=>c.rendered>0); | |
| options('video-checkpoint',catalog.map(c=>[String(c.step),'ckpt_'+c.step.toLocaleString()+' · 已渲染 '+c.rendered+'/'+c.inferred])); | |
| if(!galleryInitialized&&catalog.length){el('video-checkpoint').value=String((latest||catalog[0]).step);galleryInitialized=true} | |
| options('video-task',[['','全部任务'],...[...new Set(s.videos.map(v=>v.task))].sort().map(t=>[t,TASK_NAMES[t]||t])]); | |
| let cp=el('video-checkpoint').value,selected=catalog.find(c=>String(c.step)===cp),videos=videoSelection(s.videos,cp,el('filter').value,el('video-task').value,el('video-kind').value); | |
| el('video-status').textContent=selected?'ckpt_'+selected.step+':推理完成 '+selected.inferred+' 条,视频已渲染 '+selected.rendered+' 条;当前筛选显示 '+videos.length+' 条。'+(latest&&String(latest.step)!==cp?' 最新已渲染版本:'+latest.step+'。':''):'等待首次推理结果。'; | |
| el('video-latest').disabled=!latest; | |
| let hash=JSON.stringify(videos);if(hash===lastVideos)return;lastVideos=hash;el('videos').replaceChildren(); | |
| for(let v of videos){let box=document.createElement('article');box.className='video-card';let header=document.createElement('header'),title=document.createElement('h2');title.textContent=TASK_NAMES[v.task]||v.task;header.append(title); | |
| for(let label of [v.split==='val'?'验证集':'训练集','STEP '+v.step.toLocaleString(),{video:'视频输入',image:'图片输入',none:'纯音频输入'}[v.visual_kind]||'视觉类型未知']){let badge=document.createElement('span');badge.className='badge';badge.textContent=label;header.append(badge)} | |
| let vid=document.createElement('video');vid.controls=true;vid.preload='none';vid.setAttribute('playsinline','');vid.src='media/'+v.path.split('/').map(encodeURIComponent).join('/'); | |
| let description=document.createElement('div');description.className='description'; | |
| for(let [label,value] of [['视觉问题',v.question],['模型回答',v.answer||'未输出回答']]){if(!value)continue;let p=document.createElement('p'),b=document.createElement('b');b.textContent=label+':';p.append(b,document.createTextNode(value));description.append(p)} | |
| let details=document.createElement('details'),summary=document.createElement('summary'),note=document.createElement('p'),link=document.createElement('a');summary.textContent='参考答案、指标与原视频';note.textContent='参考答案:'+(v.reference||'无')+';'+v.summary;link.textContent='新窗口打开完整视频';link.href=vid.src;link.target='_blank';link.rel='noopener';details.append(summary,note,link);description.append(details);box.append(header,vid,description);el('videos').append(box)} | |
| if(!videos.length){let p=document.createElement('p');p.className='empty';p.textContent='此筛选下暂无已渲染视频。可切换版本/任务,或等待渲染队列完成。';el('videos').append(p)} | |
| } | |
| function number(v){return !Number.isFinite(v)?'—':Math.abs(v)>0&&Math.abs(v)<.001?v.toExponential(2):v.toLocaleString(undefined,{maximumFractionDigits:4})} | |
| function chart(id,rows,key,group=false){ | |
| chartSpecs.set(id,[id,rows,key,group]); | |
| const c=el(id);if(!c.getClientRects().length)return; | |
| const g=c.getContext('2d'),w=Math.max(280,c.clientWidth),h=270,dpr=window.devicePixelRatio||1; | |
| c.width=Math.round(w*dpr);c.height=Math.round(h*dpr);g.setTransform(dpr,0,0,dpr,0,0);g.font='13px system-ui';g.fillStyle='#66778c'; | |
| const p=rows.filter(r=>Number.isFinite(r[key])&&(key!=='train/swa_qa_token_accuracy'||r['train/swa_qa_target_tokens']>0)).sort((a,b)=>a.step-b.step); | |
| let stat=c.parentElement.querySelector('.chart-stat');if(!stat){stat=document.createElement('div');stat.className='chart-stat';c.before(stat)} | |
| let tip=c.parentElement.querySelector('.tooltip');if(!tip){tip=document.createElement('div');tip.className='tooltip';c.after(tip)}tip.style.display='none'; | |
| const legend=c.parentElement.querySelector('.legend');if(legend)legend.replaceChildren(); | |
| c.onpointermove=null;c.onpointerleave=()=>{tip.style.display='none'}; | |
| if(!p.length){stat.textContent='—';g.textAlign='center';g.fillText('当前区间尚无记录',w/2,110);g.fillText('等待评估或切换时间范围',w/2,135);return} | |
| const ratio=key.includes('accuracy')||key.includes('rate')||key.includes('macro_f1')||['reply_recall','text_em','text_f1'].includes(key); | |
| const fmt=v=>ratio?(v*100).toFixed(1)+'%':number(v); | |
| const ys=p.map(r=>r[key]);let lo=Math.min(...ys),hi=Math.max(...ys); | |
| if(ratio){lo=0;hi=1}else{let pad=Math.max((hi-lo)*.1,Math.abs(hi)*.04,1e-9);lo=Math.min(lo,Math.max(0,lo-pad));hi+=pad} | |
| let groups=group?[...new Set(p.map(r=>r.case))].sort():['all']; | |
| stat.textContent=group?groups.length+' 条系列':fmt(ys.at(-1)); | |
| let sub=document.createElement('small');sub.textContent='最近记录 @ '+p.at(-1).step.toLocaleString();stat.append(sub); | |
| const L=68,R=w-16,T=15,B=h-44,x=v=>L+(R-L)*(v-stepRange[0])/Math.max(1,stepRange[1]-stepRange[0]),y=v=>B-(B-T)*(v-lo)/(hi-lo); | |
| for(let i=0;i<=4;i++){let v=lo+(hi-lo)*i/4,yy=y(v);g.strokeStyle='#e8edf4';g.lineWidth=1;g.beginPath();g.moveTo(L,yy);g.lineTo(R,yy);g.stroke();g.fillStyle='#66778c';g.textAlign='right';g.fillText(ratio?(v*100).toFixed(0)+'%':number(v),L-10,yy+4)} | |
| const ticks=w<440?2:3; | |
| for(let i=0;i<=ticks;i++){let v=stepRange[0]+(stepRange[1]-stepRange[0])*i/ticks;g.textAlign=i===0?'left':i===ticks?'right':'center';g.fillText(Math.round(v).toLocaleString(),x(v),B+24)} | |
| g.textAlign='left'; | |
| const colors=['#2563eb','#e07b24','#0e9488','#9254c8','#c34461','#758b23']; | |
| groups.forEach((name,j)=>{let items=group?p.filter(r=>r.case===name):p; | |
| function line(values,color,width){g.strokeStyle=color;g.lineWidth=width;g.beginPath();items.forEach((r,i)=>{i?g.lineTo(x(r.step),y(values[i])):g.moveTo(x(r.step),y(values[i]))});g.stroke();if(items.length===1){g.fillStyle=color;g.beginPath();g.arc(x(items[0].step),y(values[0]),4,0,Math.PI*2);g.fill()}} | |
| line(items.map(r=>r[key]),group?colors[j%colors.length]:'#c5d8f7',group?2.3:1); | |
| if(!group)line(items.map((r,i)=>{let q=items.slice(Math.max(0,i-19),i+1);return q.reduce((a,b)=>a+b[key],0)/q.length}),'#2563eb',2.3); | |
| }); | |
| c.title=key; | |
| if(group){let legend=c.parentElement.querySelector('.legend');if(!legend){legend=document.createElement('div');legend.className='legend';c.after(legend)}legend.replaceChildren();groups.forEach((n,j)=>{let last=p.filter(r=>r.case===n).at(-1),t=document.createElement('div');t.textContent='● '+(TASK_NAMES[n]||n)+' '+fmt(last[key])+' @ '+last.step.toLocaleString()+(last.sessions?' · '+last.sessions+'条':'');t.style.color=colors[j%colors.length];legend.append(t)})} | |
| c.onpointermove=event=>{ | |
| const rect=c.getBoundingClientRect(),localX=event.clientX-rect.left,target=stepRange[0]+(localX-L)/(R-L)*(stepRange[1]-stepRange[0]); | |
| let nearest=p.reduce((a,b)=>Math.abs(b.step-target)<Math.abs(a.step-target)?b:a); | |
| let values=group?p.filter(r=>r.step===nearest.step):[nearest]; | |
| tip.textContent='STEP '+nearest.step.toLocaleString()+'\n'+values.map(r=>(group?(TASK_NAMES[r.case]||r.case)+': ':'原始记录: ')+fmt(r[key])).join('\n'); | |
| tip.style.display='block';tip.style.left=Math.max(8,Math.min(c.offsetLeft+localX+12,c.parentElement.clientWidth-tip.offsetWidth-10))+'px'; | |
| tip.style.top=Math.max(5,c.offsetTop+event.clientY-rect.top-70)+'px'; | |
| }; | |
| } | |
| function redraw(){for(const spec of chartSpecs.values())chart(...spec)} | |
| async function refresh(){ | |
| const id=++requestId; | |
| for(let mode of ['global','local','current'])el(mode).setAttribute('aria-pressed',view===mode); | |
| try{ | |
| let response=await fetch('api/status?view='+view+'&window='+el('window').value,{cache:'no-store'});if(!response.ok)throw Error(response.status); | |
| let s=await response.json();if(id!==requestId)return;let a=s.latest||{};stepRange=s.step_range; | |
| el('range').textContent=stepRange.map(v=>v.toLocaleString()).join(' — ')+' step(所有曲线统一范围)'; | |
| el('name').textContent=s.name; | |
| el('health').textContent=a.time?'最近指标 '+new Date(a.time*1000).toLocaleString()+'('+Math.round(Date.now()/1000-a.time)+'秒前)':'等待首次训练指标;启动数据准备期间暂无曲线'; | |
| el('stats').textContent='Step '+(a.step??'—')+' / '+s.max_steps+' · 每卡 Batch '+s.batch+' · '+(s.architecture||'legacy')+' · 已渲染 '+s.videos.length+' 个视频'; | |
| el('k-step').textContent=Number.isFinite(a.step)?a.step.toLocaleString():'—'; | |
| let pct=Number.isFinite(a.step)&&Number(s.max_steps)>0?100*a.step/Number(s.max_steps):0; | |
| el('progress').value=pct;el('k-budget').textContent=pct.toFixed(1)+'% · 目标 '+Number(s.max_steps).toLocaleString()+' 步'; | |
| for(let [id,key,rows,percent] of [['loss','train/loss',s.rows,false],['control','train/control_accuracy',s.semantic_rows,true]]){ | |
| let r=rows.filter(x=>Number.isFinite(x[key])).at(-1);el('k-'+id).textContent=r?(percent?(r[key]*100).toFixed(1)+'%':number(r[key])):'—'; | |
| el('k-'+id+'-step').textContent=r?'@ '+r.step.toLocaleString()+' step':'当前区间暂无记录'; | |
| } | |
| el('k-videos').textContent=s.videos.length.toLocaleString(); | |
| el('gpu').textContent=s.gpu; | |
| chart('loss',s.rows,'train/loss');chart('accuracy',s.semantic_rows,'train/control_accuracy');chart('lr',s.rows,'train/lr_act');chart('eval',s.rows,'eval/loss');chart('action',s.rows,'train/loss_action');chart('qa',s.semantic_rows,'train/swa_qa_token_accuracy'); | |
| chart('qaloss',s.semantic_rows,'train/swa_qa_loss'); | |
| for(let [id,key] of [['replyrecall','reply_recall'],['replyextra','extra_answers_per_session'],['textem','text_em'],['textf1','text_f1']])chart(id,s.qa_rows,key,true); | |
| let targets=s.semantic_rows.filter(r=>r['train/swa_qa_target_tokens']>0),q=targets.at(-1); | |
| el('qa-status').textContent=q?'区间内最近问答监督记录:step '+q.step+',该步 '+q['train/swa_qa_target_tokens']+' 个目标 token。记录来自本地日志及 WandB 抽样,空白不代表没训练。':'该区间暂未读到有效问答监督记录(不是0准确率)。'; | |
| el('qa-cases').replaceChildren();for(let q of s.qa_cases){let box=document.createElement('article'),title=document.createElement('h2'),text=document.createElement('pre');title.textContent=q.split+' / '+q.case+' / step '+q.step; | |
| text.textContent='问题:'+q.question+';参考:'+q.reference+';实际:'+(q.predictions.map(a=>'['+a.seconds+'秒] '+a.text).join(' | ')||'未回答')+';按时 '+q.on_time+'/'+q.questions+';额外开口 '+q.extra_answers+' 次;已有音频文件 '+q.audio_files+'/'+q.answers+';语音生成失败 '+(q.speech_failures||0)+' 次(失败不计成功,不发布截断音频)';box.append(title,text);el('qa-cases').append(box)} | |
| chart('valcontrol',s.rows,'eval/control_accuracy');chart('closedcontrol',s.validation_rows,'control_accuracy',true);chart('closedjoint',s.validation_rows,'joint_rmse_rad',true);chart('closedvelocity',s.validation_rows,'joint_velocity_error_rad_s',true); | |
| chart('tokenf1',s.rows,'train/token_control_macro_f1');chart('tokenfalse',s.rows,'train/token_false_transition_rate');chart('tokenflicker',s.rows,'train/token_flicker_rate'); | |
| if(s.wandb_error)el('health').textContent+=';WandB稀疏指标读取暂失败:'+s.wandb_error; | |
| renderVideos(s); | |
| }catch(e){if(id===requestId)el('health').textContent='读取失败:'+e.message} | |
| } | |
| function selectView(value){view=value;localStorage.setItem('hfd-step-view-v2',view);refresh()} | |
| el('global').onclick=()=>selectView('global');el('local').onclick=()=>selectView('local');el('current').onclick=()=>selectView('current');el('window').onchange=()=>selectView('local'); | |
| el('manual-refresh').onclick=refresh; | |
| for(let button of document.querySelectorAll('[data-panel]'))button.onclick=()=>{ | |
| for(let item of document.querySelectorAll('[data-panel]'))item.setAttribute('aria-pressed',item===button); | |
| for(let panel of document.querySelectorAll('.panel'))panel.hidden=panel.id!==button.dataset.panel; | |
| el('curve-tools').hidden=button.dataset.panel==='gallery-panel';redraw(); | |
| }; | |
| let resizeTimer;window.addEventListener('resize',()=>{clearTimeout(resizeTimer);resizeTimer=setTimeout(redraw,120)}); | |
| el('control-details').addEventListener('toggle',redraw); | |
| for(let id of ['filter','video-checkpoint','video-task','video-kind'])el(id).onchange=()=>{if(videoSnapshot)renderVideos(videoSnapshot)}; | |
| el('video-latest').onclick=()=>{let latest=videoSnapshot?.video_catalog.find(c=>c.rendered>0);if(latest){el('video-checkpoint').value=String(latest.step);renderVideos(videoSnapshot)}}; | |
| refresh();setInterval(refresh,15000); | |
| </script>""" | |
| def _verified_remote_video(filename, size, mtime_ns, metadata_text): | |
| """Mirrors omit dot-markers; verify a completed renderer receipt and MP4.""" | |
| try: | |
| meta = json.loads(metadata_text) | |
| if meta.get("swa_render_version") != 2 or int(meta.get("total_frames", 0)) <= 0: | |
| return False | |
| probe = json.loads(subprocess.check_output([ | |
| "ffprobe", "-v", "error", "-select_streams", "v:0", | |
| "-show_entries", "stream=nb_frames", "-of", "json", filename, | |
| ], text=True, timeout=5)) | |
| return int(probe["streams"][0]["nb_frames"]) == int(meta["total_frames"]) | |
| except (OSError, subprocess.SubprocessError, ValueError, KeyError, IndexError, TypeError): | |
| return False | |
| def render_ready(path, *, remote=False): | |
| if (path.parent / ".rendered").is_file(): | |
| return True | |
| if not remote: | |
| return False | |
| try: | |
| stat = path.stat() | |
| meta = (path.parent / "infer_meta.json").read_text() | |
| return _verified_remote_video(str(path), stat.st_size, stat.st_mtime_ns, meta) | |
| except OSError: | |
| return False | |
| def snapshot(run, history=None, *, remote=False): | |
| rows = (history or MetricHistory(run / "dashboard_metrics.jsonl")).read() | |
| config = json.loads((run / "run_config.json").read_text()) if (run / "run_config.json").is_file() else {} | |
| videos = [] | |
| validation_rows = [] | |
| qa_cases = [] | |
| video_catalog = {} | |
| for path in (run / "unified_overlay").glob("ckpt_*/*/**/metrics.json"): | |
| try: | |
| metric = json.loads(path.read_text()) | |
| relative = path.relative_to(run / "unified_overlay") | |
| row = dict(step=int(relative.parts[0].split("_")[1]), | |
| case="/".join(relative.parts[2:-1])) | |
| split = relative.parts[1] | |
| if split not in ("train", "val"): | |
| continue | |
| video_catalog.setdefault(row['step'], dict(step=row['step'], inferred=0, rendered=0))['inferred'] += 1 | |
| qa = qa_case(metric, path.parent) | |
| if qa is not None: | |
| qa_cases.append(dict(qa, **row, split=split)) | |
| for key, value in {"control_accuracy": metric.get("control_accuracy"), | |
| **metric.get("body_metrics", {})}.items(): | |
| if isinstance(value, (int, float)) and math.isfinite(value): | |
| row[key] = value | |
| if split == "val": | |
| validation_rows.append(row) | |
| except (OSError, ValueError, TypeError): | |
| continue | |
| validation_rows.sort(key=lambda row: row["step"]) | |
| for path in (run / "unified_overlay").glob("ckpt_*/*/**/reference_motion_with_audio.mp4"): | |
| if not render_ready(path, remote=remote): | |
| continue | |
| summary = "" | |
| m = {} | |
| try: | |
| m = json.loads((path.parent / "metrics.json").read_text()) | |
| summary = "控制准确率 %.3f | 回答:%s" % ( | |
| m.get("control_accuracy", 0), | |
| "; ".join(a.get("text", "") for a in m.get("answers", [])), | |
| ) | |
| except (OSError, ValueError): | |
| pass | |
| relative = path.relative_to(run / 'unified_overlay') | |
| step = int(relative.parts[0].split('_')[1]) | |
| split = relative.parts[1] | |
| if split not in ('train', 'val'): | |
| continue | |
| overlay = m.get('overlay', {}) | |
| kind = 'video' if overlay.get('video_path') else 'image' if overlay.get('image_path') else 'none' | |
| videos.append(dict(path=str(path.relative_to(run)), summary=summary, step=step, | |
| split=split, task='/'.join(relative.parts[2:-1]), visual_kind=kind, | |
| question=overlay.get('question', ''), reference=overlay.get('answer', ''), | |
| answer=' / '.join(a.get('text', '') for a in m.get('answers', [])))) | |
| video_catalog.setdefault(step, dict(step=step, inferred=0, rendered=0))['rendered'] += 1 | |
| videos.sort(key=lambda v: int(re.search(r"ckpt_(\d+)", v["path"])[1]), reverse=True) | |
| if remote: | |
| gpu = "远端 DLC 训练:展示 CPFS 同步曲线与视频;不把本机 GPU 状态误作训练卡状态。" | |
| else: | |
| try: | |
| gpu = subprocess.check_output( | |
| ["nvidia-smi", "--query-gpu=index,utilization.gpu,memory.used,memory.total", "--format=csv"], | |
| text=True, timeout=4, | |
| ) | |
| except (OSError, subprocess.SubprocessError): | |
| gpu = "GPU状态暂不可用" | |
| semantic = {} | |
| for row in rows: | |
| semantic.setdefault(row["step"], {}).update(row) | |
| for row in WANDB_ROWS: | |
| semantic.setdefault(row["step"], {}).update(row) | |
| return dict(name=run.name, rows=rows, videos=videos, gpu=gpu, | |
| architecture=config.get('model',{}).get('interaction_architecture','legacy'), | |
| video_catalog=sorted(video_catalog.values(), key=lambda r: r['step'], reverse=True), | |
| semantic_rows=sorted(semantic.values(), key=lambda r: r["step"]), | |
| wandb_error=WANDB_ERROR, | |
| validation_rows=validation_rows, | |
| qa_cases=qa_cases, qa_rows=qa_summary(qa_cases), | |
| max_steps=config.get("train", {}).get("max_training_steps", "—"), | |
| batch=config.get("train", {}).get("train_batch_size", "—")) | |
| def media_path(run, request, *, remote=False): | |
| path = (run / unquote(request)).resolve() | |
| if not path.is_relative_to(run.resolve()) or path.name != "reference_motion_with_audio.mp4": | |
| raise ValueError("Not public media") | |
| if not render_ready(path, remote=remote): | |
| raise ValueError("Render unfinished") | |
| return path | |
| def handler(run, *, remote=False): | |
| cache = {"time": 0, "data": None} | |
| history = MetricHistory(run / "dashboard_metrics.jsonl") | |
| lock = threading.Lock() | |
| class Handler(BaseHTTPRequestHandler): | |
| def log_message(self, *_): | |
| pass | |
| def do_GET(self): | |
| route = urlsplit(self.path).path | |
| if route in ("/", "/index.html"): | |
| body, mime = PAGE.encode(), "text/html; charset=utf-8" | |
| elif route == "/api/status": | |
| query = parse_qs(urlsplit(self.path).query) | |
| mode = query.get("view", ["global"])[0] | |
| try: | |
| window = int(query.get("window", ["50000"])[0]) | |
| if mode not in ("global", "local", "current") or window not in (1000, 5000, 10000, 50000, 100000): | |
| raise ValueError | |
| except ValueError: | |
| self.send_error(400); return | |
| with lock: | |
| if cache["data"] is None or time.monotonic() - cache["time"] > 10: | |
| cache["data"] = snapshot(run, history, remote=remote) | |
| cache["time"] = time.monotonic() | |
| body = json.dumps(curve_view(cache["data"], mode, window), ensure_ascii=False).encode() | |
| mime = "application/json; charset=utf-8" | |
| elif route.startswith("/media/"): | |
| try: | |
| path = media_path(run, route[len("/media/"):], remote=remote) | |
| size = path.stat().st_size | |
| start, end = 0, size - 1 | |
| partial = self.headers.get("Range") | |
| if partial: | |
| match = re.fullmatch(r"bytes=(\d+)-(\d*)", partial) | |
| if not match: | |
| self.send_error(416); return | |
| start = int(match[1]) | |
| end = min(int(match[2]) if match[2] else end, end) | |
| if start > end: | |
| self.send_error(416); return | |
| with path.open("rb") as stream: | |
| self.send_response(206 if partial else 200) | |
| self.send_header("Content-Type", "video/mp4") | |
| self.send_header("Accept-Ranges", "bytes") | |
| self.send_header("Content-Length", str(end - start + 1)) | |
| if partial: | |
| self.send_header("Content-Range", f"bytes {start}-{end}/{size}") | |
| self.end_headers() | |
| stream.seek(start) | |
| remaining = end - start + 1 | |
| while remaining: | |
| chunk = stream.read(min(1024 * 1024, remaining)) | |
| if not chunk: break | |
| self.wfile.write(chunk) | |
| remaining -= len(chunk) | |
| except (BrokenPipeError, ConnectionResetError): | |
| return | |
| except (ValueError, OSError): | |
| self.send_error(404) | |
| return | |
| else: | |
| self.send_error(404); return | |
| self.send_response(200) | |
| self.send_header("Content-Type", mime) | |
| self.send_header("Cache-Control", "no-store") | |
| self.send_header("Content-Length", str(len(body))) | |
| self.end_headers() | |
| self.wfile.write(body) | |
| return Handler | |
| if __name__ == "__main__": | |
| p = argparse.ArgumentParser() | |
| p.add_argument("run", type=Path) | |
| p.add_argument("--port", type=int, default=8772) | |
| p.add_argument("--host", default="127.0.0.1") | |
| p.add_argument("--wandb-run", default="") | |
| p.add_argument("--remote", action="store_true", help="Serve a remote run mirror without reporting this host's GPUs") | |
| args = p.parse_args() | |
| if args.wandb_run: | |
| threading.Thread(target=poll_wandb, args=(args.wandb_run,), daemon=True).start() | |
| ThreadingHTTPServer((args.host, args.port), handler(args.run.resolve(), remote=args.remote)).serve_forever() | |