#!/usr/bin/env python3 """Read-only, run-scoped curves and rendered videos; no external JS/CDN.""" import argparse from functools import lru_cache from collections import Counter from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json import math import mimetypes from pathlib import Path import re import subprocess import threading import time import unicodedata from urllib.parse import parse_qs, unquote, urlsplit WANDB_ROWS = [] WANDB_ERROR = "" CURVE_KEYS = ("train/loss", "train/lr_act", "train/loss_action", "train/control_accuracy", "train/swa_qa_token_accuracy", "train/swa_qa_loss", "eval/loss", "eval/control_accuracy", "train/token_control_macro_f1", "train/token_false_transition_rate", "train/token_flicker_rate", "eval/token_control_macro_f1") def answer_tokens(text): # A transparent lexical proxy, not a semantic or speech-quality judge. text = unicodedata.normalize("NFKC", str(text)).lower() return re.findall(r"[\u4e00-\u9fff]|[^\W_]+", re.sub(r"([\u4e00-\u9fff])", r" \1 ", text)) def qa_case(metric, directory): """Score actual answer triggers against labeled reply windows, never GT audio. One matched trigger per contiguous positive interval; further triggers are extras. Content uses the FIRST on-time answer, not best-of-many. Multiple questions without per-event references get timing scores only. """ if not metric.get("events") or "overlay" not in metric: return None events = sorted(metric["events"], key=lambda event: event["seconds"]) if any("reply_gt" not in event for event in events): return None intervals = [] previous = False for event in events: positive = bool(event["reply_gt"]) if positive: if not previous: intervals.append([event["seconds"], event["seconds"]]) else: intervals[-1][1] = event["seconds"] previous = positive matched, extras, audio = {}, 0, 0 answers = sorted(metric.get("answers", []), key=lambda answer: answer.get("seconds", float("inf"))) for answer in answers: seconds = answer.get("seconds") index = next((i for i, (start, end) in enumerate(intervals) if isinstance(seconds, (int, float)) and start-1e-6 <= seconds <= end+1e-6), None) if index is not None and index not in matched: matched[index] = answer else: extras += 1 path = (directory / answer.get("audio_path", "")).resolve() if path.is_relative_to(directory.resolve()) and path.is_file() and path.stat().st_size > 44: audio += 1 overlay = metric["overlay"] result = dict(question=overlay.get("question", ""), reference=overlay.get("answer", ""), speech_failures=len(metric.get('speech_failures',[])), predictions=[dict(seconds=a.get("seconds"), text=a.get("text", "")) for a in answers], questions=len(intervals), on_time=len(matched), extra_answers=extras, answers=len(answers), audio_files=audio, source_id=metric.get("source_id", ""), asset_id=overlay.get("asset_id", "")) # Missing/late answer is a failure, not omitted from content denominator. target = answer_tokens(result["reference"]) if len(intervals) == 1 and target: prediction = answer_tokens(matched.get(0, {}).get("text", "")) overlap = sum((Counter(prediction) & Counter(target)).values()) result.update(text_em=float(prediction == target), text_f1=2*overlap/(len(prediction)+len(target))) return result def qa_summary(cases): grouped = {} for case in cases: grouped.setdefault((case["step"], case["split"]), []).append(case) rows = [] for (step, split), items in sorted(grouped.items()): total = sum(item["questions"] for item in items) row = dict(step=step, case=split, sessions=len(items), questions=total, extra_answers_per_session=sum(item["extra_answers"] for item in items)/len(items)) if total: row["reply_recall"] = sum(item["on_time"] for item in items)/total for key in ("text_em", "text_f1"): values = [item[key] for item in items if key in item] if values: row[key] = sum(values)/len(values) row["text_samples"] = len(values) rows.append(row) return rows class MetricHistory: """Read the complete history once, then only appended complete JSONL lines.""" def __init__(self, path): self.path, self.identity, self.offset, self.rows = path, None, 0, [] def read(self): try: with self.path.open("rb") as stream: import os stat = os.fstat(stream.fileno()) identity = (stat.st_dev, stat.st_ino) if identity != self.identity or stat.st_size < self.offset: self.rows, self.offset = [], 0 self.identity = identity stream.seek(self.offset) while True: line = stream.readline() if not line.endswith(b"\n"): break # Retry incomplete final line next refresh. self.offset = stream.tell() try: row = json.loads(line) if not isinstance(row, dict) or not isinstance(row.get("step"), (int, float)): continue if not math.isfinite(row["step"]): continue self.rows.append({key: value for key, value in row.items() if key in (*CURVE_KEYS, "step", "time", "train/swa_qa_target_tokens") and isinstance(value, (int, float)) and math.isfinite(value)}) if self.rows[-1].get("train/swa_qa_target_tokens", 0) <= 0: for key in ("train/swa_qa_loss", "train/swa_qa_token_accuracy"): self.rows[-1].pop(key, None) except (ValueError, UnicodeDecodeError): continue except OSError: pass return self.rows def compact_curves(rows, bins=400): """Bound display size per metric, retaining endpoints and each bin's extrema. Sample metrics independently so sparse eval/control points are not discarded by the much denser loss stream. Does not alter the training log on disk. """ selected = set() for key in CURVE_KEYS: indices = [i for i, row in enumerate(rows) if key in row] if len(indices) <= bins * 2: selected.update(indices) continue selected.update((indices[0], indices[-1])) for start in range(bins): bucket = indices[len(indices)*start//bins:len(indices)*(start+1)//bins] selected.add(min(bucket, key=lambda i: rows[i][key])) selected.add(max(bucket, key=lambda i: rows[i][key])) if rows: selected.update((0, len(rows)-1)) return [rows[i] for i in sorted(selected)] def curve_view(data, mode="global", window=50000): latest = max((r["step"] for name in ("rows", "semantic_rows", "validation_rows") for r in data[name]), default=0) first = min((r["step"] for r in data["rows"]), default=0) start = max(0, latest-window) if mode == "local" else first if mode == "current" else 0 result = dict(data, step_range=[start, max(start+1, latest)], view=mode, first_step=first, latest=data["rows"][-1] if data["rows"] else {}) for name in ("rows", "semantic_rows", "validation_rows"): rows = sorted((r for r in data[name] if start <= r["step"] <= latest), key=lambda r: r["step"]) result[name] = rows if name == "validation_rows" else compact_curves(rows) result["qa_rows"] = [r for r in data.get("qa_rows", []) if start <= r["step"] <= latest] cases = [r for r in data.get("qa_cases", []) if start <= r["step"] <= latest] result["qa_cases"] = [r for r in cases if r["step"] == max( x["step"] for x in cases if x["split"] == r["split"])] return result def poll_wandb(path): """Read sparse semantic metrics off-thread; never delays the training job.""" global WANDB_ROWS, WANDB_ERROR import wandb while True: try: run = wandb.Api(timeout=15).run(path) collected = {} for keys in (["train/control_accuracy"], ["train/swa_qa_loss", "train/swa_qa_token_accuracy", "train/swa_qa_target_tokens"]): for row in run.history(samples=1000, keys=keys, pandas=False): if "_step" not in row: continue if "train/swa_qa_target_tokens" in row and row["train/swa_qa_target_tokens"] <= 0: continue step = int(row.pop("_step")) collected.setdefault(step, {"step": step}).update(row) WANDB_ROWS = list(collected.values()) WANDB_ERROR = "" except Exception as error: WANDB_ERROR = type(error).__name__ time.sleep(60) PAGE = r"""HFD 训练与推理
HFD / EXPERIMENT MONITOR

训练与推理

读取最新指标…
—等待指标
—越低越好
—越高越好
—在「推理视频」中按任务筛选
显示口径与运行配置

每15秒刷新。浅线为保留极值的展示点,深线是20个展示点均值,不是全量统计均值。悬停查看原始展示点。缺失指标不填0;「本次续训」从本地首条记录开始,不把之前没有的曲线补出来。

训练 Loss

控制准确率

动作 Loss

学习率

更多控制诊断 · 状态 F1 / 误切换 / 闪烁

状态分类宏平均 F1 ↑

状态误切换率 ↓

状态往返闪烁率 ↓

""" @lru_cache(maxsize=1024) def _verified_remote_video(filename, size, mtime_ns, metadata_text): """Mirrors omit dot-markers; verify a completed renderer receipt and MP4.""" try: meta = json.loads(metadata_text) if meta.get("swa_render_version") != 2 or int(meta.get("total_frames", 0)) <= 0: return False probe = json.loads(subprocess.check_output([ "ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=nb_frames", "-of", "json", filename, ], text=True, timeout=5)) return int(probe["streams"][0]["nb_frames"]) == int(meta["total_frames"]) except (OSError, subprocess.SubprocessError, ValueError, KeyError, IndexError, TypeError): return False def render_ready(path, *, remote=False): if (path.parent / ".rendered").is_file(): return True if not remote: return False try: stat = path.stat() meta = (path.parent / "infer_meta.json").read_text() return _verified_remote_video(str(path), stat.st_size, stat.st_mtime_ns, meta) except OSError: return False def snapshot(run, history=None, *, remote=False): rows = (history or MetricHistory(run / "dashboard_metrics.jsonl")).read() config = json.loads((run / "run_config.json").read_text()) if (run / "run_config.json").is_file() else {} videos = [] validation_rows = [] qa_cases = [] video_catalog = {} for path in (run / "unified_overlay").glob("ckpt_*/*/**/metrics.json"): try: metric = json.loads(path.read_text()) relative = path.relative_to(run / "unified_overlay") row = dict(step=int(relative.parts[0].split("_")[1]), case="/".join(relative.parts[2:-1])) split = relative.parts[1] if split not in ("train", "val"): continue video_catalog.setdefault(row['step'], dict(step=row['step'], inferred=0, rendered=0))['inferred'] += 1 qa = qa_case(metric, path.parent) if qa is not None: qa_cases.append(dict(qa, **row, split=split)) for key, value in {"control_accuracy": metric.get("control_accuracy"), **metric.get("body_metrics", {})}.items(): if isinstance(value, (int, float)) and math.isfinite(value): row[key] = value if split == "val": validation_rows.append(row) except (OSError, ValueError, TypeError): continue validation_rows.sort(key=lambda row: row["step"]) for path in (run / "unified_overlay").glob("ckpt_*/*/**/reference_motion_with_audio.mp4"): if not render_ready(path, remote=remote): continue summary = "" m = {} try: m = json.loads((path.parent / "metrics.json").read_text()) summary = "控制准确率 %.3f | 回答:%s" % ( m.get("control_accuracy", 0), "; ".join(a.get("text", "") for a in m.get("answers", [])), ) except (OSError, ValueError): pass relative = path.relative_to(run / 'unified_overlay') step = int(relative.parts[0].split('_')[1]) split = relative.parts[1] if split not in ('train', 'val'): continue overlay = m.get('overlay', {}) kind = 'video' if overlay.get('video_path') else 'image' if overlay.get('image_path') else 'none' videos.append(dict(path=str(path.relative_to(run)), summary=summary, step=step, split=split, task='/'.join(relative.parts[2:-1]), visual_kind=kind, question=overlay.get('question', ''), reference=overlay.get('answer', ''), answer=' / '.join(a.get('text', '') for a in m.get('answers', [])))) video_catalog.setdefault(step, dict(step=step, inferred=0, rendered=0))['rendered'] += 1 videos.sort(key=lambda v: int(re.search(r"ckpt_(\d+)", v["path"])[1]), reverse=True) if remote: gpu = "远端 DLC 训练:展示 CPFS 同步曲线与视频;不把本机 GPU 状态误作训练卡状态。" else: try: gpu = subprocess.check_output( ["nvidia-smi", "--query-gpu=index,utilization.gpu,memory.used,memory.total", "--format=csv"], text=True, timeout=4, ) except (OSError, subprocess.SubprocessError): gpu = "GPU状态暂不可用" semantic = {} for row in rows: semantic.setdefault(row["step"], {}).update(row) for row in WANDB_ROWS: semantic.setdefault(row["step"], {}).update(row) return dict(name=run.name, rows=rows, videos=videos, gpu=gpu, architecture=config.get('model',{}).get('interaction_architecture','legacy'), video_catalog=sorted(video_catalog.values(), key=lambda r: r['step'], reverse=True), semantic_rows=sorted(semantic.values(), key=lambda r: r["step"]), wandb_error=WANDB_ERROR, validation_rows=validation_rows, qa_cases=qa_cases, qa_rows=qa_summary(qa_cases), max_steps=config.get("train", {}).get("max_training_steps", "—"), batch=config.get("train", {}).get("train_batch_size", "—")) def media_path(run, request, *, remote=False): path = (run / unquote(request)).resolve() if not path.is_relative_to(run.resolve()) or path.name != "reference_motion_with_audio.mp4": raise ValueError("Not public media") if not render_ready(path, remote=remote): raise ValueError("Render unfinished") return path def handler(run, *, remote=False): cache = {"time": 0, "data": None} history = MetricHistory(run / "dashboard_metrics.jsonl") lock = threading.Lock() class Handler(BaseHTTPRequestHandler): def log_message(self, *_): pass def do_GET(self): route = urlsplit(self.path).path if route in ("/", "/index.html"): body, mime = PAGE.encode(), "text/html; charset=utf-8" elif route == "/api/status": query = parse_qs(urlsplit(self.path).query) mode = query.get("view", ["global"])[0] try: window = int(query.get("window", ["50000"])[0]) if mode not in ("global", "local", "current") or window not in (1000, 5000, 10000, 50000, 100000): raise ValueError except ValueError: self.send_error(400); return with lock: if cache["data"] is None or time.monotonic() - cache["time"] > 10: cache["data"] = snapshot(run, history, remote=remote) cache["time"] = time.monotonic() body = json.dumps(curve_view(cache["data"], mode, window), ensure_ascii=False).encode() mime = "application/json; charset=utf-8" elif route.startswith("/media/"): try: path = media_path(run, route[len("/media/"):], remote=remote) size = path.stat().st_size start, end = 0, size - 1 partial = self.headers.get("Range") if partial: match = re.fullmatch(r"bytes=(\d+)-(\d*)", partial) if not match: self.send_error(416); return start = int(match[1]) end = min(int(match[2]) if match[2] else end, end) if start > end: self.send_error(416); return with path.open("rb") as stream: self.send_response(206 if partial else 200) self.send_header("Content-Type", "video/mp4") self.send_header("Accept-Ranges", "bytes") self.send_header("Content-Length", str(end - start + 1)) if partial: self.send_header("Content-Range", f"bytes {start}-{end}/{size}") self.end_headers() stream.seek(start) remaining = end - start + 1 while remaining: chunk = stream.read(min(1024 * 1024, remaining)) if not chunk: break self.wfile.write(chunk) remaining -= len(chunk) except (BrokenPipeError, ConnectionResetError): return except (ValueError, OSError): self.send_error(404) return else: self.send_error(404); return self.send_response(200) self.send_header("Content-Type", mime) self.send_header("Cache-Control", "no-store") self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) return Handler if __name__ == "__main__": p = argparse.ArgumentParser() p.add_argument("run", type=Path) p.add_argument("--port", type=int, default=8772) p.add_argument("--host", default="127.0.0.1") p.add_argument("--wandb-run", default="") p.add_argument("--remote", action="store_true", help="Serve a remote run mirror without reporting this host's GPUs") args = p.parse_args() if args.wandb_run: threading.Thread(target=poll_wandb, args=(args.wandb_run,), daemon=True).start() ThreadingHTTPServer((args.host, args.port), handler(args.run.resolve(), remote=args.remote)).serve_forever()