#!/usr/bin/env python3 """Build a local, dependency-free dashboard from durable training records.""" from __future__ import annotations import argparse import html import json import math import os from pathlib import Path from typing import Any, Sequence PROJECT_ROOT = Path(__file__).resolve().parents[1] PANELS = ( ("Training loss", ("train/loss", "train/loss_action", "train/loss_control")), ("Training control accuracy", ("train/control_accuracy", "train/control_accuracy_listen", "train/control_accuracy_motion", "train/control_accuracy_idle")), ("Auxiliary loss", ("train/loss_motion_duration", "train/loss_latest_instruction", "train/loss_instruction_memory", "train/loss_smooth_weighted")), ("Learning rate", ("train/lr_act", "train/lr_control", "train/lr_vlm")), ("Gradient norm", ("train/grad_norm_act",)), ) COLORS = ("#1368ce", "#df621e", "#268343", "#a144c4", "#b03a48") def read_jsonl(path: Path) -> tuple[list[dict[str, Any]], bool]: if not path.exists(): return [], False rows, partial = [], False with path.open(encoding="utf-8") as handle: for line in handle: if not line.strip(): continue try: value = json.loads(line) except json.JSONDecodeError: if not line.endswith("\n"): partial = True break raise if not isinstance(value, dict): raise ValueError(f"Non-object training record: {path}") rows.append(value) return rows, partial def series(rows: Sequence[dict[str, Any]], key: str, max_points: int = 5000) -> list[tuple[float, float]]: if max_points < 2: raise ValueError("At least two curve samples are required") result = [] for row in rows: # Support unprefixed scalar logs without inventing missing observations. value = row.get(key, row.get(key.removeprefix("train/"))) step = row.get("step") if (isinstance(value, (int, float)) and not isinstance(value, bool) and isinstance(step, (int, float)) and not isinstance(step, bool) and math.isfinite(value) and math.isfinite(step)): result.append((float(step), float(value))) if len(result) > max_points: result = [result[round(index * (len(result) - 1) / (max_points - 1))] for index in range(max_points)] return result def render_curves(rows: Sequence[dict[str, Any]], max_points: int = 5000) -> str: panels = [] for title, keys in PANELS: curves = [(key, series(rows, key, max_points)) for key in keys] curves = [(key, points) for key, points in curves if points] if curves: panels.append((title, curves)) height = 75 + max(1, len(panels)) * 250 svg = [f'', '', '', 'Third-party training: recorded scalars', f'Optimizer step on x axis; missing values omitted; at most {max_points} sampled points per metric.'] if not panels: svg.append('No recorded training scalars yet (baseline probes may still be running).') for panel_index, (title, curves) in enumerate(panels): top = 88 + panel_index * 250 left, width, plot_height = 85, 865, 140 all_points = [point for _, points in curves for point in points] xmin, xmax = min(p[0] for p in all_points), max(p[0] for p in all_points) ymin, ymax = min(p[1] for p in all_points), max(p[1] for p in all_points) if xmax == xmin: xmax = xmin + 1 if ymax == ymin: padding = max(abs(ymin) * 0.05, 0.01) ymin, ymax = ymin - padding, ymax + padding svg.append(f'{html.escape(title)}') plot_top = top + 15 for tick in range(5): y = plot_top + tick * plot_height / 4 value = ymax - tick * (ymax - ymin) / 4 svg.extend([f'', f'{value:.4g}']) svg.append(f'{xmin:.0f}') svg.append(f'{xmax:.0f}') for index, (key, points) in enumerate(curves): color = COLORS[index % len(COLORS)] coords = [(left + (x - xmin) / (xmax - xmin) * width, plot_top + (ymax - y) / (ymax - ymin) * plot_height) for x, y in points] joined = " ".join(f"{x:.2f},{y:.2f}" for x, y in coords) svg.append(f'') x, y = coords[-1] svg.append(f'') legend_x = 85 + (index % 2) * 440 legend_y = plot_top + plot_height + 35 + (index // 2) * 16 label = f"{key}: last={points[-1][1]:.5g}" svg.append(f'{html.escape(label)}') svg.extend(["", ""]) return "\n".join(svg) + "\n" def _atomic_text(path: Path, value: str) -> None: temporary = path.with_suffix(path.suffix + ".tmp") temporary.write_text(value, encoding="utf-8") os.replace(temporary, path) def _report_link(run_dir: Path, path: str) -> Path: report_path = Path(path) if not report_path.is_absolute(): report_path = run_dir / report_path report_path = report_path.resolve() if not report_path.is_relative_to(run_dir): raise ValueError("Dashboard report link escapes this experiment directory") return report_path def update_dashboard(run_dir: str | Path) -> dict[str, Any]: run_dir = Path(run_dir).resolve() if not run_dir.is_relative_to(PROJECT_ROOT.resolve() / "runs"): raise ValueError("Dashboard outputs must remain under HFD_lzk/runs") if not run_dir.is_dir(): raise FileNotFoundError(run_dir) metrics, partial_metrics = read_jsonl(run_dir / "metrics.jsonl") events, partial_events = read_jsonl(run_dir / "events.jsonl") probes, partial_probes = read_jsonl(run_dir / "checkpoint_probes/history.jsonl") _atomic_text(run_dir / "curves.svg", render_curves(metrics)) last_step = max((int(row["step"]) for row in metrics if isinstance(row.get("step"), (int, float))), default=0) resume_path = run_dir / "resume_identity.json" resume_lines = [] if resume_path.is_file(): resume = json.loads(resume_path.read_text(encoding="utf-8")) source_run = Path(resume["source_run"]).resolve() if (not source_run.is_relative_to(PROJECT_ROOT.resolve() / "runs") or source_run == run_dir): raise ValueError("Resume history must link a separate experiment under HFD_lzk/runs") step = int(resume["step"]) restored = any(row.get("event") == "checkpoint_resume_passed" and row.get("step") == step for row in events) if restored: last_step = max(last_step, step) previous = os.path.relpath(source_run / "review.md", run_dir) videos = os.path.relpath(source_run / "checkpoint_probes" / f"step_{step:08d}" / "README.md", run_dir) resume_lines = [f"续训起点:**{step}**;状态恢复:**{'已验证' if restored else '待验证'}**。", f"[上次实验记录](<{previous}>) · [恢复点的预测视频](<{videos}>)。", "曲线只显示本次续训的数据;上次记录保留在原目录,不重复渲染已有视频。", ""] latest_event = str(events[-1].get("event", "无事件")) if events else "尚无训练事件" lines = ["# Third-party 训练监督记录", "", f"已记录的 optimizer step:**{last_step}**;最近事件:`{latest_event}`。", "这是一份自动生成的本地记录,无需实时盯终端;异常不等于已恢复训练。", "", "![训练曲线](curves.svg)", "", "| 记录 | 内容 |", "|---|---|", "| [metrics.jsonl](metrics.jsonl) | 原始 loss、控制准确率、优化器指标 |", "| [events.jsonl](events.jsonl) | 保存 checkpoint、预测开始/完成/失败事件 |", "| [run_status.json](run_status.json) | 后台任务退出码与占卡恢复状态 |", "", "## Checkpoint 闭环复查", "", "训练集固定样本;表中视频都是模型预测,不是 GT。只能判断训练集拟合,不能声称测试集泛化。", "", "| Step | 状态 | 完成样本 | 控制准确率 | 背景非动作误触发率 | 记录/视频 |", "|---:|---|---:|---:|---:|---|"] lines[2:2] = resume_lines if not probes: lines.append("| — | 尚无完整 probe 记录 | — | — | — | — |") for entry in probes: report_path = _report_link(run_dir, entry["report"]) report = json.loads(report_path.read_text(encoding="utf-8")) aggregate = report.get("aggregate", {}) def display(value): return f"{value:.4f}" if isinstance(value, (int, float)) and math.isfinite(value) else "—" readme = report_path.with_name("README.md").relative_to(run_dir) lines.append(f"| {entry.get('global_step', '—')} | {report.get('status', 'unknown')} | " f"{len(report.get('samples', []))} | {display(aggregate.get('control_accuracy'))} | " f"{display(aggregate.get('background_only_false_motion_rate'))} | [查看]({readme}) |") failures = [event for event in events if any(word in str(event.get("event", "")) for word in ("failed", "nonfinite", "error"))] if failures: lines.extend(["", "## 已记录异常", ""]) for event in failures[-10:]: lines.append(f"- step {event.get('step', '—')}:`{event.get('event', 'unknown')}`;详见 events.jsonl。") partial = partial_metrics or partial_events or partial_probes if partial: lines.extend(["", "末尾一条日志仍在写入,本次暂略;未把缺失值补为 0。"]) lines.extend(["", "曲线只显示已记录的观测值;最多均匀抽样 5000 点/指标。完整数据保留在原始 JSONL。"]) _atomic_text(run_dir / "review.md", "\n".join(lines) + "\n") return {"metric_records": len(metrics), "event_records": len(events), "probe_records": len(probes), "last_recorded_step": last_step, "partial_trailing_record": partial, "curves": str(run_dir / "curves.svg"), "review": str(run_dir / "review.md")} if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("run_dir", type=Path) args = parser.parse_args() print(json.dumps(update_dashboard(args.run_dir), ensure_ascii=False, indent=2))