File size: 11,534 Bytes
55e0599 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 | #!/usr/bin/env python3
"""Build a local, dependency-free dashboard from durable training records."""
from __future__ import annotations
import argparse
import html
import json
import math
import os
from pathlib import Path
from typing import Any, Sequence
PROJECT_ROOT = Path(__file__).resolve().parents[1]
PANELS = (
("Training loss", ("train/loss", "train/loss_action", "train/loss_control")),
("Training control accuracy", ("train/control_accuracy", "train/control_accuracy_listen",
"train/control_accuracy_motion", "train/control_accuracy_idle")),
("Auxiliary loss", ("train/loss_motion_duration", "train/loss_latest_instruction",
"train/loss_instruction_memory", "train/loss_smooth_weighted")),
("Learning rate", ("train/lr_act", "train/lr_control", "train/lr_vlm")),
("Gradient norm", ("train/grad_norm_act",)),
)
COLORS = ("#1368ce", "#df621e", "#268343", "#a144c4", "#b03a48")
def read_jsonl(path: Path) -> tuple[list[dict[str, Any]], bool]:
if not path.exists():
return [], False
rows, partial = [], False
with path.open(encoding="utf-8") as handle:
for line in handle:
if not line.strip():
continue
try:
value = json.loads(line)
except json.JSONDecodeError:
if not line.endswith("\n"):
partial = True
break
raise
if not isinstance(value, dict):
raise ValueError(f"Non-object training record: {path}")
rows.append(value)
return rows, partial
def series(rows: Sequence[dict[str, Any]], key: str, max_points: int = 5000) -> list[tuple[float, float]]:
if max_points < 2:
raise ValueError("At least two curve samples are required")
result = []
for row in rows:
# Support unprefixed scalar logs without inventing missing observations.
value = row.get(key, row.get(key.removeprefix("train/")))
step = row.get("step")
if (isinstance(value, (int, float)) and not isinstance(value, bool)
and isinstance(step, (int, float)) and not isinstance(step, bool)
and math.isfinite(value) and math.isfinite(step)):
result.append((float(step), float(value)))
if len(result) > max_points:
result = [result[round(index * (len(result) - 1) / (max_points - 1))]
for index in range(max_points)]
return result
def render_curves(rows: Sequence[dict[str, Any]], max_points: int = 5000) -> str:
panels = []
for title, keys in PANELS:
curves = [(key, series(rows, key, max_points)) for key in keys]
curves = [(key, points) for key, points in curves if points]
if curves:
panels.append((title, curves))
height = 75 + max(1, len(panels)) * 250
svg = [f'<svg xmlns="http://www.w3.org/2000/svg" width="1000" height="{height}" viewBox="0 0 1000 {height}">',
'<rect width="100%" height="100%" fill="#fff"/>',
'<g font-family="sans-serif" fill="#202b38">',
'<text x="35" y="28" font-size="19">Third-party training: recorded scalars</text>',
f'<text x="35" y="49" font-size="12">Optimizer step on x axis; missing values omitted; at most {max_points} sampled points per metric.</text>']
if not panels:
svg.append('<text x="35" y="105" font-size="15">No recorded training scalars yet (baseline probes may still be running).</text>')
for panel_index, (title, curves) in enumerate(panels):
top = 88 + panel_index * 250
left, width, plot_height = 85, 865, 140
all_points = [point for _, points in curves for point in points]
xmin, xmax = min(p[0] for p in all_points), max(p[0] for p in all_points)
ymin, ymax = min(p[1] for p in all_points), max(p[1] for p in all_points)
if xmax == xmin:
xmax = xmin + 1
if ymax == ymin:
padding = max(abs(ymin) * 0.05, 0.01)
ymin, ymax = ymin - padding, ymax + padding
svg.append(f'<text x="35" y="{top}" font-size="16">{html.escape(title)}</text>')
plot_top = top + 15
for tick in range(5):
y = plot_top + tick * plot_height / 4
value = ymax - tick * (ymax - ymin) / 4
svg.extend([f'<line x1="{left}" y1="{y}" x2="{left + width}" y2="{y}" stroke="#e4e9ef"/>',
f'<text x="{left - 8}" y="{y + 4}" text-anchor="end" font-size="11">{value:.4g}</text>'])
svg.append(f'<text x="{left}" y="{plot_top + plot_height + 16}" font-size="11">{xmin:.0f}</text>')
svg.append(f'<text x="{left + width}" y="{plot_top + plot_height + 16}" text-anchor="end" font-size="11">{xmax:.0f}</text>')
for index, (key, points) in enumerate(curves):
color = COLORS[index % len(COLORS)]
coords = [(left + (x - xmin) / (xmax - xmin) * width,
plot_top + (ymax - y) / (ymax - ymin) * plot_height) for x, y in points]
joined = " ".join(f"{x:.2f},{y:.2f}" for x, y in coords)
svg.append(f'<polyline points="{joined}" fill="none" stroke="{color}" stroke-width="1.6"/>')
x, y = coords[-1]
svg.append(f'<circle cx="{x:.2f}" cy="{y:.2f}" r="2.5" fill="{color}"/>')
legend_x = 85 + (index % 2) * 440
legend_y = plot_top + plot_height + 35 + (index // 2) * 16
label = f"{key}: last={points[-1][1]:.5g}"
svg.append(f'<text x="{legend_x}" y="{legend_y}" fill="{color}" font-size="11">{html.escape(label)}</text>')
svg.extend(["</g>", "</svg>"])
return "\n".join(svg) + "\n"
def _atomic_text(path: Path, value: str) -> None:
temporary = path.with_suffix(path.suffix + ".tmp")
temporary.write_text(value, encoding="utf-8")
os.replace(temporary, path)
def _report_link(run_dir: Path, path: str) -> Path:
report_path = Path(path)
if not report_path.is_absolute():
report_path = run_dir / report_path
report_path = report_path.resolve()
if not report_path.is_relative_to(run_dir):
raise ValueError("Dashboard report link escapes this experiment directory")
return report_path
def update_dashboard(run_dir: str | Path) -> dict[str, Any]:
run_dir = Path(run_dir).resolve()
if not run_dir.is_relative_to(PROJECT_ROOT.resolve() / "runs"):
raise ValueError("Dashboard outputs must remain under HFD_lzk/runs")
if not run_dir.is_dir():
raise FileNotFoundError(run_dir)
metrics, partial_metrics = read_jsonl(run_dir / "metrics.jsonl")
events, partial_events = read_jsonl(run_dir / "events.jsonl")
probes, partial_probes = read_jsonl(run_dir / "checkpoint_probes/history.jsonl")
_atomic_text(run_dir / "curves.svg", render_curves(metrics))
last_step = max((int(row["step"]) for row in metrics if isinstance(row.get("step"), (int, float))), default=0)
resume_path = run_dir / "resume_identity.json"
resume_lines = []
if resume_path.is_file():
resume = json.loads(resume_path.read_text(encoding="utf-8"))
source_run = Path(resume["source_run"]).resolve()
if (not source_run.is_relative_to(PROJECT_ROOT.resolve() / "runs")
or source_run == run_dir):
raise ValueError("Resume history must link a separate experiment under HFD_lzk/runs")
step = int(resume["step"])
restored = any(row.get("event") == "checkpoint_resume_passed" and row.get("step") == step
for row in events)
if restored:
last_step = max(last_step, step)
previous = os.path.relpath(source_run / "review.md", run_dir)
videos = os.path.relpath(source_run / "checkpoint_probes" / f"step_{step:08d}" / "README.md", run_dir)
resume_lines = [f"续训起点:**{step}**;状态恢复:**{'已验证' if restored else '待验证'}**。",
f"[上次实验记录](<{previous}>) · [恢复点的预测视频](<{videos}>)。",
"曲线只显示本次续训的数据;上次记录保留在原目录,不重复渲染已有视频。", ""]
latest_event = str(events[-1].get("event", "无事件")) if events else "尚无训练事件"
lines = ["# Third-party 训练监督记录", "",
f"已记录的 optimizer step:**{last_step}**;最近事件:`{latest_event}`。",
"这是一份自动生成的本地记录,无需实时盯终端;异常不等于已恢复训练。", "",
"", "",
"| 记录 | 内容 |", "|---|---|",
"| [metrics.jsonl](metrics.jsonl) | 原始 loss、控制准确率、优化器指标 |",
"| [events.jsonl](events.jsonl) | 保存 checkpoint、预测开始/完成/失败事件 |",
"| [run_status.json](run_status.json) | 后台任务退出码与占卡恢复状态 |", "",
"## Checkpoint 闭环复查", "",
"训练集固定样本;表中视频都是模型预测,不是 GT。只能判断训练集拟合,不能声称测试集泛化。", "",
"| Step | 状态 | 完成样本 | 控制准确率 | 背景非动作误触发率 | 记录/视频 |",
"|---:|---|---:|---:|---:|---|"]
lines[2:2] = resume_lines
if not probes:
lines.append("| — | 尚无完整 probe 记录 | — | — | — | — |")
for entry in probes:
report_path = _report_link(run_dir, entry["report"])
report = json.loads(report_path.read_text(encoding="utf-8"))
aggregate = report.get("aggregate", {})
def display(value):
return f"{value:.4f}" if isinstance(value, (int, float)) and math.isfinite(value) else "—"
readme = report_path.with_name("README.md").relative_to(run_dir)
lines.append(f"| {entry.get('global_step', '—')} | {report.get('status', 'unknown')} | "
f"{len(report.get('samples', []))} | {display(aggregate.get('control_accuracy'))} | "
f"{display(aggregate.get('background_only_false_motion_rate'))} | [查看]({readme}) |")
failures = [event for event in events if any(word in str(event.get("event", ""))
for word in ("failed", "nonfinite", "error"))]
if failures:
lines.extend(["", "## 已记录异常", ""])
for event in failures[-10:]:
lines.append(f"- step {event.get('step', '—')}:`{event.get('event', 'unknown')}`;详见 events.jsonl。")
partial = partial_metrics or partial_events or partial_probes
if partial:
lines.extend(["", "末尾一条日志仍在写入,本次暂略;未把缺失值补为 0。"])
lines.extend(["", "曲线只显示已记录的观测值;最多均匀抽样 5000 点/指标。完整数据保留在原始 JSONL。"])
_atomic_text(run_dir / "review.md", "\n".join(lines) + "\n")
return {"metric_records": len(metrics), "event_records": len(events), "probe_records": len(probes),
"last_recorded_step": last_step, "partial_trailing_record": partial,
"curves": str(run_dir / "curves.svg"), "review": str(run_dir / "review.md")}
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("run_dir", type=Path)
args = parser.parse_args()
print(json.dumps(update_dashboard(args.run_dir), ensure_ascii=False, indent=2))
|