code-HFD_lzk / scripts /third_party_dashboard.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
55e0599 verified
Raw History Blame Contribute Delete
11.5 kB
#!/usr/bin/env python3
"""Build a local, dependency-free dashboard from durable training records."""
from __future__ import annotations
import argparse
import html
import json
import math
import os
from pathlib import Path
from typing import Any, Sequence
PROJECT_ROOT = Path(__file__).resolve().parents[1]
PANELS = (
("Training loss", ("train/loss", "train/loss_action", "train/loss_control")),
("Training control accuracy", ("train/control_accuracy", "train/control_accuracy_listen",
"train/control_accuracy_motion", "train/control_accuracy_idle")),
("Auxiliary loss", ("train/loss_motion_duration", "train/loss_latest_instruction",
"train/loss_instruction_memory", "train/loss_smooth_weighted")),
("Learning rate", ("train/lr_act", "train/lr_control", "train/lr_vlm")),
("Gradient norm", ("train/grad_norm_act",)),
)
COLORS = ("#1368ce", "#df621e", "#268343", "#a144c4", "#b03a48")
def read_jsonl(path: Path) -> tuple[list[dict[str, Any]], bool]:
if not path.exists():
return [], False
rows, partial = [], False
with path.open(encoding="utf-8") as handle:
for line in handle:
if not line.strip():
continue
try:
value = json.loads(line)
except json.JSONDecodeError:
if not line.endswith("\n"):
partial = True
break
raise
if not isinstance(value, dict):
raise ValueError(f"Non-object training record: {path}")
rows.append(value)
return rows, partial
def series(rows: Sequence[dict[str, Any]], key: str, max_points: int = 5000) -> list[tuple[float, float]]:
if max_points < 2:
raise ValueError("At least two curve samples are required")
result = []
for row in rows:
# Support unprefixed scalar logs without inventing missing observations.
value = row.get(key, row.get(key.removeprefix("train/")))
step = row.get("step")
if (isinstance(value, (int, float)) and not isinstance(value, bool)
and isinstance(step, (int, float)) and not isinstance(step, bool)
and math.isfinite(value) and math.isfinite(step)):
result.append((float(step), float(value)))
if len(result) > max_points:
result = [result[round(index * (len(result) - 1) / (max_points - 1))]
for index in range(max_points)]
return result
def render_curves(rows: Sequence[dict[str, Any]], max_points: int = 5000) -> str:
panels = []
for title, keys in PANELS:
curves = [(key, series(rows, key, max_points)) for key in keys]
curves = [(key, points) for key, points in curves if points]
if curves:
panels.append((title, curves))
height = 75 + max(1, len(panels)) * 250
svg = [f'<svg xmlns="http://www.w3.org/2000/svg" width="1000" height="{height}" viewBox="0 0 1000 {height}">',
'<rect width="100%" height="100%" fill="#fff"/>',
'<g font-family="sans-serif" fill="#202b38">',
'<text x="35" y="28" font-size="19">Third-party training: recorded scalars</text>',
f'<text x="35" y="49" font-size="12">Optimizer step on x axis; missing values omitted; at most {max_points} sampled points per metric.</text>']
if not panels:
svg.append('<text x="35" y="105" font-size="15">No recorded training scalars yet (baseline probes may still be running).</text>')
for panel_index, (title, curves) in enumerate(panels):
top = 88 + panel_index * 250
left, width, plot_height = 85, 865, 140
all_points = [point for _, points in curves for point in points]
xmin, xmax = min(p[0] for p in all_points), max(p[0] for p in all_points)
ymin, ymax = min(p[1] for p in all_points), max(p[1] for p in all_points)
if xmax == xmin:
xmax = xmin + 1
if ymax == ymin:
padding = max(abs(ymin) * 0.05, 0.01)
ymin, ymax = ymin - padding, ymax + padding
svg.append(f'<text x="35" y="{top}" font-size="16">{html.escape(title)}</text>')
plot_top = top + 15
for tick in range(5):
y = plot_top + tick * plot_height / 4
value = ymax - tick * (ymax - ymin) / 4
svg.extend([f'<line x1="{left}" y1="{y}" x2="{left + width}" y2="{y}" stroke="#e4e9ef"/>',
f'<text x="{left - 8}" y="{y + 4}" text-anchor="end" font-size="11">{value:.4g}</text>'])
svg.append(f'<text x="{left}" y="{plot_top + plot_height + 16}" font-size="11">{xmin:.0f}</text>')
svg.append(f'<text x="{left + width}" y="{plot_top + plot_height + 16}" text-anchor="end" font-size="11">{xmax:.0f}</text>')
for index, (key, points) in enumerate(curves):
color = COLORS[index % len(COLORS)]
coords = [(left + (x - xmin) / (xmax - xmin) * width,
plot_top + (ymax - y) / (ymax - ymin) * plot_height) for x, y in points]
joined = " ".join(f"{x:.2f},{y:.2f}" for x, y in coords)
svg.append(f'<polyline points="{joined}" fill="none" stroke="{color}" stroke-width="1.6"/>')
x, y = coords[-1]
svg.append(f'<circle cx="{x:.2f}" cy="{y:.2f}" r="2.5" fill="{color}"/>')
legend_x = 85 + (index % 2) * 440
legend_y = plot_top + plot_height + 35 + (index // 2) * 16
label = f"{key}: last={points[-1][1]:.5g}"
svg.append(f'<text x="{legend_x}" y="{legend_y}" fill="{color}" font-size="11">{html.escape(label)}</text>')
svg.extend(["</g>", "</svg>"])
return "\n".join(svg) + "\n"
def _atomic_text(path: Path, value: str) -> None:
temporary = path.with_suffix(path.suffix + ".tmp")
temporary.write_text(value, encoding="utf-8")
os.replace(temporary, path)
def _report_link(run_dir: Path, path: str) -> Path:
report_path = Path(path)
if not report_path.is_absolute():
report_path = run_dir / report_path
report_path = report_path.resolve()
if not report_path.is_relative_to(run_dir):
raise ValueError("Dashboard report link escapes this experiment directory")
return report_path
def update_dashboard(run_dir: str | Path) -> dict[str, Any]:
run_dir = Path(run_dir).resolve()
if not run_dir.is_relative_to(PROJECT_ROOT.resolve() / "runs"):
raise ValueError("Dashboard outputs must remain under HFD_lzk/runs")
if not run_dir.is_dir():
raise FileNotFoundError(run_dir)
metrics, partial_metrics = read_jsonl(run_dir / "metrics.jsonl")
events, partial_events = read_jsonl(run_dir / "events.jsonl")
probes, partial_probes = read_jsonl(run_dir / "checkpoint_probes/history.jsonl")
_atomic_text(run_dir / "curves.svg", render_curves(metrics))
last_step = max((int(row["step"]) for row in metrics if isinstance(row.get("step"), (int, float))), default=0)
resume_path = run_dir / "resume_identity.json"
resume_lines = []
if resume_path.is_file():
resume = json.loads(resume_path.read_text(encoding="utf-8"))
source_run = Path(resume["source_run"]).resolve()
if (not source_run.is_relative_to(PROJECT_ROOT.resolve() / "runs")
or source_run == run_dir):
raise ValueError("Resume history must link a separate experiment under HFD_lzk/runs")
step = int(resume["step"])
restored = any(row.get("event") == "checkpoint_resume_passed" and row.get("step") == step
for row in events)
if restored:
last_step = max(last_step, step)
previous = os.path.relpath(source_run / "review.md", run_dir)
videos = os.path.relpath(source_run / "checkpoint_probes" / f"step_{step:08d}" / "README.md", run_dir)
resume_lines = [f"续训起点:**{step}**;状态恢复:**{'已验证' if restored else '待验证'}**。",
f"[上次实验记录](<{previous}>) · [恢复点的预测视频](<{videos}>)。",
"曲线只显示本次续训的数据;上次记录保留在原目录,不重复渲染已有视频。", ""]
latest_event = str(events[-1].get("event", "无事件")) if events else "尚无训练事件"
lines = ["# Third-party 训练监督记录", "",
f"已记录的 optimizer step:**{last_step}**;最近事件:`{latest_event}`。",
"这是一份自动生成的本地记录,无需实时盯终端;异常不等于已恢复训练。", "",
"![训练曲线](curves.svg)", "",
"| 记录 | 内容 |", "|---|---|",
"| [metrics.jsonl](metrics.jsonl) | 原始 loss、控制准确率、优化器指标 |",
"| [events.jsonl](events.jsonl) | 保存 checkpoint、预测开始/完成/失败事件 |",
"| [run_status.json](run_status.json) | 后台任务退出码与占卡恢复状态 |", "",
"## Checkpoint 闭环复查", "",
"训练集固定样本;表中视频都是模型预测,不是 GT。只能判断训练集拟合,不能声称测试集泛化。", "",
"| Step | 状态 | 完成样本 | 控制准确率 | 背景非动作误触发率 | 记录/视频 |",
"|---:|---|---:|---:|---:|---|"]
lines[2:2] = resume_lines
if not probes:
lines.append("| — | 尚无完整 probe 记录 | — | — | — | — |")
for entry in probes:
report_path = _report_link(run_dir, entry["report"])
report = json.loads(report_path.read_text(encoding="utf-8"))
aggregate = report.get("aggregate", {})
def display(value):
return f"{value:.4f}" if isinstance(value, (int, float)) and math.isfinite(value) else "—"
readme = report_path.with_name("README.md").relative_to(run_dir)
lines.append(f"| {entry.get('global_step', '—')} | {report.get('status', 'unknown')} | "
f"{len(report.get('samples', []))} | {display(aggregate.get('control_accuracy'))} | "
f"{display(aggregate.get('background_only_false_motion_rate'))} | [查看]({readme}) |")
failures = [event for event in events if any(word in str(event.get("event", ""))
for word in ("failed", "nonfinite", "error"))]
if failures:
lines.extend(["", "## 已记录异常", ""])
for event in failures[-10:]:
lines.append(f"- step {event.get('step', '—')}:`{event.get('event', 'unknown')}`;详见 events.jsonl。")
partial = partial_metrics or partial_events or partial_probes
if partial:
lines.extend(["", "末尾一条日志仍在写入,本次暂略;未把缺失值补为 0。"])
lines.extend(["", "曲线只显示已记录的观测值;最多均匀抽样 5000 点/指标。完整数据保留在原始 JSONL。"])
_atomic_text(run_dir / "review.md", "\n".join(lines) + "\n")
return {"metric_records": len(metrics), "event_records": len(events), "probe_records": len(probes),
"last_recorded_step": last_step, "partial_trailing_record": partial,
"curves": str(run_dir / "curves.svg"), "review": str(run_dir / "review.md")}
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("run_dir", type=Path)
args = parser.parse_args()
print(json.dumps(update_dashboard(args.run_dir), ensure_ascii=False, indent=2))