Download scripts/third_party_dashboard.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 11.5 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/third_party_dashboard.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/third_party_dashboard.py
-
curl -L -o third_party_dashboard.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/third_party_dashboard.py
11.5 kB
| #!/usr/bin/env python3 | |
| """Build a local, dependency-free dashboard from durable training records.""" | |
| from __future__ import annotations | |
| import argparse | |
| import html | |
| import json | |
| import math | |
| import os | |
| from pathlib import Path | |
| from typing import Any, Sequence | |
| PROJECT_ROOT = Path(__file__).resolve().parents[1] | |
| PANELS = ( | |
| ("Training loss", ("train/loss", "train/loss_action", "train/loss_control")), | |
| ("Training control accuracy", ("train/control_accuracy", "train/control_accuracy_listen", | |
| "train/control_accuracy_motion", "train/control_accuracy_idle")), | |
| ("Auxiliary loss", ("train/loss_motion_duration", "train/loss_latest_instruction", | |
| "train/loss_instruction_memory", "train/loss_smooth_weighted")), | |
| ("Learning rate", ("train/lr_act", "train/lr_control", "train/lr_vlm")), | |
| ("Gradient norm", ("train/grad_norm_act",)), | |
| ) | |
| COLORS = ("#1368ce", "#df621e", "#268343", "#a144c4", "#b03a48") | |
| def read_jsonl(path: Path) -> tuple[list[dict[str, Any]], bool]: | |
| if not path.exists(): | |
| return [], False | |
| rows, partial = [], False | |
| with path.open(encoding="utf-8") as handle: | |
| for line in handle: | |
| if not line.strip(): | |
| continue | |
| try: | |
| value = json.loads(line) | |
| except json.JSONDecodeError: | |
| if not line.endswith("\n"): | |
| partial = True | |
| break | |
| raise | |
| if not isinstance(value, dict): | |
| raise ValueError(f"Non-object training record: {path}") | |
| rows.append(value) | |
| return rows, partial | |
| def series(rows: Sequence[dict[str, Any]], key: str, max_points: int = 5000) -> list[tuple[float, float]]: | |
| if max_points < 2: | |
| raise ValueError("At least two curve samples are required") | |
| result = [] | |
| for row in rows: | |
| # Support unprefixed scalar logs without inventing missing observations. | |
| value = row.get(key, row.get(key.removeprefix("train/"))) | |
| step = row.get("step") | |
| if (isinstance(value, (int, float)) and not isinstance(value, bool) | |
| and isinstance(step, (int, float)) and not isinstance(step, bool) | |
| and math.isfinite(value) and math.isfinite(step)): | |
| result.append((float(step), float(value))) | |
| if len(result) > max_points: | |
| result = [result[round(index * (len(result) - 1) / (max_points - 1))] | |
| for index in range(max_points)] | |
| return result | |
| def render_curves(rows: Sequence[dict[str, Any]], max_points: int = 5000) -> str: | |
| panels = [] | |
| for title, keys in PANELS: | |
| curves = [(key, series(rows, key, max_points)) for key in keys] | |
| curves = [(key, points) for key, points in curves if points] | |
| if curves: | |
| panels.append((title, curves)) | |
| height = 75 + max(1, len(panels)) * 250 | |
| svg = [f'<svg xmlns="http://www.w3.org/2000/svg" width="1000" height="{height}" viewBox="0 0 1000 {height}">', | |
| '<rect width="100%" height="100%" fill="#fff"/>', | |
| '<g font-family="sans-serif" fill="#202b38">', | |
| '<text x="35" y="28" font-size="19">Third-party training: recorded scalars</text>', | |
| f'<text x="35" y="49" font-size="12">Optimizer step on x axis; missing values omitted; at most {max_points} sampled points per metric.</text>'] | |
| if not panels: | |
| svg.append('<text x="35" y="105" font-size="15">No recorded training scalars yet (baseline probes may still be running).</text>') | |
| for panel_index, (title, curves) in enumerate(panels): | |
| top = 88 + panel_index * 250 | |
| left, width, plot_height = 85, 865, 140 | |
| all_points = [point for _, points in curves for point in points] | |
| xmin, xmax = min(p[0] for p in all_points), max(p[0] for p in all_points) | |
| ymin, ymax = min(p[1] for p in all_points), max(p[1] for p in all_points) | |
| if xmax == xmin: | |
| xmax = xmin + 1 | |
| if ymax == ymin: | |
| padding = max(abs(ymin) * 0.05, 0.01) | |
| ymin, ymax = ymin - padding, ymax + padding | |
| svg.append(f'<text x="35" y="{top}" font-size="16">{html.escape(title)}</text>') | |
| plot_top = top + 15 | |
| for tick in range(5): | |
| y = plot_top + tick * plot_height / 4 | |
| value = ymax - tick * (ymax - ymin) / 4 | |
| svg.extend([f'<line x1="{left}" y1="{y}" x2="{left + width}" y2="{y}" stroke="#e4e9ef"/>', | |
| f'<text x="{left - 8}" y="{y + 4}" text-anchor="end" font-size="11">{value:.4g}</text>']) | |
| svg.append(f'<text x="{left}" y="{plot_top + plot_height + 16}" font-size="11">{xmin:.0f}</text>') | |
| svg.append(f'<text x="{left + width}" y="{plot_top + plot_height + 16}" text-anchor="end" font-size="11">{xmax:.0f}</text>') | |
| for index, (key, points) in enumerate(curves): | |
| color = COLORS[index % len(COLORS)] | |
| coords = [(left + (x - xmin) / (xmax - xmin) * width, | |
| plot_top + (ymax - y) / (ymax - ymin) * plot_height) for x, y in points] | |
| joined = " ".join(f"{x:.2f},{y:.2f}" for x, y in coords) | |
| svg.append(f'<polyline points="{joined}" fill="none" stroke="{color}" stroke-width="1.6"/>') | |
| x, y = coords[-1] | |
| svg.append(f'<circle cx="{x:.2f}" cy="{y:.2f}" r="2.5" fill="{color}"/>') | |
| legend_x = 85 + (index % 2) * 440 | |
| legend_y = plot_top + plot_height + 35 + (index // 2) * 16 | |
| label = f"{key}: last={points[-1][1]:.5g}" | |
| svg.append(f'<text x="{legend_x}" y="{legend_y}" fill="{color}" font-size="11">{html.escape(label)}</text>') | |
| svg.extend(["</g>", "</svg>"]) | |
| return "\n".join(svg) + "\n" | |
| def _atomic_text(path: Path, value: str) -> None: | |
| temporary = path.with_suffix(path.suffix + ".tmp") | |
| temporary.write_text(value, encoding="utf-8") | |
| os.replace(temporary, path) | |
| def _report_link(run_dir: Path, path: str) -> Path: | |
| report_path = Path(path) | |
| if not report_path.is_absolute(): | |
| report_path = run_dir / report_path | |
| report_path = report_path.resolve() | |
| if not report_path.is_relative_to(run_dir): | |
| raise ValueError("Dashboard report link escapes this experiment directory") | |
| return report_path | |
| def update_dashboard(run_dir: str | Path) -> dict[str, Any]: | |
| run_dir = Path(run_dir).resolve() | |
| if not run_dir.is_relative_to(PROJECT_ROOT.resolve() / "runs"): | |
| raise ValueError("Dashboard outputs must remain under HFD_lzk/runs") | |
| if not run_dir.is_dir(): | |
| raise FileNotFoundError(run_dir) | |
| metrics, partial_metrics = read_jsonl(run_dir / "metrics.jsonl") | |
| events, partial_events = read_jsonl(run_dir / "events.jsonl") | |
| probes, partial_probes = read_jsonl(run_dir / "checkpoint_probes/history.jsonl") | |
| _atomic_text(run_dir / "curves.svg", render_curves(metrics)) | |
| last_step = max((int(row["step"]) for row in metrics if isinstance(row.get("step"), (int, float))), default=0) | |
| resume_path = run_dir / "resume_identity.json" | |
| resume_lines = [] | |
| if resume_path.is_file(): | |
| resume = json.loads(resume_path.read_text(encoding="utf-8")) | |
| source_run = Path(resume["source_run"]).resolve() | |
| if (not source_run.is_relative_to(PROJECT_ROOT.resolve() / "runs") | |
| or source_run == run_dir): | |
| raise ValueError("Resume history must link a separate experiment under HFD_lzk/runs") | |
| step = int(resume["step"]) | |
| restored = any(row.get("event") == "checkpoint_resume_passed" and row.get("step") == step | |
| for row in events) | |
| if restored: | |
| last_step = max(last_step, step) | |
| previous = os.path.relpath(source_run / "review.md", run_dir) | |
| videos = os.path.relpath(source_run / "checkpoint_probes" / f"step_{step:08d}" / "README.md", run_dir) | |
| resume_lines = [f"续训起点:**{step}**;状态恢复:**{'已验证' if restored else '待验证'}**。", | |
| f"[上次实验记录](<{previous}>) · [恢复点的预测视频](<{videos}>)。", | |
| "曲线只显示本次续训的数据;上次记录保留在原目录,不重复渲染已有视频。", ""] | |
| latest_event = str(events[-1].get("event", "无事件")) if events else "尚无训练事件" | |
| lines = ["# Third-party 训练监督记录", "", | |
| f"已记录的 optimizer step:**{last_step}**;最近事件:`{latest_event}`。", | |
| "这是一份自动生成的本地记录,无需实时盯终端;异常不等于已恢复训练。", "", | |
| "", "", | |
| "| 记录 | 内容 |", "|---|---|", | |
| "| [metrics.jsonl](metrics.jsonl) | 原始 loss、控制准确率、优化器指标 |", | |
| "| [events.jsonl](events.jsonl) | 保存 checkpoint、预测开始/完成/失败事件 |", | |
| "| [run_status.json](run_status.json) | 后台任务退出码与占卡恢复状态 |", "", | |
| "## Checkpoint 闭环复查", "", | |
| "训练集固定样本;表中视频都是模型预测,不是 GT。只能判断训练集拟合,不能声称测试集泛化。", "", | |
| "| Step | 状态 | 完成样本 | 控制准确率 | 背景非动作误触发率 | 记录/视频 |", | |
| "|---:|---|---:|---:|---:|---|"] | |
| lines[2:2] = resume_lines | |
| if not probes: | |
| lines.append("| — | 尚无完整 probe 记录 | — | — | — | — |") | |
| for entry in probes: | |
| report_path = _report_link(run_dir, entry["report"]) | |
| report = json.loads(report_path.read_text(encoding="utf-8")) | |
| aggregate = report.get("aggregate", {}) | |
| def display(value): | |
| return f"{value:.4f}" if isinstance(value, (int, float)) and math.isfinite(value) else "—" | |
| readme = report_path.with_name("README.md").relative_to(run_dir) | |
| lines.append(f"| {entry.get('global_step', '—')} | {report.get('status', 'unknown')} | " | |
| f"{len(report.get('samples', []))} | {display(aggregate.get('control_accuracy'))} | " | |
| f"{display(aggregate.get('background_only_false_motion_rate'))} | [查看]({readme}) |") | |
| failures = [event for event in events if any(word in str(event.get("event", "")) | |
| for word in ("failed", "nonfinite", "error"))] | |
| if failures: | |
| lines.extend(["", "## 已记录异常", ""]) | |
| for event in failures[-10:]: | |
| lines.append(f"- step {event.get('step', '—')}:`{event.get('event', 'unknown')}`;详见 events.jsonl。") | |
| partial = partial_metrics or partial_events or partial_probes | |
| if partial: | |
| lines.extend(["", "末尾一条日志仍在写入,本次暂略;未把缺失值补为 0。"]) | |
| lines.extend(["", "曲线只显示已记录的观测值;最多均匀抽样 5000 点/指标。完整数据保留在原始 JSONL。"]) | |
| _atomic_text(run_dir / "review.md", "\n".join(lines) + "\n") | |
| return {"metric_records": len(metrics), "event_records": len(events), "probe_records": len(probes), | |
| "last_recorded_step": last_step, "partial_trailing_record": partial, | |
| "curves": str(run_dir / "curves.svg"), "review": str(run_dir / "review.md")} | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("run_dir", type=Path) | |
| args = parser.parse_args() | |
| print(json.dumps(update_dashboard(args.run_dir), ensure_ascii=False, indent=2)) | |