File size: 11,534 Bytes
55e0599
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
#!/usr/bin/env python3
"""Build a local, dependency-free dashboard from durable training records."""
from __future__ import annotations

import argparse
import html
import json
import math
import os
from pathlib import Path
from typing import Any, Sequence

PROJECT_ROOT = Path(__file__).resolve().parents[1]
PANELS = (
    ("Training loss", ("train/loss", "train/loss_action", "train/loss_control")),
    ("Training control accuracy", ("train/control_accuracy", "train/control_accuracy_listen",
                                  "train/control_accuracy_motion", "train/control_accuracy_idle")),
    ("Auxiliary loss", ("train/loss_motion_duration", "train/loss_latest_instruction",
                        "train/loss_instruction_memory", "train/loss_smooth_weighted")),
    ("Learning rate", ("train/lr_act", "train/lr_control", "train/lr_vlm")),
    ("Gradient norm", ("train/grad_norm_act",)),
)
COLORS = ("#1368ce", "#df621e", "#268343", "#a144c4", "#b03a48")


def read_jsonl(path: Path) -> tuple[list[dict[str, Any]], bool]:
    if not path.exists():
        return [], False
    rows, partial = [], False
    with path.open(encoding="utf-8") as handle:
        for line in handle:
            if not line.strip():
                continue
            try:
                value = json.loads(line)
            except json.JSONDecodeError:
                if not line.endswith("\n"):
                    partial = True
                    break
                raise
            if not isinstance(value, dict):
                raise ValueError(f"Non-object training record: {path}")
            rows.append(value)
    return rows, partial


def series(rows: Sequence[dict[str, Any]], key: str, max_points: int = 5000) -> list[tuple[float, float]]:
    if max_points < 2:
        raise ValueError("At least two curve samples are required")
    result = []
    for row in rows:
        # Support unprefixed scalar logs without inventing missing observations.
        value = row.get(key, row.get(key.removeprefix("train/")))
        step = row.get("step")
        if (isinstance(value, (int, float)) and not isinstance(value, bool)
                and isinstance(step, (int, float)) and not isinstance(step, bool)
                and math.isfinite(value) and math.isfinite(step)):
            result.append((float(step), float(value)))
    if len(result) > max_points:
        result = [result[round(index * (len(result) - 1) / (max_points - 1))]
                  for index in range(max_points)]
    return result


def render_curves(rows: Sequence[dict[str, Any]], max_points: int = 5000) -> str:
    panels = []
    for title, keys in PANELS:
        curves = [(key, series(rows, key, max_points)) for key in keys]
        curves = [(key, points) for key, points in curves if points]
        if curves:
            panels.append((title, curves))
    height = 75 + max(1, len(panels)) * 250
    svg = [f'<svg xmlns="http://www.w3.org/2000/svg" width="1000" height="{height}" viewBox="0 0 1000 {height}">',
           '<rect width="100%" height="100%" fill="#fff"/>',
           '<g font-family="sans-serif" fill="#202b38">',
           '<text x="35" y="28" font-size="19">Third-party training: recorded scalars</text>',
           f'<text x="35" y="49" font-size="12">Optimizer step on x axis; missing values omitted; at most {max_points} sampled points per metric.</text>']
    if not panels:
        svg.append('<text x="35" y="105" font-size="15">No recorded training scalars yet (baseline probes may still be running).</text>')
    for panel_index, (title, curves) in enumerate(panels):
        top = 88 + panel_index * 250
        left, width, plot_height = 85, 865, 140
        all_points = [point for _, points in curves for point in points]
        xmin, xmax = min(p[0] for p in all_points), max(p[0] for p in all_points)
        ymin, ymax = min(p[1] for p in all_points), max(p[1] for p in all_points)
        if xmax == xmin:
            xmax = xmin + 1
        if ymax == ymin:
            padding = max(abs(ymin) * 0.05, 0.01)
            ymin, ymax = ymin - padding, ymax + padding
        svg.append(f'<text x="35" y="{top}" font-size="16">{html.escape(title)}</text>')
        plot_top = top + 15
        for tick in range(5):
            y = plot_top + tick * plot_height / 4
            value = ymax - tick * (ymax - ymin) / 4
            svg.extend([f'<line x1="{left}" y1="{y}" x2="{left + width}" y2="{y}" stroke="#e4e9ef"/>',
                        f'<text x="{left - 8}" y="{y + 4}" text-anchor="end" font-size="11">{value:.4g}</text>'])
        svg.append(f'<text x="{left}" y="{plot_top + plot_height + 16}" font-size="11">{xmin:.0f}</text>')
        svg.append(f'<text x="{left + width}" y="{plot_top + plot_height + 16}" text-anchor="end" font-size="11">{xmax:.0f}</text>')
        for index, (key, points) in enumerate(curves):
            color = COLORS[index % len(COLORS)]
            coords = [(left + (x - xmin) / (xmax - xmin) * width,
                       plot_top + (ymax - y) / (ymax - ymin) * plot_height) for x, y in points]
            joined = " ".join(f"{x:.2f},{y:.2f}" for x, y in coords)
            svg.append(f'<polyline points="{joined}" fill="none" stroke="{color}" stroke-width="1.6"/>')
            x, y = coords[-1]
            svg.append(f'<circle cx="{x:.2f}" cy="{y:.2f}" r="2.5" fill="{color}"/>')
            legend_x = 85 + (index % 2) * 440
            legend_y = plot_top + plot_height + 35 + (index // 2) * 16
            label = f"{key}: last={points[-1][1]:.5g}"
            svg.append(f'<text x="{legend_x}" y="{legend_y}" fill="{color}" font-size="11">{html.escape(label)}</text>')
    svg.extend(["</g>", "</svg>"])
    return "\n".join(svg) + "\n"


def _atomic_text(path: Path, value: str) -> None:
    temporary = path.with_suffix(path.suffix + ".tmp")
    temporary.write_text(value, encoding="utf-8")
    os.replace(temporary, path)


def _report_link(run_dir: Path, path: str) -> Path:
    report_path = Path(path)
    if not report_path.is_absolute():
        report_path = run_dir / report_path
    report_path = report_path.resolve()
    if not report_path.is_relative_to(run_dir):
        raise ValueError("Dashboard report link escapes this experiment directory")
    return report_path


def update_dashboard(run_dir: str | Path) -> dict[str, Any]:
    run_dir = Path(run_dir).resolve()
    if not run_dir.is_relative_to(PROJECT_ROOT.resolve() / "runs"):
        raise ValueError("Dashboard outputs must remain under HFD_lzk/runs")
    if not run_dir.is_dir():
        raise FileNotFoundError(run_dir)
    metrics, partial_metrics = read_jsonl(run_dir / "metrics.jsonl")
    events, partial_events = read_jsonl(run_dir / "events.jsonl")
    probes, partial_probes = read_jsonl(run_dir / "checkpoint_probes/history.jsonl")
    _atomic_text(run_dir / "curves.svg", render_curves(metrics))
    last_step = max((int(row["step"]) for row in metrics if isinstance(row.get("step"), (int, float))), default=0)
    resume_path = run_dir / "resume_identity.json"
    resume_lines = []
    if resume_path.is_file():
        resume = json.loads(resume_path.read_text(encoding="utf-8"))
        source_run = Path(resume["source_run"]).resolve()
        if (not source_run.is_relative_to(PROJECT_ROOT.resolve() / "runs")
                or source_run == run_dir):
            raise ValueError("Resume history must link a separate experiment under HFD_lzk/runs")
        step = int(resume["step"])
        restored = any(row.get("event") == "checkpoint_resume_passed" and row.get("step") == step
                       for row in events)
        if restored:
            last_step = max(last_step, step)
        previous = os.path.relpath(source_run / "review.md", run_dir)
        videos = os.path.relpath(source_run / "checkpoint_probes" / f"step_{step:08d}" / "README.md", run_dir)
        resume_lines = [f"续训起点:**{step}**;状态恢复:**{'已验证' if restored else '待验证'}**。",
                        f"[上次实验记录](<{previous}>) · [恢复点的预测视频](<{videos}>)。",
                        "曲线只显示本次续训的数据;上次记录保留在原目录,不重复渲染已有视频。", ""]
    latest_event = str(events[-1].get("event", "无事件")) if events else "尚无训练事件"
    lines = ["# Third-party 训练监督记录", "",
             f"已记录的 optimizer step:**{last_step}**;最近事件:`{latest_event}`。",
             "这是一份自动生成的本地记录,无需实时盯终端;异常不等于已恢复训练。", "",
             "![训练曲线](curves.svg)", "",
             "| 记录 | 内容 |", "|---|---|",
             "| [metrics.jsonl](metrics.jsonl) | 原始 loss、控制准确率、优化器指标 |",
             "| [events.jsonl](events.jsonl) | 保存 checkpoint、预测开始/完成/失败事件 |",
             "| [run_status.json](run_status.json) | 后台任务退出码与占卡恢复状态 |", "",
             "## Checkpoint 闭环复查", "",
             "训练集固定样本;表中视频都是模型预测,不是 GT。只能判断训练集拟合,不能声称测试集泛化。", "",
             "| Step | 状态 | 完成样本 | 控制准确率 | 背景非动作误触发率 | 记录/视频 |",
             "|---:|---|---:|---:|---:|---|"]
    lines[2:2] = resume_lines
    if not probes:
        lines.append("| — | 尚无完整 probe 记录 | — | — | — | — |")
    for entry in probes:
        report_path = _report_link(run_dir, entry["report"])
        report = json.loads(report_path.read_text(encoding="utf-8"))
        aggregate = report.get("aggregate", {})
        def display(value):
            return f"{value:.4f}" if isinstance(value, (int, float)) and math.isfinite(value) else "—"
        readme = report_path.with_name("README.md").relative_to(run_dir)
        lines.append(f"| {entry.get('global_step', '—')} | {report.get('status', 'unknown')} | "
                     f"{len(report.get('samples', []))} | {display(aggregate.get('control_accuracy'))} | "
                     f"{display(aggregate.get('background_only_false_motion_rate'))} | [查看]({readme}) |")
    failures = [event for event in events if any(word in str(event.get("event", ""))
                                               for word in ("failed", "nonfinite", "error"))]
    if failures:
        lines.extend(["", "## 已记录异常", ""])
        for event in failures[-10:]:
            lines.append(f"- step {event.get('step', '—')}:`{event.get('event', 'unknown')}`;详见 events.jsonl。")
    partial = partial_metrics or partial_events or partial_probes
    if partial:
        lines.extend(["", "末尾一条日志仍在写入,本次暂略;未把缺失值补为 0。"])
    lines.extend(["", "曲线只显示已记录的观测值;最多均匀抽样 5000 点/指标。完整数据保留在原始 JSONL。"])
    _atomic_text(run_dir / "review.md", "\n".join(lines) + "\n")
    return {"metric_records": len(metrics), "event_records": len(events), "probe_records": len(probes),
            "last_recorded_step": last_step, "partial_trailing_record": partial,
            "curves": str(run_dir / "curves.svg"), "review": str(run_dir / "review.md")}


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("run_dir", type=Path)
    args = parser.parse_args()
    print(json.dumps(update_dashboard(args.run_dir), ensure_ascii=False, indent=2))