code-HFD / src /psi /runtime /local_metrics.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
1.18 kB
"""Small rank-zero dashboard log; observation must never abort training."""
import json
import math
from pathlib import Path
import time
import warnings
_warned = False
def append_metrics(directory, step, metrics):
global _warned
sparse = ("train/control_accuracy" in metrics
or "train/token_control_accuracy" in metrics
or metrics.get("train/swa_qa_target_tokens", 0) > 0)
if step % 20 and not sparse and not any(k.startswith("eval/") for k in metrics):
return
row = {"step": int(step), "time": time.time()}
for key, value in metrics.items():
if key in {"train/swa_qa_token_accuracy", "train/swa_qa_loss"} and metrics.get("train/swa_qa_target_tokens", 0) <= 0:
continue
if isinstance(value, (float, int)) and math.isfinite(value):
row[key] = value
try:
with (Path(directory) / "dashboard_metrics.jsonl").open("a") as stream:
stream.write(json.dumps(row, allow_nan=False) + "\n")
except OSError as error:
if not _warned:
warnings.warn(f"Local dashboard metrics unavailable (training continues): {error}")
_warned = True