#!/usr/bin/env python3
"""Read-only, run-scoped curves and rendered videos; no external JS/CDN."""
import argparse
from functools import lru_cache
from collections import Counter
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import math
import mimetypes
from pathlib import Path
import re
import subprocess
import threading
import time
import unicodedata
from urllib.parse import parse_qs, unquote, urlsplit
WANDB_ROWS = []
WANDB_ERROR = ""
CURVE_KEYS = ("train/loss", "train/lr_act", "train/loss_action",
"train/control_accuracy", "train/swa_qa_token_accuracy",
"train/swa_qa_loss",
"eval/loss", "eval/control_accuracy",
"train/token_control_macro_f1", "train/token_false_transition_rate",
"train/token_flicker_rate", "eval/token_control_macro_f1")
def answer_tokens(text):
# A transparent lexical proxy, not a semantic or speech-quality judge.
text = unicodedata.normalize("NFKC", str(text)).lower()
return re.findall(r"[\u4e00-\u9fff]|[^\W_]+", re.sub(r"([\u4e00-\u9fff])", r" \1 ", text))
def qa_case(metric, directory):
"""Score actual answer triggers against labeled reply windows, never GT audio.
One matched trigger per contiguous positive interval; further triggers are
extras. Content uses the FIRST on-time answer, not best-of-many. Multiple
questions without per-event references get timing scores only.
"""
if not metric.get("events") or "overlay" not in metric:
return None
events = sorted(metric["events"], key=lambda event: event["seconds"])
if any("reply_gt" not in event for event in events):
return None
intervals = []
previous = False
for event in events:
positive = bool(event["reply_gt"])
if positive:
if not previous:
intervals.append([event["seconds"], event["seconds"]])
else:
intervals[-1][1] = event["seconds"]
previous = positive
matched, extras, audio = {}, 0, 0
answers = sorted(metric.get("answers", []), key=lambda answer: answer.get("seconds", float("inf")))
for answer in answers:
seconds = answer.get("seconds")
index = next((i for i, (start, end) in enumerate(intervals)
if isinstance(seconds, (int, float)) and start-1e-6 <= seconds <= end+1e-6), None)
if index is not None and index not in matched:
matched[index] = answer
else:
extras += 1
path = (directory / answer.get("audio_path", "")).resolve()
if path.is_relative_to(directory.resolve()) and path.is_file() and path.stat().st_size > 44:
audio += 1
overlay = metric["overlay"]
result = dict(question=overlay.get("question", ""), reference=overlay.get("answer", ""),
speech_failures=len(metric.get('speech_failures',[])),
predictions=[dict(seconds=a.get("seconds"), text=a.get("text", "")) for a in answers],
questions=len(intervals), on_time=len(matched), extra_answers=extras,
answers=len(answers), audio_files=audio, source_id=metric.get("source_id", ""),
asset_id=overlay.get("asset_id", ""))
# Missing/late answer is a failure, not omitted from content denominator.
target = answer_tokens(result["reference"])
if len(intervals) == 1 and target:
prediction = answer_tokens(matched.get(0, {}).get("text", ""))
overlap = sum((Counter(prediction) & Counter(target)).values())
result.update(text_em=float(prediction == target),
text_f1=2*overlap/(len(prediction)+len(target)))
return result
def qa_summary(cases):
grouped = {}
for case in cases:
grouped.setdefault((case["step"], case["split"]), []).append(case)
rows = []
for (step, split), items in sorted(grouped.items()):
total = sum(item["questions"] for item in items)
row = dict(step=step, case=split, sessions=len(items), questions=total,
extra_answers_per_session=sum(item["extra_answers"] for item in items)/len(items))
if total:
row["reply_recall"] = sum(item["on_time"] for item in items)/total
for key in ("text_em", "text_f1"):
values = [item[key] for item in items if key in item]
if values:
row[key] = sum(values)/len(values)
row["text_samples"] = len(values)
rows.append(row)
return rows
class MetricHistory:
"""Read the complete history once, then only appended complete JSONL lines."""
def __init__(self, path):
self.path, self.identity, self.offset, self.rows = path, None, 0, []
def read(self):
try:
with self.path.open("rb") as stream:
import os
stat = os.fstat(stream.fileno())
identity = (stat.st_dev, stat.st_ino)
if identity != self.identity or stat.st_size < self.offset:
self.rows, self.offset = [], 0
self.identity = identity
stream.seek(self.offset)
while True:
line = stream.readline()
if not line.endswith(b"\n"):
break # Retry incomplete final line next refresh.
self.offset = stream.tell()
try:
row = json.loads(line)
if not isinstance(row, dict) or not isinstance(row.get("step"), (int, float)):
continue
if not math.isfinite(row["step"]):
continue
self.rows.append({key: value for key, value in row.items()
if key in (*CURVE_KEYS, "step", "time", "train/swa_qa_target_tokens")
and isinstance(value, (int, float)) and math.isfinite(value)})
if self.rows[-1].get("train/swa_qa_target_tokens", 0) <= 0:
for key in ("train/swa_qa_loss", "train/swa_qa_token_accuracy"):
self.rows[-1].pop(key, None)
except (ValueError, UnicodeDecodeError):
continue
except OSError:
pass
return self.rows
def compact_curves(rows, bins=400):
"""Bound display size per metric, retaining endpoints and each bin's extrema.
Sample metrics independently so sparse eval/control points are not discarded
by the much denser loss stream. Does not alter the training log on disk.
"""
selected = set()
for key in CURVE_KEYS:
indices = [i for i, row in enumerate(rows) if key in row]
if len(indices) <= bins * 2:
selected.update(indices)
continue
selected.update((indices[0], indices[-1]))
for start in range(bins):
bucket = indices[len(indices)*start//bins:len(indices)*(start+1)//bins]
selected.add(min(bucket, key=lambda i: rows[i][key]))
selected.add(max(bucket, key=lambda i: rows[i][key]))
if rows:
selected.update((0, len(rows)-1))
return [rows[i] for i in sorted(selected)]
def curve_view(data, mode="global", window=50000):
latest = max((r["step"] for name in ("rows", "semantic_rows", "validation_rows")
for r in data[name]), default=0)
first = min((r["step"] for r in data["rows"]), default=0)
start = max(0, latest-window) if mode == "local" else first if mode == "current" else 0
result = dict(data, step_range=[start, max(start+1, latest)], view=mode,
first_step=first, latest=data["rows"][-1] if data["rows"] else {})
for name in ("rows", "semantic_rows", "validation_rows"):
rows = sorted((r for r in data[name] if start <= r["step"] <= latest), key=lambda r: r["step"])
result[name] = rows if name == "validation_rows" else compact_curves(rows)
result["qa_rows"] = [r for r in data.get("qa_rows", []) if start <= r["step"] <= latest]
cases = [r for r in data.get("qa_cases", []) if start <= r["step"] <= latest]
result["qa_cases"] = [r for r in cases if r["step"] == max(
x["step"] for x in cases if x["split"] == r["split"])]
return result
def poll_wandb(path):
"""Read sparse semantic metrics off-thread; never delays the training job."""
global WANDB_ROWS, WANDB_ERROR
import wandb
while True:
try:
run = wandb.Api(timeout=15).run(path)
collected = {}
for keys in (["train/control_accuracy"],
["train/swa_qa_loss", "train/swa_qa_token_accuracy", "train/swa_qa_target_tokens"]):
for row in run.history(samples=1000, keys=keys, pandas=False):
if "_step" not in row:
continue
if "train/swa_qa_target_tokens" in row and row["train/swa_qa_target_tokens"] <= 0:
continue
step = int(row.pop("_step"))
collected.setdefault(step, {"step": step}).update(row)
WANDB_ROWS = list(collected.values())
WANDB_ERROR = ""
except Exception as error:
WANDB_ERROR = type(error).__name__
time.sleep(60)
PAGE = r"""