Upload folder using huggingface_hub

0d00bbe verified 6 months ago

7.55 kB

"""
Log Visualizer for HiF8 QAT Training
Usage:
    python log_visualizer.py baseline.log exp1.log exp2.log ...
Produces 3 plots:
  1. Loss curves for all runs
  2. Learning rate curves for all runs
  3. Absolute Percentage Error of each experiment vs baseline (per step)
"""

import re
import sys
from pathlib import Path

import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker


# ── Log parsing ──────────────────────────────────────────────────────────────

STEP_RE = re.compile(
    r"step=(\d+)/\d+"
    r".*?loss=([\d.]+)"
    r".*?lr=([\d.e+\-]+)"
)


def parse_log(path: str) -> dict:
    """Return {step: int, loss: float, lr: float} records from a training log."""
    records = []
    with open(path, "r", errors="replace") as f:
        for line in f:
            m = STEP_RE.search(line)
            if m:
                records.append({
                    "step": int(m.group(1)),
                    "loss": float(m.group(2)),
                    "lr":   float(m.group(3)),
                })
    if not records:
        raise ValueError(f"No training step lines found in: {path}")
    return records


def to_arrays(records):
    steps = [r["step"] for r in records]
    losses = [r["loss"] for r in records]
    lrs    = [r["lr"]   for r in records]
    return steps, losses, lrs


# ── Plotting helpers ──────────────────────────────────────────────────────────

COLORS = [
    "#1f77b4",  # blue   – baseline
    "#ff7f0e",  # orange
    "#2ca02c",  # green
    "#d62728",  # red
    "#9467bd",  # purple
    "#8c564b",  # brown
    "#e377c2",  # pink
    "#7f7f7f",  # gray
]

BASELINE_STYLE = dict(linewidth=2.0, linestyle="-")
EXP_STYLE      = dict(linewidth=1.5, linestyle="--")


def short_name(path: str) -> str:
    return Path(path).stem


# ── Main ──────────────────────────────────────────────────────────────────────

def main():
    if len(sys.argv) < 2:
        print(__doc__)
        sys.exit(1)

    log_paths = sys.argv[1:]
    names     = [short_name(p) for p in log_paths]

    print(f"Parsing {len(log_paths)} log file(s)...")
    all_records = []
    for p in log_paths:
        recs = parse_log(p)
        all_records.append(recs)
        print(f"  {Path(p).name}: {len(recs)} steps")

    baseline_records = all_records[0]
    baseline_steps, baseline_losses, baseline_lrs = to_arrays(baseline_records)
    baseline_step_set = {r["step"]: r for r in baseline_records}

    fig, axes = plt.subplots(3, 1, figsize=(12, 14))
    fig.suptitle("HiF8 QAT Training Comparison", fontsize=14, fontweight="bold")

    ax_loss, ax_lr, ax_ape = axes

    # ── Plot 1: Loss curves ───────────────────────────────────────────────────
    ax_loss.set_title("Training Loss")
    ax_loss.set_xlabel("Step")
    ax_loss.set_ylabel("Loss")

    for i, (recs, name) in enumerate(zip(all_records, names)):
        steps, losses, _ = to_arrays(recs)
        style = BASELINE_STYLE if i == 0 else EXP_STYLE
        label = f"{name} (baseline)" if i == 0 else name
        ax_loss.plot(steps, losses, color=COLORS[i % len(COLORS)],
                     label=label, **style)

    ax_loss.legend(fontsize=8)
    ax_loss.grid(True, alpha=0.3)

    # ── Plot 2: Learning rate curves ──────────────────────────────────────────
    ax_lr.set_title("Learning Rate")
    ax_lr.set_xlabel("Step")
    ax_lr.set_ylabel("LR")

    for i, (recs, name) in enumerate(zip(all_records, names)):
        steps, _, lrs = to_arrays(recs)
        style = BASELINE_STYLE if i == 0 else EXP_STYLE
        label = f"{name} (baseline)" if i == 0 else name
        ax_lr.plot(steps, lrs, color=COLORS[i % len(COLORS)],
                   label=label, **style)

    ax_lr.yaxis.set_major_formatter(ticker.ScalarFormatter(useMathText=True))
    ax_lr.ticklabel_format(style="sci", axis="y", scilimits=(0, 0))
    ax_lr.legend(fontsize=8)
    ax_lr.grid(True, alpha=0.3)

    # ── Plot 3: Absolute Percentage Error vs baseline ─────────────────────────
    ax_ape.set_title("Loss Absolute Percentage Error vs Baseline")
    ax_ape.set_xlabel("Step")
    ax_ape.set_ylabel("APE (%)")

    has_exp = False
    for i, (recs, name) in enumerate(zip(all_records, names)):
        if i == 0:
            continue  # skip baseline itself

        ape_steps, ape_vals = [], []
        for r in recs:
            s = r["step"]
            if s in baseline_step_set:
                base_loss = baseline_step_set[s]["loss"]
                if base_loss != 0:
                    ape = abs(r["loss"] - base_loss) / abs(base_loss) * 100.0
                    ape_steps.append(s)
                    ape_vals.append(ape)

        if ape_steps:
            ax_ape.plot(ape_steps, ape_vals,
                        color=COLORS[i % len(COLORS)],
                        label=name, **EXP_STYLE)
            has_exp = True

    if not has_exp:
        ax_ape.text(0.5, 0.5, "No experiment logs provided\n(need at least 2 log files)",
                    ha="center", va="center", transform=ax_ape.transAxes, fontsize=11)
    else:
        ax_ape.axhline(0, color="black", linewidth=0.8, linestyle=":")
        ax_ape.legend(fontsize=8)

    ax_ape.grid(True, alpha=0.3)

    # ── Print average APE over all steps and last 1000 steps vs baseline ────────
    def _compute_ape(recs, step_threshold=None):
        """返回与 baseline 重叠步的 APE 列表，step_threshold 为 None 时取全部步。"""
        vals = []
        for r in recs:
            if step_threshold is not None and r["step"] <= step_threshold:
                continue
            s = r["step"]
            if s in baseline_step_set:
                base_loss = baseline_step_set[s]["loss"]
                if base_loss != 0:
                    vals.append(abs(r["loss"] - base_loss) / abs(base_loss) * 100.0)
        return vals

    for label, threshold_fn in [
        ("all steps",    lambda recs: None),
        ("last 1000 steps", lambda recs: max(r["step"] for r in recs) - 1000),
    ]:
        print(f"\nAverage APE vs baseline ({label}):")
        for i, (recs, name) in enumerate(zip(all_records, names)):
            if i == 0:
                continue
            ape_vals = _compute_ape(recs, threshold_fn(recs))
            if ape_vals:
                print(f"  {name}: {sum(ape_vals)/len(ape_vals):.4f}%  (n={len(ape_vals)})")
            else:
                print(f"  {name}: N/A (no overlapping steps with baseline)")

    # ── Save ──────────────────────────────────────────────────────────────────
    out_dir  = Path(__file__).parent
    out_path = out_dir / "training_comparison.png"
    plt.tight_layout()
    fig.savefig(out_path, dpi=150)
    print(f"\nSaved: {out_path}")


if __name__ == "__main__":
    main()

	"""
	Log Visualizer for HiF8 QAT Training

	Usage:
	python log_visualizer.py baseline.log exp1.log exp2.log ...

	Produces 3 plots:
	1. Loss curves for all runs
	2. Learning rate curves for all runs
	3. Absolute Percentage Error of each experiment vs baseline (per step)
	"""

	import re
	import sys
	from pathlib import Path

	import matplotlib
	matplotlib.use("Agg")
	import matplotlib.pyplot as plt
	import matplotlib.ticker as ticker


	# ── Log parsing ──────────────────────────────────────────────────────────────

	STEP_RE = re.compile(
	r"step=(\d+)/\d+"
	r".*?loss=([\d.]+)"
	r".*?lr=([\d.e+\-]+)"
	)


	def parse_log(path: str) -> dict:
	"""Return {step: int, loss: float, lr: float} records from a training log."""
	records = []
	with open(path, "r", errors="replace") as f:
	for line in f:
	m = STEP_RE.search(line)
	if m:
	records.append({
	"step": int(m.group(1)),
	"loss": float(m.group(2)),
	"lr": float(m.group(3)),
	})
	if not records:
	raise ValueError(f"No training step lines found in: {path}")
	return records


	def to_arrays(records):
	steps = [r["step"] for r in records]
	losses = [r["loss"] for r in records]
	lrs = [r["lr"] for r in records]
	return steps, losses, lrs


	# ── Plotting helpers ──────────────────────────────────────────────────────────

	COLORS = [
	"#1f77b4", # blue – baseline
	"#ff7f0e", # orange
	"#2ca02c", # green
	"#d62728", # red
	"#9467bd", # purple
	"#8c564b", # brown
	"#e377c2", # pink
	"#7f7f7f", # gray
	]

	BASELINE_STYLE = dict(linewidth=2.0, linestyle="-")
	EXP_STYLE = dict(linewidth=1.5, linestyle="--")


	def short_name(path: str) -> str:
	return Path(path).stem


	# ── Main ──────────────────────────────────────────────────────────────────────

	def main():
	if len(sys.argv) < 2:
	print(__doc__)
	sys.exit(1)

	log_paths = sys.argv[1:]
	names = [short_name(p) for p in log_paths]

	print(f"Parsing {len(log_paths)} log file(s)...")
	all_records = []
	for p in log_paths:
	recs = parse_log(p)
	all_records.append(recs)
	print(f" {Path(p).name}: {len(recs)} steps")

	baseline_records = all_records[0]
	baseline_steps, baseline_losses, baseline_lrs = to_arrays(baseline_records)
	baseline_step_set = {r["step"]: r for r in baseline_records}

	fig, axes = plt.subplots(3, 1, figsize=(12, 14))
	fig.suptitle("HiF8 QAT Training Comparison", fontsize=14, fontweight="bold")

	ax_loss, ax_lr, ax_ape = axes

	# ── Plot 1: Loss curves ───────────────────────────────────────────────────
	ax_loss.set_title("Training Loss")
	ax_loss.set_xlabel("Step")
	ax_loss.set_ylabel("Loss")

	for i, (recs, name) in enumerate(zip(all_records, names)):
	steps, losses, _ = to_arrays(recs)
	style = BASELINE_STYLE if i == 0 else EXP_STYLE
	label = f"{name} (baseline)" if i == 0 else name
	ax_loss.plot(steps, losses, color=COLORS[i % len(COLORS)],
	label=label, **style)

	ax_loss.legend(fontsize=8)
	ax_loss.grid(True, alpha=0.3)

	# ── Plot 2: Learning rate curves ──────────────────────────────────────────
	ax_lr.set_title("Learning Rate")
	ax_lr.set_xlabel("Step")
	ax_lr.set_ylabel("LR")

	for i, (recs, name) in enumerate(zip(all_records, names)):
	steps, _, lrs = to_arrays(recs)
	style = BASELINE_STYLE if i == 0 else EXP_STYLE
	label = f"{name} (baseline)" if i == 0 else name
	ax_lr.plot(steps, lrs, color=COLORS[i % len(COLORS)],
	label=label, **style)

	ax_lr.yaxis.set_major_formatter(ticker.ScalarFormatter(useMathText=True))
	ax_lr.ticklabel_format(style="sci", axis="y", scilimits=(0, 0))
	ax_lr.legend(fontsize=8)
	ax_lr.grid(True, alpha=0.3)

	# ── Plot 3: Absolute Percentage Error vs baseline ─────────────────────────
	ax_ape.set_title("Loss Absolute Percentage Error vs Baseline")
	ax_ape.set_xlabel("Step")
	ax_ape.set_ylabel("APE (%)")

	has_exp = False
	for i, (recs, name) in enumerate(zip(all_records, names)):
	if i == 0:
	continue # skip baseline itself

	ape_steps, ape_vals = [], []
	for r in recs:
	s = r["step"]
	if s in baseline_step_set:
	base_loss = baseline_step_set[s]["loss"]
	if base_loss != 0:
	ape = abs(r["loss"] - base_loss) / abs(base_loss) * 100.0
	ape_steps.append(s)
	ape_vals.append(ape)

	if ape_steps:
	ax_ape.plot(ape_steps, ape_vals,
	color=COLORS[i % len(COLORS)],
	label=name, **EXP_STYLE)
	has_exp = True

	if not has_exp:
	ax_ape.text(0.5, 0.5, "No experiment logs provided\n(need at least 2 log files)",
	ha="center", va="center", transform=ax_ape.transAxes, fontsize=11)
	else:
	ax_ape.axhline(0, color="black", linewidth=0.8, linestyle=":")
	ax_ape.legend(fontsize=8)

	ax_ape.grid(True, alpha=0.3)

	# ── Print average APE over all steps and last 1000 steps vs baseline ────────
	def _compute_ape(recs, step_threshold=None):
	"""返回与 baseline 重叠步的 APE 列表，step_threshold 为 None 时取全部步。"""
	vals = []
	for r in recs:
	if step_threshold is not None and r["step"] <= step_threshold:
	continue
	s = r["step"]
	if s in baseline_step_set:
	base_loss = baseline_step_set[s]["loss"]
	if base_loss != 0:
	vals.append(abs(r["loss"] - base_loss) / abs(base_loss) * 100.0)
	return vals

	for label, threshold_fn in [
	("all steps", lambda recs: None),
	("last 1000 steps", lambda recs: max(r["step"] for r in recs) - 1000),
	]:
	print(f"\nAverage APE vs baseline ({label}):")
	for i, (recs, name) in enumerate(zip(all_records, names)):
	if i == 0:
	continue
	ape_vals = _compute_ape(recs, threshold_fn(recs))
	if ape_vals:
	print(f" {name}: {sum(ape_vals)/len(ape_vals):.4f}% (n={len(ape_vals)})")
	else:
	print(f" {name}: N/A (no overlapping steps with baseline)")

	# ── Save ──────────────────────────────────────────────────────────────────
	out_dir = Path(__file__).parent
	out_path = out_dir / "training_comparison.png"
	plt.tight_layout()
	fig.savefig(out_path, dpi=150)
	print(f"\nSaved: {out_path}")


	if __name__ == "__main__":
	main()