Refresh presentation bundle (RUN_LOG 25, README 5.11, builder --colab) and add the Colab notebook
07cdb81 verified 15 days ago
105 kB
#!/usr/bin/env python
"""Build FinalProject_PhishingDetector.ipynb with nbformat (idempotent: same cells every run).
Two variants are built from ONE cell list: every cell is tagged for the local notebook, the Colab
notebook or both (``variant("local")`` / ``variant("colab")`` / ``variant(None)`` switch the tag for
the cells appended after them), and ``build(colab=...)`` keeps the matching cells. The LOCAL variant
(default) is unchanged: it reads the project files on the workstation. The COLAB variant
(``--colab``, "notebook = narrative, Space = compute") downloads the ``presentation/`` bundle from
the model repo on the Hub (reports, curves, metrics, configs, source: no weights, no data), points
``ROOT`` at it so the shared cells read the same relative paths, and sends every model computation to
the deployed Space through ``gradio_client`` / ``gr.load``.
The notebook mirrors the instructor's reference (reference/FashionMNIST_Gradio_Full.ipynb): a
title cell, then EXACTLY his nine bold "**Step N: ...**" markdown headers in his order, each
followed by a short explanation and code cells that load OUR artifacts, then "Results" and
"Challenges and what we learned". Every code cell runs on the CPU today: the first cell sets
``USE_GPU = False`` (hides the GPU before torch is imported, so opening the notebook in JupyterLab
cannot touch the live training run), and artifacts that do not exist yet are reported by the
``artifact()`` helper as "not available yet (produced by ...)" instead of raising. Scratch files
go to ``<project>/.notebook_tmp`` and are removed by the last cell.
    .venv/bin/python scripts/build_notebook.py            # write the notebook (no outputs)
    CUDA_VISIBLE_DEVICES="" .venv/bin/python -m jupyter nbconvert --to notebook --execute \
        --inplace --ExecutePreprocessor.timeout=600 FinalProject_PhishingDetector.ipynb
    .venv/bin/python scripts/build_notebook.py --colab    # -> FinalProject_PhishingDetector_Colab.ipynb
    .venv/bin/python scripts/build_notebook.py --out /path/to/other.ipynb [--colab]
"""

from __future__ import annotations

import argparse
import sys
from pathlib import Path

import nbformat
from nbformat.v4 import new_code_cell, new_markdown_cell, new_notebook

ROOT = Path(__file__).resolve().parent.parent
OUT = ROOT / "FinalProject_PhishingDetector.ipynb"
OUT_COLAB = ROOT / "FinalProject_PhishingDetector_Colab.ipynb"

# (kind, source, variant): variant None = both notebooks, "local" = workstation notebook only, "colab" = Colab notebook only.
CELLS: list[tuple[str, str, str | None]] = []
_VARIANT: str | None = None


def variant(tag: str | None) -> None:
    """Tag every cell appended after this call: "local", "colab" or None (shared by both variants)."""
    global _VARIANT
    assert tag in (None, "local", "colab"), tag
    _VARIANT = tag


def md(text: str) -> None:
    CELLS.append(("markdown", text.strip("\n"), _VARIANT))


def code(text: str) -> None:
    CELLS.append(("code", text.strip("\n"), _VARIANT))


# =============================================================================== title
md("""
# BITS Phishing Detector: a from-scratch 337M-parameter email classifier
**Nathan King, Build IT Systems LLC (BITS)**
Skillspire AI & Machine Learning course, final project. Presented in class, late October / early November 2026.
**Abstract.** Small-business inboxes are triaged by people, not security teams, so this project builds a
*defensive* phishing / spam triage model that a person can paste an email into. It is a text **classifier**
(three classes: **legitimate**, **phishing**, **spam**) and nothing in it is pretrained or borrowed: the model is a
24-layer BERT-style encoder written in plain PyTorch (`src/model.py`, 337M parameters), the tokenizer is a
byte-level BPE trained on our own corpus, and the weights come from two training phases on one RTX 3090 Ti:
masked-language-model pretraining on deduplicated Enron email, then fine-tuning on public labelled corpora
with a partial-label loss (the TREC / CEAS positives never separated spam from phishing, so they train as
"unwanted"). The clean TREC-06 corpus is held out as the binary test set and a separate clean-provenance
three-way holdout gives the 3x3 confusion matrix. The classifier is exported to safetensors and deployed as a
Gradio app on a Hugging Face Space (ZeroGPU). This notebook follows the nine steps of the course reference
notebook and loads the real artifacts of the project; anything that is still being produced while the GPU
trains is reported as "not available yet" together with the command that produces it.
""")

variant("colab")
md("""
**Colab edition.** This copy of the notebook runs with none of the project files present: the narrative, tables and
curves are read from the `presentation/` bundle inside the model repo on the Hugging Face Hub (reports, metrics, curves,
configs and source files, mirrored at the project's relative paths: no weights, no datasets), and every computation that
needs the 337M-parameter model is delegated to the deployed Space on ZeroGPU. Nothing trains here and no weights are
downloaded. While the two repos are private, add a Colab secret named `HF_TOKEN` (notebook access switched on). Make it a
**fine-grained read-only token scoped to the two repos** (`BuildITSystems/phishing-detector-encoder` and
`BuildITSystems/phishing-detector-demo`): `gr.load` and `gradio_client` forward the token to the Space with every proxied
call, so it should never be a write token. Once the repos are public no token is needed.
""")
code("""
# The Hub identifiers the notebook depends on (change them here if the repos are renamed or forked).
MODEL_REPO = "BuildITSystems/phishing-detector-encoder"                  # model repo: bf16 weights + the presentation/ bundle
SPACE = "BuildITSystems/phishing-detector-demo"                          # ZeroGPU Space running app/app.py (weights bundled)
SPACE_HOST = "https://builditsystems-phishing-detector-demo.hf.space"    # the Space's embed host (iframe; needs a public Space)
PRESENTATION_PREFIX = "presentation"                                     # folder in MODEL_REPO mirroring the project's relative paths
""")
variant("local")

# =============================================================================== Step 1
md("""
**Step 1: Install Dependencies & Setup Device**
Nothing is installed here: the project runs in a pinned virtual environment (`requirements-train.txt`, every pin
verified to ship a Python 3.14 wheel). This cell imports the libraries, sets up device-agnostic code, seeds the
RNGs and prints the runtime facts that drove the VRAM planning. The GPU is currently occupied by the pretraining
run, so the notebook pins itself to the CPU (`USE_GPU = False` hides the GPU from this kernel before torch is
imported; the executed copy was additionally run with `CUDA_VISIBLE_DEVICES=""`). Flip the flag only once the
training queue has finished.
""")
code("""
import os, sys, json, time, inspect, hashlib, subprocess, shutil
from pathlib import Path
from timeit import default_timer as timer
# Keep this kernel off the GPU while the pretraining / fine-tuning queue owns it (20+ GiB of 24 GiB in use):
# the variable must be set before torch is imported anywhere in this process (app/app.py in Step 7 included).
USE_GPU = False
if not USE_GPU:
    os.environ["CUDA_VISIBLE_DEVICES"] = ""
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from IPython.display import display, Image, Markdown
T_NOTEBOOK_START = timer()
pd.set_option("display.width", 160); pd.set_option("display.max_columns", 40)
# Project root = the directory that holds config.yaml (works from the notebook's folder or a subfolder).
ROOT = Path.cwd().resolve()
while not (ROOT / "config.yaml").exists() and ROOT.parent != ROOT:
    ROOT = ROOT.parent
sys.path.insert(0, str(ROOT / "src"))
import common
CFG = common.load_config(ROOT / "config.yaml")
PRETRAIN_RUN = "large-24L"          # RUN_NAME in scripts/launch_pretrain.sh
print("project root:", ROOT)
MISSING: list[tuple[str, str]] = []
def artifact(rel: str, produced_by: str):
    \"\"\"Path to a project artifact, or None (with a one-line notice) if it does not exist yet.\"\"\"
    p = ROOT / rel
    if p.exists():
        return p
    print(f"not available yet — {rel} (produced by: {produced_by})")
    MISSING.append((rel, produced_by))
    return None
def show(df, title=None, fmt=None):
    if title:
        display(Markdown(f"**{title}**"))
    display(df.style.format(fmt) if fmt else df)
""")
code("""
import torch
import torch.nn as nn
from torch.utils.data import DataLoader
# Setup device-agnostic code (the training scripts do the same; here the GPU is busy pretraining)
device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"Using device: {device}  (USE_GPU={USE_GPU}, CUDA_VISIBLE_DEVICES={os.environ.get('CUDA_VISIBLE_DEVICES')!r})")
# Safety net: while a training session owns the GPU this notebook must not see it (a 337M fp32 model + the live
# pretraining run would exceed 24 GiB and crash the trainer).
def tmux_session_alive(name: str) -> bool:
    try:
        return subprocess.run(["tmux", "has-session", "-t", name], capture_output=True).returncode == 0
    except FileNotFoundError:
        return False
TRAINING_LIVE = [s for s in ("phish-pretrain", "phish-finetune") if tmux_session_alive(s)]
print("live training tmux sessions:", TRAINING_LIVE or "none")
assert not (TRAINING_LIVE and device == "cuda"), f"GPU visible while {TRAINING_LIVE} is training: set USE_GPU = False and restart the kernel"
# Set seeds for reproducibility (random, numpy, torch CPU + CUDA)
common.set_seed(CFG["seed"])
torch.manual_seed(CFG["seed"])
if torch.cuda.is_available():
    torch.cuda.manual_seed(CFG["seed"])
print("\\nRuntime facts (common.print_device_info):")
DEVICE_INFO = common.print_device_info()
print(f"\\nCPU threads for torch: {torch.get_num_threads()}")
""")
variant("colab")
md("""
**Step 1: Install Dependencies & Setup Device**
Colab preinstalls torch, pandas and matplotlib; the two pins installed here are `gradio` 6.29.1 (the Space's own SDK
version, so `gr.load` speaks the same protocol) and `huggingface_hub` 1.33.0 (the version the project's upload script was
written against). The token is resolved in this order and only its *source* is printed, never its value: Colab secret
`HF_TOKEN` -> environment variable `HF_TOKEN` -> the `hf auth login` cache -> none (public repos still work). Then the
`presentation/` bundle (a few MB) is downloaded from the model repo and `ROOT` points at it, so every later cell reads the
same relative paths as the local notebook; a file that is not in the bundle is reported by `artifact()` as "not available
yet" with the command that produces it, exactly as on the workstation. "Device" here means the remote one: the Space's
runtime is reported next to whatever accelerator this Colab runtime happens to have, which the model never uses.
""")
code("""
import os, sys, json, time, re, tempfile
from pathlib import Path
from timeit import default_timer as timer
IN_COLAB = "google.colab" in sys.modules
if IN_COLAB:
    %pip install -q gradio==6.29.1 huggingface_hub==1.33.0
else:
    print("not running in Colab: no pip install, the already-installed gradio / huggingface_hub are used")
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")      # quiet downloads (must precede the huggingface_hub import)
os.environ.setdefault("GRADIO_ANALYTICS_ENABLED", "False")      # gradio / gradio_client telemetry off
os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")           # huggingface_hub telemetry off
import warnings
warnings.filterwarnings("ignore", message="IProgress not found")   # tqdm's ipywidgets nag in kernels without ipywidgets (Colab has it)
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import yaml
import gradio
import huggingface_hub
from huggingface_hub import HfApi, snapshot_download
from IPython.display import display, Image, Markdown
T_NOTEBOOK_START = timer()
pd.set_option("display.width", 160); pd.set_option("display.max_columns", 40)
print(f"python {sys.version.split()[0]}  gradio {gradio.__version__}  huggingface_hub {huggingface_hub.__version__}  "
      f"pandas {pd.__version__}  running in Colab: {IN_COLAB}")
# Token: Colab secret HF_TOKEN -> environment variable HF_TOKEN -> `hf auth login` cache -> none. Only the SOURCE is printed.
# The token is forwarded to the Space by gr.load / gradio_client, so it should be a read-only token scoped to the two repos.
# Outside Colab the login cache usually holds the workstation's WRITE token, so that fallback is opt-in there:
# set NOTEBOOK_ALLOW_LOGIN_CACHE=1 (or export HF_TOKEN=<read token>) before running the headless test.
def resolve_token():
    if IN_COLAB:
        try:
            from google.colab import userdata
            if userdata.get("HF_TOKEN"):
                return userdata.get("HF_TOKEN"), "Colab secret HF_TOKEN"
        except Exception:        # the secret is not defined, or notebook access is off in the Secrets panel
            pass
    if os.environ.get("HF_TOKEN"):
        return os.environ["HF_TOKEN"], "environment variable HF_TOKEN"
    if (IN_COLAB or os.environ.get("NOTEBOOK_ALLOW_LOGIN_CACHE")) and huggingface_hub.get_token():
        return huggingface_hub.get_token(), "huggingface_hub login cache (hf auth login)"
    return None, "none (anonymous: works only once the repos are public)"
TOKEN, TOKEN_SOURCE = resolve_token()
print("Hugging Face token source:", TOKEN_SOURCE)
api = HfApi(token=TOKEN)
""")
code("""
# The presentation/ bundle: reports, curves, metrics, configs and source files mirrored at the project's relative paths (no
# weights, no datasets). ROOT points at it, so every later cell reads the same paths as the local notebook.
BUNDLE_DIR = Path("/content/phishing_bundle") if IN_COLAB else Path(tempfile.gettempdir()) / "phishing_bundle"
try:
    snap = Path(snapshot_download(MODEL_REPO, repo_type="model", allow_patterns=[f"{PRESENTATION_PREFIX}/*", "config.json"],
                                  token=TOKEN, local_dir=BUNDLE_DIR))
except Exception as e:
    raise RuntimeError(f"cannot download the bundle from {MODEL_REPO} ({type(e).__name__}): while the repo is private this notebook "
                       "needs a read token for the BuildITSystems account (Colab: Secrets panel -> HF_TOKEN, notebook access on)") from e
ROOT = snap / PRESENTATION_PREFIX
EXPORT_CONFIG = json.loads((snap / "config.json").read_text())     # the deployed model's config.json: labels, dials, test metrics
bundle_files = sorted(p for p in ROOT.rglob("*") if p.is_file())
print(f"bundle: {len(bundle_files)} files, {sum(p.stat().st_size for p in bundle_files) / 2**20:.1f} MiB under {ROOT}")
PRETRAIN_RUN = "large-24L"          # RUN_NAME in scripts/launch_pretrain.sh
CFG = yaml.safe_load((ROOT / "config.yaml").read_text())     # the project config; its paths stay relative (nothing resolves them here)
MISSING: list[tuple[str, str]] = []
def artifact(rel: str, produced_by: str):
    \"\"\"Path to a project artifact in the bundle, or None (with a one-line notice) if it is not there.\"\"\"
    p = ROOT / rel
    if p.exists():
        return p
    print(f"not available yet — {rel} (produced by: {produced_by})")
    MISSING.append((rel, produced_by))
    return None
def show(df, title=None, fmt=None):
    if title:
        display(Markdown(f"**{title}**"))
    display(df.style.format(fmt) if fmt else df)
""")
code("""
# "Device": the model never runs in this notebook. Compute lives on the Space; the accelerator Colab gave this runtime is informational.
try:
    rt = api.get_space_runtime(SPACE)
    SPACE_HW = rt.hardware
    print(f"Space {SPACE}: stage {rt.stage}, hardware {rt.hardware}   <- the device every prediction in this notebook runs on")
except Exception as e:
    SPACE_HW = CFG["hf"]["space_hardware"]
    print(f"Space runtime not readable ({type(e).__name__}): private Space without a token, or the Space does not exist; "
          f"config.yaml says hardware {SPACE_HW}")
try:
    import torch
    TORCH_OK = True
    local = f"torch {torch.__version__}, cuda available: {torch.cuda.is_available()}" + (f" ({torch.cuda.get_device_name(0)})" if torch.cuda.is_available() else "")
except Exception as e:   # torch is preinstalled on Colab; elsewhere the notebook falls back to the counts recorded in config.json
    TORCH_OK = False
    local = f"torch not importable ({type(e).__name__})"
print(f"this runtime: {local}   (informational: not used for the model)")
""")
variant(None)
code("""
# The GPU facts recorded by the smoke test on 2026-10-07 (the GPU is not visible to this kernel).
smoke = artifact("reports/smoke_test.json", "src/smoke_test.py")
SMOKE = json.loads(smoke.read_text()) if smoke else {}     # reused by Step 5 and Results (no retyped numbers)
if smoke:
    s = SMOKE["device"]
    print("Training GPU (from reports/smoke_test.json):")
    for k in ("device", "sm", "vram_gib", "bf16", "torch", "cuda_build", "cudnn", "python"):
        print(f"{k:>10}: {s[k]}")
print("\\nPinned training environment (requirements-train.txt):")
print((ROOT / "requirements-train.txt").read_text())
""")

# =============================================================================== Step 2
md("""
**Step 2: Download Data & Create DataLoaders**
The course reference downloads FashionMNIST from `torchvision.datasets`; there is no phishing corpus in torchvision,
so our data comes from the Hugging Face Hub and is rebuilt locally by `src/data_prep.py` (download, normalise,
dedupe, decontaminate, split, assert zero leakage). The Hub datasets that were inspected:
| dataset | role | verdict |
|---|---|---|
| `puyang2025/seven-phishing-email-datasets` | labelled emails (TREC-05/06/07, CEAS-08, Enron-Spam, SpamAssassin, Ling) | **used**; its own train/test split leaks (5.1% exact / ~40% near duplicates), so we split ourselves |
| `corbt/enron-emails` | unlabelled Enron mail for MLM pretraining | **used** after removing 52.7% exact duplicates and stripping quoted tails |
| `it4lia/PhishingEmailCuratedDatasets_Cleaned` | labelled | **excluded**: no body text, only subjects + 53 numeric features |
| `zefang-liu/phishing-email-dataset` | labelled | **excluded**: 91.9% exact subset of the primary set, lowercased, and 20% of its "safe" rows contain the token *enron* (a shortcut) |
| `kudzaiprichard/aura-phishing-email-corpus` (Nazario + Nigerian fraud) | phishing-only corpora for the three-way labels | **used** (added in RUN_LOG entry 7) |
| `locuoco/...-300000` | ready-made 3-class set | **rejected**: 9.9% of its rows are copies of our TREC-06 test set and its "phish" label is a preprocessing artifact |
**Holdouts.** TREC-06 (16,302 rows, 23.9% positive) is the binary test set: it is the one sub-corpus that is not
Enron-derived (89% of TREC-05 ham and 86% of Enron-Spam ham appear in the pretraining corpus), and it was never seen
by the tokenizer, the pretraining or the fine-tuning. A second, clean-provenance **three-way holdout** (3,220 rows:
legitimate / phishing / spam) gives the 3x3 confusion matrix. Train and validation are split by near-duplicate
cluster so no spam template straddles the two.
""")
code("""
manifest = json.loads((ROOT / "data/processed/splits_manifest.json").read_text())
ds = manifest["datasets"]
rows = [
    ("labelled primary", ds["labelled_primary"]["repo"], ds["labelled_primary"]["rows_loaded"],
     f"hub splits {ds['labelled_primary']['hub_splits_concatenated']} concatenated (the hub split leaks)"),
    ("pretraining corpus", ds["pretrain_corpus"]["repo"], ds["pretrain_corpus"]["rows_total"],
     f"-> {manifest['pretrain']['rows_out']:,} docs after dedup/quote stripping"),
] + [("excluded", r, "-", reason) for r, reason in zip(ds["excluded"],
       ("no body text (subjects + 53 numeric features only)", "91.9% exact subset of the primary set, lowercased, 'enron' token shortcut"))]
rows.append(("phishing corpora (3-way)", CFG["data"]["threeway"]["champa"]["repo"],
             sum(manifest["threeway"]["new_rows"]["loaded_by_source"].values()), "Nazario + Nigerian_Fraud, CC-BY-4.0"))
show(pd.DataFrame(rows, columns=["role", "Hub dataset", "rows loaded", "note"]).astype({"rows loaded": "string"}), "Hub datasets")
print(f"data build: {manifest['timestamp']}  seed {manifest['seed']}  elapsed {manifest['elapsed_seconds']} s  peak RSS {manifest['peak_rss_gb']} GB")
""")
code("""
# Dedup / leakage procedure: every row is normalised (NFKC -> lowercase -> [a-z0-9]) and keyed by sha1 of the full
# text, a 300-char prefix and subject+body; near-duplicates share >= 3 of their 50-char shingles (stride 25).
# These are the checks data_prep asserts to be ZERO (recomputed independently from the written files by `check3`).
leak = manifest["threeway"]["leakage"]
tbl = pd.DataFrame([{"check": k, "count": v["count"], "asserted zero": v["asserted"], "pass": v["pass"], "detail": v.get("detail", "")}
                    for k, v in leak.items()])
show(tbl[tbl["asserted zero"]].reset_index(drop=True), "Leakage checks asserted to be zero (binary gate + three-way)")
show(tbl[~tbl["asserted zero"]].reset_index(drop=True), "Informational counters (not asserted)")
D = manifest["drops"]
drops = pd.DataFrame([
    ("labelled rows with < 20 alphanumerics (garbage / empty)", D["labelled_min_alnum"]["rows"]),
    ("Ling corpus excluded (academic mailing list)", sum(D["ling_excluded_by_label"].values())),
    ("identical body carrying both labels (label noise)", D["pool_label_conflicts"]["rows_dropped"]),
    ("exact duplicates in the labelled pool", sum(sum(v.values()) for v in D["pool_exact_dups_by_source_label"].values())),
    ("training rows that are exact/near duplicates of TREC-06 holdout rows", D["decontamination_vs_holdout"]["by_reason"]["total"]),
    ("pretraining rows: exact duplicates", manifest["pretrain"]["exact_dups_raw"]),
    ("pretraining rows matching a validation email", manifest["pretrain"]["val_key_removed"]),
    ("pretraining rows near-duplicating a holdout email", manifest["pretrain"]["holdout_near_dup_removed"]),
], columns=["removed", "rows"]).set_index("removed")
show(drops, "Rows removed by the dedup / decontamination procedure", fmt="{:,}")
""")
variant("local")
code("""
import pyarrow.parquet as pq
PROCESSED = Path(CFG["paths"]["data_processed"])
LABEL3_NAME = {0: "Legitimate", 1: "Phishing", 2: "Spam", -1: "unwanted (unsplit)"}
SPLITS = ["finetune_train", "finetune_val", "test_holdout", "test_holdout3"]
frames = []
for name in SPLITS:
    t = pq.read_table(PROCESSED / f"{name}.parquet", columns=["source", "label", "label3"]).to_pandas()
    t["split"] = name
    frames.append(t)
lab = pd.concat(frames, ignore_index=True)
lab["label3"] = lab["label3"].astype(int); lab["label"] = lab["label"].astype(int)
lab["class"] = lab["label3"].map(LABEL3_NAME)
by_split = pd.crosstab(lab["split"], lab["class"]).reindex(SPLITS)
by_split["rows"] = by_split.sum(axis=1)
by_split["binary positive rate"] = lab.groupby("split")["label"].mean().reindex(SPLITS).round(4)
show(by_split, "Rows per split by label3 (EDA)")
show(pd.crosstab([lab["split"], lab["source"]], lab["class"]), "Rows per split x source x class")
""")
variant("colab")
code("""
# Rows per split by class, from the counts data_prep.py wrote into splits_manifest.json (the parquet splits themselves stay on
# the workstation: the bundle carries reports, configs and source files, never data or weights).
LABEL3_NAME = {0: "Legitimate", 1: "Phishing", 2: "Spam", -1: "unwanted (unsplit)"}
SPLITS = ["finetune_train", "finetune_val", "test_holdout", "test_holdout3"]
tw = manifest["threeway"]
lab = pd.DataFrame([{"split": split, "source": src, "class": LABEL3_NAME[int(l3)], "rows": n}
                    for split in SPLITS for src, by_l3 in tw["counts"][split].items() for l3, n in by_l3.items()])
by_split = lab.pivot_table(index="split", columns="class", values="rows", aggfunc="sum", fill_value=0).reindex(SPLITS)
by_split["rows"] = by_split.sum(axis=1)
by_split["binary positive rate"] = pd.Series({s: tw["totals"][s]["binary_base_rate"] for s in SPLITS})
show(by_split, "Rows per split by label3 (EDA, from splits_manifest.json)")
show(lab.pivot_table(index=["split", "source"], columns="class", values="rows", aggfunc="sum", fill_value=0), "Rows per split x source x class")
""")
variant(None)
code("""
fig, ax = plt.subplots(figsize=(9, 3.8))
cols = ["Legitimate", "Phishing", "Spam", "unwanted (unsplit)"]
colors = ["#2a78d6", "#eb6834", "#eda100", "#9aa0a6"]
bottom = np.zeros(len(SPLITS))
for c, col in zip(cols, colors):
    vals = by_split[c].reindex(SPLITS).fillna(0).to_numpy() if c in by_split else np.zeros(len(SPLITS))
    ax.bar(SPLITS, vals, bottom=bottom, label=c, color=col)
    bottom += vals
ax.set_ylabel("rows"); ax.set_title("Labelled splits by class", loc="left"); ax.legend(frameon=False)
for s in ("top", "right"): ax.spines[s].set_visible(False)
plt.tight_layout(); plt.show()
""")
variant("local")
code("""
# Tokenizer: byte-level BPE, vocab 32,768, NFKC only (case preserved: ALL-CAPS is a spam signal), trained on the
# pretraining corpus + finetune_train ONLY (val and the holdouts are refused by name; 0 holdout hash collisions).
from tokenizer import load_tokenizer, encode_batch
tok_report = json.loads((ROOT / "data/processed/tokenizer_report.json").read_text())
tr = tok_report["train"]
print(f"vocab {tr['final_vocab_size']:,}  specials {tr['special_token_ids']}  normalizer {tr['normalizer']}  "
      f"pre-tokenizer {tr['pre_tokenizer']}\\ntrained on {list(tr['training_files'])} in {tr['train_seconds']} s; "
      f"excluded {tr['excluded_files']}; holdout exact collisions {tr['holdout_exact_collisions']}")
show(pd.DataFrame(tok_report["stats"]["splits"]).set_index("split")
       [["rows", "mean_tokens", "median_tokens", "p95_tokens", "chars_per_token", "pct_truncated", "mean_tokens_by_label"]],
     "Token-length statistics per split (tokenizer.py stats; max_seq_len 512)")
TOK = load_tokenizer(CFG["paths"]["tokenizer"])
for ex in tok_report["stats"]["examples"][:3]:
    enc = TOK.encode(ex["text"])
    print(f"\\n[{ex['name']}] {ex['text'][:90]!r}\\n  {len(enc.ids)} tokens: {enc.tokens[:28]}{' ...' if len(enc.ids) > 28 else ''}")
""")
code("""
# Token-length histogram per class, tokenised live on a seeded 5,000-row sample of finetune_train.
rng = np.random.default_rng(CFG["seed"])
full = pq.read_table(PROCESSED / "finetune_train.parquet", columns=["text", "label3"])
idx = np.sort(rng.choice(full.num_rows, size=5000, replace=False))
sample = full.take(idx).to_pandas()
sample["label3"] = sample["label3"].astype(int)
sample["n_tokens"] = [len(e.ids) for e in TOK.encode_batch_fast(sample["text"].fillna("").tolist())]
fig, ax = plt.subplots(figsize=(9, 3.8))
bins = np.logspace(1, 4, 50)
for c, col in zip([0, 1, 2, -1], ["#2a78d6", "#eb6834", "#eda100", "#9aa0a6"]):
    v = sample.loc[sample["label3"] == c, "n_tokens"]
    if len(v):
        ax.hist(v, bins=bins, histtype="step", linewidth=1.6, color=col, label=f"{LABEL3_NAME[c]} (n={len(v)}, mean {v.mean():.0f})")
ax.axvline(CFG["model"]["max_seq_len"], color="black", linestyle="--", linewidth=1, label="max_seq_len 512 (truncation)")
ax.set_xscale("log"); ax.set_xlabel("tokens per email (log)"); ax.set_ylabel("emails")
ax.set_title("Token length by class, 5k-row sample of finetune_train", loc="left"); ax.legend(frameon=False, fontsize=8)
for s in ("top", "right"): ax.spines[s].set_visible(False)
plt.tight_layout(); plt.show()
print(f"{100 * (sample['n_tokens'] > 512).mean():.1f}% of the sample is longer than 512 tokens and gets truncated. "
      "Length is a known confound: legitimate mail is ~2x longer on average; the error analysis checks whether the model leans on it.")
""")
code("""
# Build REAL DataLoaders with the training script's own functions (src/finetune.py) on a 512-row seeded sample.
import finetune
ft_cfg = CFG["finetune"]
MULTIPLE = int(ft_cfg["dynamic_padding_multiple"]); MAX_LEN = int(CFG["model"]["max_seq_len"])
BATCH_SIZE = int(ft_cfg["batch_size"]); NUM_LABELS = int(ft_cfg["num_labels"])
labelled = finetune.read_split(PROCESSED / "finetune_train.parquet", CFG["data"]["holdout_test_source"],
                               limit=512, seed=CFG["seed"], num_labels=NUM_LABELS)        # refuses the TREC-06 holdout by name AND source
encoded = finetune.encode_split("train-sample", TOK, labelled, MAX_LEN, MULTIPLE)        # [CLS] text [SEP], truncate 512, store trimmed
batches = finetune.epoch_batches(encoded.lengths, BATCH_SIZE, seed=CFG["seed"], epoch=0)  # seeded shuffle + length bucketing
train_dataloader = finetune.make_loader(encoded, batches, MULTIPLE)                      # torch.utils.data.DataLoader
test_batches = finetune.eval_batches(encoded.lengths, int(ft_cfg["eval_batch_size"]))
test_dataloader = finetune.make_loader(encoded, test_batches, MULTIPLE)
print(f"{len(labelled)} rows -> {len(batches)} training batches of {BATCH_SIZE} (shuffled, length-bucketed) and "
      f"{len(test_batches)} eval batches of {ft_cfg['eval_batch_size']} (sorted by length, no shuffle)")
print("class counts in the sample:", finetune.label_counts(labelled.label3, NUM_LABELS))
idx, ids, mask, labels, allowed = next(iter(train_dataloader))
print(f"\\none batch: input_ids {tuple(ids.shape)} {ids.dtype} | attention_mask {tuple(mask.shape)} {mask.dtype} | "
      f"labels {tuple(labels.shape)} | allowed-class mask {tuple(allowed.shape)}")
print(f"padding rule: pad to the longest row in the batch ({int(mask.sum(1).max())} tokens) rounded up to a multiple of "
      f"{MULTIPLE} -> L = {ids.shape[1]}; padded positions carry [PAD]=0 and mask False")
print("\\nfirst rows of the batch (label_set -> allowed-class mask over [Legitimate, Phishing, Spam]):")
for r in range(min(6, len(idx))):
    i = int(idx[r])
    print(f"  row {i:>4}  source {labelled.sources[i]:<9} label3 {int(labelled.label3[i]):>2}  allowed {allowed[r].int().tolist()}  "
          f"tokens {int(mask[r].sum()):>3}  text {labelled.texts[i][:60]!r}")
print("\\n'unwanted' rows (TREC/CEAS positives) allow BOTH phishing and spam: the loss is -log(P(phishing)+P(spam)), the partial-label NLL.")
""")
variant("colab")
code("""
# Tokenizer: byte-level BPE, vocab 32,768, NFKC only (case preserved: ALL-CAPS is a spam signal), trained on the
# pretraining corpus + finetune_train ONLY (val and the holdouts are refused by name; 0 holdout hash collisions).
# tokenizer.json travels with the weights (model repo root and the Space); the report written next to it carries the
# statistics and the worked examples, so nothing is tokenised on this runtime.
tok_report = json.loads((ROOT / "data/processed/tokenizer_report.json").read_text())
tr = tok_report["train"]
print(f"vocab {tr['final_vocab_size']:,}  specials {tr['special_token_ids']}  normalizer {tr['normalizer']}  "
      f"pre-tokenizer {tr['pre_tokenizer']}\\ntrained on {list(tr['training_files'])} in {tr['train_seconds']} s; "
      f"excluded {tr['excluded_files']}; holdout exact collisions {tr['holdout_exact_collisions']}")
show(pd.DataFrame(tok_report["stats"]["splits"]).set_index("split")
       [["rows", "mean_tokens", "median_tokens", "p95_tokens", "chars_per_token", "pct_truncated", "mean_tokens_by_label"]],
     "Token-length statistics per split (tokenizer.py stats; max_seq_len 512)")
for ex in tok_report["stats"]["examples"][:3]:
    toks = ex["tokens"]
    print(f"\\n[{ex['name']}] {ex['text'][:90]!r}\\n  {ex['n_tokens_with_specials']} tokens: {toks[:28]}{' ...' if len(toks) > 28 else ''}")
""")
code("""
# Token length per split from the tokenizer report. The local notebook tokenises a seeded 5,000-row sample of finetune_train
# live and draws a histogram per class; that sample needs the parquet data, so here the per-split statistics computed by
# tokenizer.py on the workstation stand in for it.
st = pd.DataFrame(tok_report["stats"]["splits"]).set_index("split")
fig, ax = plt.subplots(figsize=(9, 3.8))
x = np.arange(len(st)); w = 0.27
for off, col, color in ((-w, "median_tokens", "#2a78d6"), (0, "mean_tokens", "#eb6834"), (w, "p95_tokens", "#eda100")):
    ax.bar(x + off, st[col], width=w, color=color, label=col.replace("_", " "))
ax.axhline(CFG["model"]["max_seq_len"], color="black", linestyle="--", linewidth=1, label="max_seq_len 512 (truncation)")
ax.set_xticks(x, st.index); ax.set_yscale("log"); ax.set_ylabel("tokens per email (log)")
ax.set_title("Token length per split (tokenizer.py stats)", loc="left"); ax.legend(frameon=False, fontsize=8)
for s in ("top", "right"): ax.spines[s].set_visible(False)
plt.tight_layout(); plt.show()
print("rows longer than 512 tokens (truncated): " + ", ".join(f"{s} {v:.1f}%" for s, v in st["pct_truncated"].items())
      + ". Length is a known confound: legitimate mail is ~2x longer on average; the error analysis checks whether the model leans on it.")
""")
code("""
# The DataLoaders are built by src/finetune.py on the workstation (they need the parquet splits and torch). The bundle carries
# the source, so the three definitions that make the torch.utils.data.DataLoader are shown from the file text: a seeded,
# length-bucketed batch sampler, a row-index Dataset and the padding collate.
ft_src = (ROOT / "src/finetune.py").read_text()
def source_of(name, text):
    \"\"\"Top-level `def`/`class` <name> of a source file read as text (no import needed).\"\"\"
    m = re.search(rf"^(?:def|class) {name}\\b.*?(?=^\\S|\\Z)", text, re.S | re.M)
    return m.group(0).rstrip() if m else f"# {name}: not found"
for fn in ("epoch_batches", "RowIndexDataset", "make_loader"):
    print(source_of(fn, ft_src), "\\n")
ft_cfg = CFG["finetune"]
MULTIPLE = int(ft_cfg["dynamic_padding_multiple"]); MAX_LEN = int(CFG["model"]["max_seq_len"])
print(f"dials: batch_size {ft_cfg['batch_size']}, eval_batch_size {ft_cfg['eval_batch_size']}, max_seq_len {MAX_LEN}, pad to a multiple of {MULTIPLE}; "
      "padded positions carry [PAD]=0 and mask False. 'unwanted' rows (TREC/CEAS positives) allow BOTH phishing and spam: "
      "the loss is -log(P(phishing)+P(spam)), the partial-label NLL.")
""")
variant(None)
code("""
# Pretraining data: documents are tokenised once, concatenated with [SEP] after each, and cut into packed windows of
# [CLS] + 511 stream tokens (no padding, no attention mask -> the Flash SDPA kernel). Dynamic 15% masking per batch.
pt = json.loads((ROOT / "data/processed/pretrain_tokens.json").read_text())
print(f"documents {pt['n_docs']:,} ({pt['n_docs_train']:,} train / {pt['n_docs_val']:,} val: {pt['val_rule']})")
print(f"tokens    {pt['n_tokens_train']:,} train + {pt['n_tokens_val']:,} val  ({pt['tokens_per_char']} tokens/char over {pt['chars']:,} chars; "
      f"encoded in {pt['encode_seconds']} s, stored as {pt['dtype']})")
print(f"packing   {pt['packing']}")
summ = artifact(f"checkpoints/pretrain/{PRETRAIN_RUN}/run_summary.json", f"src/pretrain.py train --run-name {PRETRAIN_RUN}")
if summ:
    d = json.loads(summ.read_text())["config"]
    print(f"windows   {d['data']['n_windows']:,} x 512 per epoch -> {d['steps_per_epoch']} optimizer steps at effective batch "
          f"{d['effective_batch']} (micro-batch {d['micro_batch']} x {d['grad_accumulation']} accumulation); {d['data']['n_val_windows']:,} val windows")
""")

# =============================================================================== Step 3
md("""
**Step 3: Define the Neural Network**
The reference notebook defines a small CNN; ours is `src/model.py`, a from-scratch **pre-LayerNorm BERT-style
encoder** in plain PyTorch (no `transformers`, no pretrained weights): token + learned position embeddings, 24
encoder layers (`x = x + Dropout(OutProj(SDPA(QKV(LN1(x)))))`, `x = x + Dropout(FFN_down(GELU(FFN_up(LN2(x)))))`),
a final LayerNorm, and two heads. Pre-LN is chosen because a 24-layer stack trained from scratch tolerates the
learning rate and warmup far better than post-LN. Attention runs through `F.scaled_dot_product_attention` so the fused
Flash / memory-efficient kernels never materialise the 512x512 score matrix. The **MLM head** ties its decoder to the
token embedding (saves 33.5M parameters); the **classification head** reads the `[CLS]` position and outputs three
logits (legitimate / phishing / spam); rows whose corpus never separated spam from phishing train on the probability
mass of {phishing, spam}. The course requires at least 2 layers; this model has 24.
""")
variant("local")
code("""
import model as phish_model
from model import EncoderConfig, EncoderLayer, EncoderForMLM, EncoderForClassification, count_parameters
import re
src_cfg = inspect.getsource(EncoderConfig).splitlines()
print("class EncoderConfig:   # dataclass: the dials of the encoder")
print("\\n".join(l for l in src_cfg if re.match(r"^    \\w+: [\\w| ]+ = ", l)))
print("\\nclass EncoderLayer(nn.Module):   # one pre-LN block; 24 of these are stacked")
print(inspect.getsource(EncoderLayer.__init__))
print(inspect.getsource(EncoderLayer.forward))
""")
code("""
# Instantiate both wrappers on the CPU (fp32, ~1.3 GB each) and count parameters.
model_cfg = EncoderConfig.from_dict({**CFG["model"], "num_labels": CFG["finetune"]["num_labels"]})
print("config:", model_cfg.to_dict())
mlm_model = EncoderForMLM(model_cfg)
cls_model = EncoderForClassification.from_pretrained_encoder(mlm_model)   # encoder weights copied, fresh 3-class head
counts = pd.DataFrame({"EncoderForMLM (tied head)": count_parameters(mlm_model),
                       "EncoderForClassification (3 classes)": count_parameters(cls_model)})
show(counts, "Parameter counts", fmt="{:,}")
assert mlm_model.head.decoder.weight is mlm_model.encoder.embeddings.tok.weight, "MLM head must be tied"
print(f"the classifier has {model_cfg.n_layers} encoder layers (course minimum: 2); head: CLS -> dropout -> Linear -> tanh -> dropout -> Linear({model_cfg.num_labels})")
del mlm_model, cls_model
""")
code("""
# Why 24 layers / d=1024 / vocab 32,768: the parameter-count table from docs/research/research_model-math.md section 1
# (configs A-F; the brief asked for ~330M parameters). Config A is the BERT-large shape and lands +2.3% over the target.
math_md = (ROOT / "docs/research/research_model-math.md").read_text().splitlines()
start = next(i for i, l in enumerate(math_md) if l.startswith("## 1."))
end = next(i for i, l in enumerate(math_md) if l.startswith("## 2."))
table = [l for l in math_md[start:end] if l.startswith("|")]
header = [h.strip() for h in table[0].strip("|").split("|")]
body = [[c.strip().replace("**", "") for c in l.strip("|").split("|")] for l in table[2:]]
show(pd.DataFrame(body, columns=header).set_index("config"), "Candidate configurations (research_model-math.md section 1)")
print(f"Chosen: A = {model_cfg.n_layers}L d{model_cfg.d_model} h{model_cfg.n_heads} f{model_cfg.d_ff} V{model_cfg.vocab_size} -> "
      f"{counts.loc['total'].iloc[0]:,} (pretraining, tied head) / {counts.loc['total'].iloc[1]:,} (classifier), counted above.")
""")

variant("colab")
code("""
# src/model.py from the bundle. With torch present (Colab preinstalls it) the module is imported and inspected exactly as the
# local notebook does; without torch the same definitions are shown from the file text.
sys.path.insert(0, str(ROOT / "src"))
model_src = (ROOT / "src/model.py").read_text()
if TORCH_OK:
    import inspect
    from model import EncoderConfig, EncoderLayer, EncoderForMLM, EncoderForClassification, count_parameters
    src_cfg = inspect.getsource(EncoderConfig).splitlines()
    layer_init, layer_fwd = inspect.getsource(EncoderLayer.__init__), inspect.getsource(EncoderLayer.forward)
else:
    src_cfg = source_of("EncoderConfig", model_src).splitlines()
    parts = re.split(r"(?=^    def )", source_of("EncoderLayer", model_src), flags=re.M)
    layer_init = next(p for p in parts if p.startswith("    def __init__"))
    layer_fwd = next(p for p in parts if p.startswith("    def forward"))
print("class EncoderConfig:   # dataclass: the dials of the encoder")
print("\\n".join(l for l in src_cfg if re.match(r"^    \\w+: [\\w| ]+ = ", l)))
print("\\nclass EncoderLayer(nn.Module):   # one pre-LN block; 24 of these are stacked")
print(layer_init)
print(layer_fwd)
""")
code("""
# Parameter counts: with torch, both wrappers are instantiated on this runtime's CPU (fp32, ~1.3 GB each) and counted, as in
# the local notebook; without torch the counts recorded in the deployed model's config.json are shown.
model_cfg_d = {**CFG["model"], "num_labels": CFG["finetune"]["num_labels"]}
if TORCH_OK:
    model_cfg = EncoderConfig.from_dict(model_cfg_d)
    print("config:", model_cfg.to_dict())
    mlm_model = EncoderForMLM(model_cfg)
    cls_model = EncoderForClassification.from_pretrained_encoder(mlm_model)   # encoder weights copied, fresh 3-class head
    counts = pd.DataFrame({"EncoderForMLM (tied head)": count_parameters(mlm_model),
                           "EncoderForClassification (3 classes)": count_parameters(cls_model)})
    assert mlm_model.head.decoder.weight is mlm_model.encoder.embeddings.tok.weight, "MLM head must be tied"
    del mlm_model, cls_model
    how = "instantiated on this runtime's CPU and counted"
else:
    from types import SimpleNamespace
    model_cfg = SimpleNamespace(**model_cfg_d)
    print("config:", model_cfg_d)
    counts = pd.DataFrame({"EncoderForClassification (3 classes)": EXPORT_CONFIG["finetune_checkpoint"]["params"]})
    how = "recorded in the deployed model's config.json; torch is not importable here"
show(counts, f"Parameter counts ({how})", fmt="{:,}")
print(f"the classifier has {model_cfg.n_layers} encoder layers (course minimum: 2); head: CLS -> dropout -> Linear -> tanh -> dropout -> Linear({model_cfg.num_labels})")
""")
code("""
# Why 24 layers / d=1024 / vocab 32,768: the parameter-count table from docs/research/research_model-math.md section 1
# (configs A-F; the brief asked for ~330M parameters). Config A is the BERT-large shape and lands +2.3% over the target.
math_md = (ROOT / "docs/research/research_model-math.md").read_text().splitlines()
start = next(i for i, l in enumerate(math_md) if l.startswith("## 1."))
end = next(i for i, l in enumerate(math_md) if l.startswith("## 2."))
table = [l for l in math_md[start:end] if l.startswith("|")]
header = [h.strip() for h in table[0].strip("|").split("|")]
body = [[c.strip().replace("**", "") for c in l.strip("|").split("|")] for l in table[2:]]
show(pd.DataFrame(body, columns=header).set_index("config"), "Candidate configurations (research_model-math.md section 1)")
print(f"Chosen: A = {model_cfg.n_layers}L d{model_cfg.d_model} h{model_cfg.n_heads} f{model_cfg.d_ff} V{model_cfg.vocab_size} -> "
      f"{counts.loc['total'].iloc[-1]:,} parameters in the classifier, counted above.")
""")
variant(None)

# =============================================================================== Step 4
md("""
**Step 4: Train the Model**
Training happens in two phases, both far too long for a notebook cell, so they run as scripts inside `tmux` on the
RTX 3090 Ti and this notebook reads their metrics. **Phase 1, MLM pretraining** (`src/pretrain.py`): AdamW, peak LR
1.5e-4, 6% warmup, cosine decay to 10%, effective batch 264 sequences x 512 tokens, bf16 autocast, 12 epochs over 107.7M
tokens (~9,552 optimizer steps). **Phase 2, fine-tuning** (`src/finetune.py`): the instructor's dials are CLI flags
(`--n-layers`, `--epochs`, `--loss cross_entropy|weighted_cross_entropy|focal`, `--lr`, `--freeze-layers`, `--synthetic`),
AdamW with linear warmup/decay, **early stopping** on the validation F1 of the unwanted roll-up (patience 2), and every
run appends one line to `reports/experiments.jsonl` so the experimentation table is generated from evidence. The six
fine-tuning experiments (loss function x3, synthetic data, 12 layers, scratch init) are queued in
`scripts/run_experiments.sh` (tmux `phish-finetune`), which waits for `final/` and then trains, evaluates and exports
each run unattended (RUN_LOG entry 9).
""")
code("""
print("Phase 1 launch (scripts/launch_pretrain.sh; Nathan's settings on 2026-10-08):")
print("    EPOCHS=12 MICRO_BATCH=24 scripts/launch_pretrain.sh        # -> tmux session 'phish-pretrain'")
print("    tmux attach -t phish-pretrain                              # Ctrl-b d detaches; Ctrl-C saves a checkpoint and exits")
print("    RESUME=1 scripts/launch_pretrain.sh                        # continue from the latest checkpoint")
print("Phase 2 (after final/ exists), the dials are flags:")
print("    .venv/bin/python src/finetune.py train --run-name ft-ce   --loss cross_entropy --epochs 3 --lr 3e-5")
print("    .venv/bin/python src/finetune.py train --run-name ft-focal --loss focal --focal-gamma 2.0")
print("    .venv/bin/python src/finetune.py train --run-name ft-12L   --n-layers 12          # fewer layers")
print("    .venv/bin/python src/finetune.py list                                              # experiments table\\n")
queue = ROOT / "scripts/run_experiments.sh"
if queue.exists():
    lines = queue.read_text().splitlines()
    a = next(i for i, l in enumerate(lines) if l.startswith("EXPERIMENTS=("))
    b = next(i for i in range(a, len(lines)) if lines[i].strip() == ")")
    print("Queued experiments (scripts/run_experiments.sh, tmux 'phish-finetune'; name | flags, all --init pretrained unless scratch):")
    print("\\n".join(lines[a:b + 1]), "\\n")
print((ROOT / "scripts/launch_pretrain.sh").read_text())
""")
code("""
# Phase 1 live metrics: checkpoints/pretrain/<run>/metrics.csv grows while the run trains.
mpath = artifact(f"checkpoints/pretrain/{PRETRAIN_RUN}/metrics.csv", f"src/pretrain.py train --run-name {PRETRAIN_RUN}")
final_cfg = artifact(f"checkpoints/pretrain/{PRETRAIN_RUN}/final/config.json",
                     f"written automatically when the pretraining run completes (or: src/pretrain.py export --run-name {PRETRAIN_RUN})")
if mpath:
    # the trainer appends while we read: drop a partially written last row instead of failing on NaN
    pm = pd.read_csv(mpath).dropna(subset=["step", "loss", "loss_avg", "lr", "tokens_per_s", "train_seconds_total"])
    last = pm.iloc[-1]
    plan = json.loads((ROOT / f"checkpoints/pretrain/{PRETRAIN_RUN}/run_summary.json").read_text())["config"]
    status = "COMPLETE" if final_cfg else "RUN IN PROGRESS"
    print(f"[{status}] step {int(last.step):,} / {plan['total_steps']:,}  epoch {last.epoch_frac:.2f} / {plan['epochs']}  "
          f"loss {last.loss:.4f} (avg {last.loss_avg:.4f})  lr {last.lr:.2e}  {int(last.tokens_per_s):,} tokens/s  "
          f"peak reserved {last.max_reserved_gb} GiB")
    val = pm.dropna(subset=["val_mlm_loss"])
    if len(val):
        v = val.iloc[-1]
        print(f"last val MLM loss {v.val_mlm_loss:.4f}  val masked-token accuracy {v.val_mlm_acc:.3f} at step {int(v.step)}  (ln V = {np.log(CFG['model']['vocab_size']):.2f})")
    print(f"Total training time: {last.train_seconds_total:.2f} seconds{' so far' if not final_cfg else ''} "
          f"({last.train_seconds_total / 3600:.2f} h; projected {last.train_seconds_total / last.step * plan['total_steps'] / 3600:.1f} h for the full plan)")
    fig, ax = plt.subplots(figsize=(9.5, 4))
    ax.plot(pm.step, pm.loss, color="#9ec3ee", linewidth=0.9, label="train MLM loss (20-step)")
    ax.plot(pm.step, pm.loss_avg, color="#2a78d6", linewidth=1.8, label="train MLM loss (running avg)")
    if len(val):
        ax.plot(val.step, val.val_mlm_loss, "o-", color="#eb6834", label="val MLM loss (every 500 steps)")
    ax.axhline(np.log(CFG["model"]["vocab_size"]), color="grey", linestyle=":", label="ln(vocab) = uniform guess")
    ax.set_xlabel("optimizer step"); ax.set_ylabel("MLM loss"); ax.set_title(f"Pretraining {PRETRAIN_RUN}: {status.lower()}", loc="left")
    ax2 = ax.twinx(); ax2.plot(pm.step, pm.lr, color="#1baf7a", linewidth=1, label="learning rate"); ax2.set_ylabel("lr", color="#1baf7a")
    h1, l1 = ax.get_legend_handles_labels(); h2, l2 = ax2.get_legend_handles_labels(); ax.legend(h1 + h2, l1 + l2, frameon=False, fontsize=8)
    for s in ("top",): ax.spines[s].set_visible(False); ax2.spines[s].set_visible(False)
    plt.tight_layout(); plt.show()
""")
code("""
# reports/pretrain_curves.png is written by `pretrain.py plot --run-name <run>`; show it only if it belongs to this run.
png = artifact("reports/pretrain_curves.png", f"src/pretrain.py plot --run-name {PRETRAIN_RUN}")
if png and summ:
    import re as _re
    S = json.loads(summ.read_text())
    # run_summary.json records the finish time and the log path (pretrain-YYYYMMDD-HHMMSS.log); the launch time comes from the latter.
    started = S.get("started")
    if not started and S.get("log"):
        m = _re.search(r"(\\d{4})(\\d{2})(\\d{2})-(\\d{2})(\\d{2})(\\d{2})", Path(S["log"]).name)
        if m:
            started = "{}-{}-{}T{}:{}:{}".format(*m.groups())
    started = started or S.get("finished") or ""
    if time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(png.stat().st_mtime)) >= started:
        display(Image(filename=str(png), width=900))
    else:
        print(f"reports/pretrain_curves.png is from an earlier DEV self-test run (older than the {PRETRAIN_RUN} launch at {started}); "
              f"regenerate with: src/pretrain.py plot --run-name {PRETRAIN_RUN}")
""")
code("""
# Phase 2 experiments table (reports/experiments.jsonl, one JSON line per run). Dev self-test runs (tiny subsets, a
# handful of steps, scratch init) are marked: they prove the code path, they are not results.
def is_dev(rec):
    # A dev self-test trains on a row subset (--limit-rows) or is named dev*; a real run capped with --max-steps
    # (a legitimate time budget) is NOT a dev run.
    d = rec.get("dials", {})
    return bool(rec.get("dev_run") or d.get("limit_rows") or rec["run_name"].startswith("dev"))
exp_path = artifact("reports/experiments.jsonl", "src/finetune.py train ... (one line per run) and src/evaluate.py run")
EXPERIMENTS = [json.loads(l) for l in exp_path.read_text().splitlines() if l.strip()] if exp_path else []
if exp_path and not EXPERIMENTS:
    print("reports/experiments.jsonl exists but is empty: the dev self-test lines were removed (RUN_LOG entry 9); "
          "the queued runs append to it (produced by: scripts/run_experiments.sh -> src/finetune.py train).")
if EXPERIMENTS:
    rows = []
    for r in EXPERIMENTS:
        if r.get("stage") == "evaluate":
            continue
        d, v = r.get("dials", {}), r.get("val") or {}
        rows.append({"run": r["run_name"], "dev run?": "DEV" if is_dev(r) else "", "init": r.get("init"), "classes": d.get("num_labels", 2),
                     "layers": d.get("n_layers"), "frozen": d.get("frozen_layers"), "epochs run/planned": f"{d.get('epochs_run')}/{d.get('epochs_planned')}",
                     "loss": d.get("loss"), "lr": d.get("lr"), "batch": d.get("batch_size"), "synthetic": d.get("synthetic"),
                     "train rows": r.get("n_train"), "train min": round(r.get("train_seconds", 0) / 60, 1), "peak GiB": r.get("peak_vram_gb"),
                     "val acc": v.get("accuracy"), "val P": v.get("precision"), "val R": v.get("recall"), "val F1": v.get("f1"),
                     "val ROC-AUC": v.get("roc_auc"), "val macro-F1 3w": v.get("macro_f1_3way"), "stopped by": r.get("stopped_by")})
    EXP_DF = pd.DataFrame(rows).round(4)
    if len(EXP_DF):
        show(EXP_DF.set_index("run"), "Fine-tuning experiments (finetune.py list)")
    real = [r for r in EXPERIMENTS if r.get("stage") != "evaluate" and not is_dev(r)]
    if not real:
        print("Only DEV self-test runs exist so far: the real fine-tuning runs start once checkpoints/pretrain/large-24L/final/ exists "
              "(produced by: scripts/run_experiments.sh -> src/finetune.py train --run-name <run> ...).")
""")
code("""
# Fine-tuning curves of the best real run (highest val F1 of the unwanted roll-up); until one exists, the newest dev run is
# shown and labelled as such. Early stopping: evaluation every 500 steps, stop after `patience` evaluations without a
# better val F1; LR: linear warmup over 6% of the steps, then linear decay to 0 (BERT fine-tuning schedule).
real = [r for r in EXPERIMENTS if r.get("stage") != "evaluate" and not is_dev(r) and (r.get("val") or {}).get("f1") is not None]
dev = [r for r in EXPERIMENTS if r.get("stage") != "evaluate" and is_dev(r)]
best = max(real, key=lambda r: r["val"]["f1"]) if real else None
tag = ""
if best is None:
    artifact("checkpoints/finetune/<best run>/metrics.csv", "src/finetune.py train --run-name <run> (after pretraining completes)")
    if dev:
        best, tag = dev[-1], " (DEV self-test run, not a result)"
BEST_FT_RUN = best["run_name"] if best else None
if best:
    run_dir = ROOT / "checkpoints/finetune" / best["run_name"]
    m = pd.read_csv(run_dir / "metrics.csv") if (run_dir / "metrics.csv").exists() else None
    e = pd.read_csv(run_dir / "epochs.csv") if (run_dir / "epochs.csv").exists() else None
    print(f"run {best['run_name']}{tag}: init {best['init']}, loss {best['dials']['loss']}, lr {best['dials']['lr']}, "
          f"{best['steps_run']} steps, best step {best['best_step']}, val F1 {best['val']['f1']:.4f}, stopped by {best['stopped_by']}")
    if m is not None and len(m):
        fig, axes = plt.subplots(1, 2, figsize=(11, 3.6))
        axes[0].plot(m.step, m.loss, color="#2a78d6", label="train loss"); axes[0].set_xlabel("step"); axes[0].set_title(f"{best['run_name']}{tag}: train loss", loc="left", fontsize=10)
        if e is not None and len(e):
            axes[1].plot(e.step, e.f1, "o-", color="#eb6834", label="val F1 (unwanted roll-up)")
            axes[1].plot(e.step, e.roc_auc, "s--", color="#1baf7a", label="val ROC-AUC")
            best_rows = e[e.is_best == 1]
            axes[1].scatter(best_rows.step, best_rows.f1, s=90, facecolors="none", edgecolors="black", label="best (checkpoint kept)")
        axes[1].set_xlabel("step"); axes[1].set_title("validation metrics at each evaluation", loc="left", fontsize=10); axes[1].legend(frameon=False, fontsize=8)
        for ax in axes:
            for s in ("top", "right"): ax.spines[s].set_visible(False)
        plt.tight_layout(); plt.show()
    # The queue writes reports/finetune_curves_<run>.png per run and reports/finetune_curves.png for the main run; a PNG
    # older than the run's own experiments.jsonl line is a leftover from an earlier run and is not shown.
    curves = artifact(f"reports/finetune_curves_{best['run_name']}.png", f"src/finetune.py plot --run-name {best['run_name']} --out reports/finetune_curves_{best['run_name']}.png")
    curves = curves or (ROOT / "reports/finetune_curves.png" if (ROOT / "reports/finetune_curves.png").exists() else None)
    if curves and not tag:
        png_time = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(curves.stat().st_mtime))
        if png_time >= best.get("timestamp", ""):
            display(Image(filename=str(curves), width=900))
        else:
            print(f"{curves.relative_to(ROOT)} predates run {best['run_name']} ({png_time} < {best.get('timestamp')}): stale, not shown; "
                  f"regenerate with src/finetune.py plot --run-name {best['run_name']}")
""")
code("""
# Loss-function comparison (cross_entropy vs weighted_cross_entropy vs focal) over the REAL runs, same dials otherwise.
real = [r for r in EXPERIMENTS if r.get("stage") != "evaluate" and not is_dev(r) and r.get("val")]
if real:
    cmp = pd.DataFrame([{"run": r["run_name"], "loss": r["dials"]["loss"], "focal_gamma": r["dials"].get("focal_gamma"),
                         "class_weights": r["dials"].get("class_weights"), "val F1": r["val"]["f1"], "val P": r["val"]["precision"],
                         "val R": r["val"]["recall"], "val ROC-AUC": r["val"]["roc_auc"], "val PR-AUC": r["val"]["pr_auc"],
                         "val macro-F1 3w": r["val"].get("macro_f1_3way")} for r in real]).sort_values("val F1", ascending=False)
    show(cmp.set_index("run"), "Loss-function comparison (real runs)")
else:
    artifact("reports/experiments.jsonl (real runs with --loss cross_entropy / weighted_cross_entropy / focal)",
             "scripts/run_experiments.sh -> src/finetune.py train --run-name ft-ce|ft-weighted|ft-focal --loss <loss>")
    dev_losses = sorted({(r["dials"].get("loss"), r.get("steps_run"), r.get("n_train")) for r in EXPERIMENTS if r.get("stage") != "evaluate" and is_dev(r)})
    if dev_losses:
        print("The DEV lines above (" + "; ".join(f"{l}: {s} steps on {n:,} rows" for l, s, n in dev_losses) + ") only prove the loss paths run.")
    print("The three loss paths share one code path (src/finetune.py); focal(gamma=0) was verified to equal cross_entropy exactly (RUN_LOG entry 4).")
""")

# =============================================================================== Step 5
variant("local")
md("""
**Step 5: Test Inference Speed and Save the Model**
Exactly like the reference `predict_and_time`: for each of 100 real emails from the TREC-06 holdout, start the timer,
tokenize, run the forward pass, softmax, stop the timer. The classifier is loaded from the export directory
(`model_export/<run>/`) when it exists, otherwise from the newest fine-tune `best/` checkpoint, on the **CPU in fp32**
(the GPU is pretraining). The instructor's benchmark is 0.074 s per sample; the smoke test measured 10.7 ms on the
3090 Ti in bf16 and ~0.7 s on this CPU at 512 tokens, which is why the Space runs on ZeroGPU. The model is then saved
as **safetensors** (the format the Space loads; `torch.save` of the same `state_dict` is shown to be equivalent).
""")
code("""
from safetensors.torch import load_file, save_file
def locate_classifier():
    \"\"\"(directory, kind): the newest export, else the newest real fine-tune best/, else a dev self-test best/.\"\"\"
    exports = sorted((ROOT / "model_export").glob("*/model.safetensors"), key=lambda p: p.stat().st_mtime) if (ROOT / "model_export").exists() else []
    if exports:
        return exports[-1].parent, "export"
    artifact("model_export/<run>/model.safetensors", "scripts/run_experiments.sh -> src/export.py run --checkpoint checkpoints/finetune/<run>/best")
    bests = sorted((ROOT / "checkpoints/finetune").glob("*/best/model.safetensors"), key=lambda p: p.stat().st_mtime)
    real = [b for b in bests if not b.parent.parent.name.startswith("dev")]
    if real:
        return real[-1].parent, "fine-tune best checkpoint"
    if bests:
        return bests[-1].parent, "dev"
    artifact("checkpoints/finetune/<run>/best/model.safetensors", "scripts/run_experiments.sh -> src/finetune.py train --run-name <run>")
    return None, None
CLS_DIR, CLS_KIND = locate_classifier()
classifier = None
if CLS_DIR:
    cj = json.loads((CLS_DIR / "config.json").read_text())
    if CLS_KIND == "dev":
        # Describe the self-test from its own metadata (best/config.json 'finetune' = the CLI args, 'init'), never from a literal.
        ft = cj.get("finetune", {})
        CLS_KIND = (f"DEV self-test checkpoint ({cj.get('init', '?')} init, {ft.get('max_steps', '?')} steps on "
                    f"{ft.get('limit_rows', '?')} rows: timing is representative, predictions are not)")
    cls_cfg = EncoderConfig.from_dict(cj["model"])
    classifier = EncoderForClassification(cls_cfg)
    missing_unexpected = classifier.load_state_dict(load_file(str(CLS_DIR / "model.safetensors"), device="cpu"), strict=True)
    classifier = classifier.float().to("cpu").eval()
    CLS_LABELS = [cj.get("labels", {}).get(str(i), {0: "Legitimate", 1: "Phishing", 2: "Spam"}.get(i, f"class {i}")) for i in range(cls_cfg.num_labels)]
    if cls_cfg.num_labels == 2:
        CLS_LABELS = ["Legitimate", "Spam/Phishing"]
    print(f"loaded {CLS_KIND}: {CLS_DIR.relative_to(ROOT)}  ({count_parameters(classifier)['total']:,} params, {cls_cfg.n_layers} layers, "
          f"{cls_cfg.num_labels} classes {CLS_LABELS}, fp32 on cpu)")
""")
code("""
def predict_and_time(model: nn.Module, texts: list[str], device: str = "cpu", max_len: int = 512):
    \"\"\"Tokenize + forward + softmax per email, timed with timeit.default_timer (the instructor's predict_and_time).\"\"\"
    model.to(device); model.eval()
    stats = []
    with torch.inference_mode():
        for text in texts:
            start_time = timer()
            ids, mask = encode_batch(TOK, [text], max_len, pad_to_multiple_of=MULTIPLE)
            probs = torch.softmax(model(ids.to(device), mask.to(device)).float(), dim=1)[0]
            stats.append({"time": timer() - start_time, "tokens": int(mask.sum()), "probs": probs.tolist()})
    return stats
if classifier is not None:
    holdout = pq.read_table(PROCESSED / "test_holdout.parquet", columns=["text", "label"])   # read-only, 100 rows, timing only
    pick = np.random.default_rng(CFG["seed"]).choice(holdout.num_rows, size=100, replace=False)
    sample_texts = [holdout.column("text")[int(i)].as_py() or "" for i in pick]
    predict_and_time(classifier, sample_texts[:3])                                             # warm-up (oneDNN kernels), untimed
    sample_stats = predict_and_time(classifier, sample_texts)
    times = np.array([s["time"] for s in sample_stats]); toks = np.array([s["tokens"] for s in sample_stats])
    print(f"Average prediction time per sample: {times.mean():.5f} seconds  (median {np.median(times):.4f}, max {times.max():.4f}; "
          f"{len(times)} emails, mean {toks.mean():.0f} tokens after truncation to {MAX_LEN}; CPU fp32, {torch.get_num_threads()} threads)")
    BENCH_S = SMOKE["latency"]["instructor_benchmark_ms"] / 1000 if SMOKE else 0.074
    msg = f"Instructor benchmark: {BENCH_S:.5f} seconds -> CPU is {times.mean() / BENCH_S:.1f}x slower; the Space therefore runs on ZeroGPU"
    if SMOKE:
        msg += f", where the smoke test measured {SMOKE['latency']['gpu_bf16_512_no_mask']['avg_ms'] / 1000:.4f} s (bf16, 512 tokens)"
    print(msg + ".")
""")
code("""
# Compare with the recorded latencies: reports/latency.json (evaluate.py, GPU bf16 + CPU fp32) and the smoke test.
rows = []
lat = artifact("reports/latency.json", f"src/evaluate.py run --checkpoint checkpoints/finetune/<run>/best --split test,test3")
if lat:
    L = json.loads(lat.read_text())
    for k in ("gpu_bf16", "cpu_fp32"):
        if L.get(k):
            rows.append({"source": f"evaluate.py {k}", "avg s/sample": L[k]["avg_ms"] / 1000, "p50 s": L[k].get("p50_ms", np.nan) / 1000, "n": L[k].get("n")})
if SMOKE:
    SL = SMOKE["latency"]
    for k in ("gpu_bf16_512_no_mask", "gpu_bf16_512_bool_mask", "cpu_fp32_512", "cpu_fp32_128"):
        rows.append({"source": f"smoke_test {k}", "avg s/sample": SL[k]["avg_ms"] / 1000, "p50 s": SL[k]["p50_ms"] / 1000, "n": SL[k]["n"]})
if classifier is not None:
    rows.append({"source": "this notebook, CPU fp32 (100 holdout emails)", "avg s/sample": float(times.mean()), "p50 s": float(np.median(times)), "n": len(times)})
rows.append({"source": "instructor benchmark", "avg s/sample": SMOKE["latency"]["instructor_benchmark_ms"] / 1000 if SMOKE else 0.074, "p50 s": np.nan, "n": None})
show(pd.DataFrame(rows).set_index("source"), "Single-email prediction time (tokenize + forward + softmax)", fmt={"avg s/sample": "{:.4f}", "p50 s": "{:.4f}"})
""")
code("""
# Save the trained model: safetensors (what the Space loads) and torch.save of the same state_dict (the instructor's way).
NOTEBOOK_TMP = ROOT / ".notebook_tmp"          # project-local scratch (not /tmp, which is RAM-backed here); removed by the last cell
if classifier is not None:
    save_dir = NOTEBOOK_TMP / "save"
    shutil.rmtree(save_dir, ignore_errors=True); save_dir.mkdir(parents=True)
    sd = {k: v.contiguous() for k, v in classifier.state_dict().items()}
    SAFE_PATH, PTH_PATH = save_dir / "phishing_classifier.safetensors", save_dir / "phishing_classifier.pth"
    t0 = timer(); save_file(sd, str(SAFE_PATH), metadata={"format": "pt", "architecture": "EncoderForClassification"}); t_safe = timer() - t0
    t0 = timer(); torch.save(obj=sd, f=PTH_PATH); t_pth = timer() - t0
    print(f"Model saved to: {SAFE_PATH}  ({SAFE_PATH.stat().st_size / 2**20:,.0f} MiB, {t_safe:.1f} s)")
    print(f"Model saved to: {PTH_PATH}  ({PTH_PATH.stat().st_size / 2**20:,.0f} MiB, {t_pth:.1f} s)")
    back = load_file(str(SAFE_PATH)); pth = torch.load(PTH_PATH, map_location="cpu")
    same = back.keys() == pth.keys() and all(torch.equal(back[k], pth[k]) for k in back)
    print(f"safetensors and torch.save round-trip to identical tensors: {same}  ({len(back)} tensors). "
          f"The real export (src/export.py) writes bf16 safetensors (~675 MB) + config.json + tokenizer.json + model card.")
    shutil.rmtree(save_dir, ignore_errors=True)
""")

variant("colab")
md("""
**Step 5: Test Inference Speed and Save the Model**
The reference times `predict_and_time` on 100 images; the local notebook does the same on 100 TREC-06 emails on the
workstation CPU. Here the model is not present, so the timing is taken where the model actually serves: each of the app's
own synthetic example emails (read out of `app/app.py` in the bundle) is sent to the Space's `/predict` endpoint through
`gradio_client`, and the Space answers with the in-app **Prediction Time (s)** (tokenize + forward + softmax on the ZeroGPU,
the number the Gradio UI shows) next to the full round-trip from this runtime. The recorded workstation numbers
(`reports/latency.json`: RTX 3090 Ti bf16 and CPU fp32 on 100 holdout emails) and the instructor's 0.074 s benchmark are
shown for comparison. "Saving the model" happened on the workstation (`src/export.py` -> bf16 safetensors + config +
tokenizer + model card, then `scripts/upload_to_hf.py`); this notebook lists the model repo's files with their sizes instead
of writing a copy.
""")
code("""
import ast
def app_examples(path):
    \"\"\"The EXAMPLES list of app/app.py, read from the file TEXT: importing app.py would import torch and spaces at module level.\"\"\"
    for node in ast.parse(path.read_text()).body:
        targets = node.targets if isinstance(node, ast.Assign) else [node.target] if isinstance(node, ast.AnnAssign) else []
        if any(isinstance(t, ast.Name) and t.id == "EXAMPLES" for t in targets):
            return [[str(x[0]), str(x[1])] for x in ast.literal_eval(node.value)]
    raise ValueError("EXAMPLES not found in app.py")
FALLBACK_EMAILS = [   # three hard-coded synthetic emails (lure, spam, legitimate) if the extraction fails
    ["Action Required: verify your account within 24 hours",
     "Dear customer,\\n\\nUnusual sign-in activity was detected. Verify your account within 24 hours at http://secure-login-verify.example "
     "or it will be suspended.\\n\\nSecurity Team"],
    ["FINAL HOURS: 70% OFF designer watches + FREE shipping!!!",
     "CONGRATULATIONS! You were SELECTED for our VIP SALE. 70% OFF everything, FREE shipping TODAY ONLY. Shop now: http://bit.ly/vip-deal"],
    ["Agenda for Thursday's project sync",
     "Hi all,\\n\\nAttaching the agenda for Thursday's 10:00 sync: firewall migration status, Q4 hardware budget, onboarding checklist. "
     "Reply by Wednesday noon with additions.\\n\\nThanks,\\nNathan"],
]
app_py = artifact("app/app.py", "the Space source (app/ on the workstation, uploaded by scripts/upload_to_hf.py)")
try:
    SAMPLES = app_examples(app_py) if app_py else FALLBACK_EMAILS
    print(f"{len(SAMPLES)} synthetic example emails extracted from app/app.py (EXAMPLES)" if app_py else "using the 3 built-in synthetic emails")
except Exception as e:
    SAMPLES = FALLBACK_EMAILS
    print(f"could not extract EXAMPLES from app.py ({type(e).__name__}): using the 3 built-in synthetic emails")
for subject, body in SAMPLES:
    print(f"  - {subject[:72]!r}  ({len(body)} chars)")
""")
code("""
# Inference speed where the model runs: the Space's /predict endpoint, one call per email, through gradio_client.
from gradio_client import Client
SPACE_STATS = []
client = None
try:
    t0 = timer()
    client = Client(SPACE, token=TOKEN, verbose=False, analytics_enabled=False)
    print(f"connected to {SPACE} in {timer() - t0:.1f} s  (endpoint /predict: subject, body -> label, Prediction Time (s), highlights, signals)")
except Exception as e:
    print(f"cannot connect to the Space ({type(e).__name__}: {str(e)[:160]}): private Space without a token, or the Space is not running")
if client is not None:
    for subject, body in SAMPLES:
        t0 = timer()
        try:
            label, secs, highlights, signals = client.predict(subject, body, api_name="/predict")
        except Exception as e:     # ZeroGPU quota / "GPU task aborted" / timeout: skip this email, keep the rest
            print(f"\\n=== {subject}\\n  Space call failed ({type(e).__name__}: {str(e)[:160]}); skipped")
            continue
        round_trip = timer() - t0
        conf = {c["label"]: float(c["confidence"]) for c in (label.get("confidences") or [])} if isinstance(label, dict) else {}
        verdict = label.get("label") if isinstance(label, dict) else str(label)
        first_line = (str(signals).strip().splitlines() or [""])[0]
        SPACE_STATS.append({"subject": subject[:52], "verdict": verdict, **{f"P({k})": v for k, v in conf.items()},
                            "Prediction Time (s), on the Space": float(secs), "round-trip (s)": round_trip})
        print(f"\\n=== {subject}\\n  verdict {verdict}: " + ", ".join(f"{k} {100 * v:.1f}%" for k, v in conf.items())
              + f"\\n  Prediction Time (s): {float(secs):.3f} on the Space   round-trip from here: {round_trip:.2f} s   {first_line}")
if SPACE_STATS:
    SP = pd.DataFrame(SPACE_STATS).set_index("subject")
    show(SP, f"{len(SP)} synthetic emails through the Space's /predict endpoint", fmt={c: "{:.3f}" for c in SP.columns if c != "verdict"})
    t_app, t_rt = SP["Prediction Time (s), on the Space"], SP["round-trip (s)"]
    print(f"Average in-app prediction time: {t_app.mean():.4f} s on the Space's GPU ({SPACE_HW}; the first call, {t_app.iloc[0]:.3f} s, "
          f"includes the ZeroGPU warm-up); average round-trip {t_rt.mean():.2f} s (queue + network + the app's saliency pass). "
          f"Instructor benchmark: 0.074 s.")
elif client is not None:
    print("no successful Space call: every request was rejected (ZeroGPU quota / Space restarting); the recorded workstation latencies follow")
""")
code("""
# The recorded workstation latencies (reports/latency.json: evaluate.py on 100 TREC-06 holdout emails) next to the Space numbers.
lat = artifact("reports/latency.json", "src/evaluate.py run --checkpoint checkpoints/finetune/<run>/best --split test,test3")
BENCH_S = 0.074
rows = []
if lat:
    L = json.loads(lat.read_text()); BENCH_S = L.get("instructor_benchmark_ms", 74.0) / 1000
    for k, what in (("gpu_bf16", "RTX 3090 Ti bf16"), ("cpu_fp32", "CPU fp32, 16 threads")):
        if L.get(k):
            rows.append({"source": f"workstation {what} (evaluate.py, {L[k]['n']} holdout emails)", "avg s/sample": L[k]["avg_ms"] / 1000,
                         "p50 s": L[k].get("p50_ms", np.nan) / 1000, "n": L[k]["n"]})
if SPACE_STATS:
    rows.append({"source": f"Space {SPACE_HW}, in-app Prediction Time (this notebook, synthetic emails)", "avg s/sample": float(t_app.mean()),
                 "p50 s": float(t_app.median()), "n": len(t_app)})
    rows.append({"source": "Space, full round-trip from this runtime", "avg s/sample": float(t_rt.mean()), "p50 s": float(t_rt.median()), "n": len(t_rt)})
rows.append({"source": "instructor benchmark", "avg s/sample": BENCH_S, "p50 s": np.nan, "n": None})
show(pd.DataFrame(rows).set_index("source"), "Single-email prediction time (tokenize + forward + softmax)", fmt={"avg s/sample": "{:.4f}", "p50 s": "{:.4f}"})
if lat:
    print(f"workstation GPU {L['gpu_bf16']['avg_ms']:.1f} ms = {L['gpu_bf16']['ratio_to_benchmark']:.2f}x the {BENCH_S * 1000:.0f} ms benchmark; "
          f"CPU {L['cpu_fp32']['avg_ms']:.0f} ms = {L['cpu_fp32']['ratio_to_benchmark']:.1f}x: hence a ZeroGPU Space rather than a CPU one.")
""")
code("""
# "Save the model": the export already lives on the Hub. The model repo's own files (what the Space and anyone with access load).
info = api.repo_info(MODEL_REPO, repo_type="model", files_metadata=True)
own = [s for s in info.siblings if not s.rfilename.startswith(PRESENTATION_PREFIX + "/")]
bundle = [s for s in info.siblings if s.rfilename.startswith(PRESENTATION_PREFIX + "/")]
show(pd.DataFrame([{"file": s.rfilename, "MiB": (s.size or 0) / 2**20, "LFS": s.lfs is not None} for s in own]).set_index("file"),
     f"https://huggingface.co/{MODEL_REPO}  (private: {info.private}, last modified {str(info.last_modified)[:16]})", fmt={"MiB": "{:,.1f}"})
print(f"+ {len(bundle)} files / {sum((s.size or 0) for s in bundle) / 2**20:.1f} MiB under {PRESENTATION_PREFIX}/ (this notebook's inputs)")
fc = EXPORT_CONFIG["finetune_checkpoint"]
print(f"model.safetensors: {EXPORT_CONFIG['export_dtype']} export of checkpoints/finetune/{Path(EXPORT_CONFIG['source_checkpoint']).parent.name}/best "
      f"({fc['params']['total']:,} parameters, {EXPORT_CONFIG['num_labels']} classes {fc['label_names']}), exported {EXPORT_CONFIG['exported']}; "
      f"config.json carries the test metrics: TREC-06 F1 {EXPORT_CONFIG['metrics']['f1']:.4f}, recall {EXPORT_CONFIG['metrics']['recall']:.4f}, "
      f"ROC-AUC {EXPORT_CONFIG['metrics']['roc_auc']:.4f}; three-way macro-F1 {EXPORT_CONFIG['metrics_threeway']['macro_f1']:.4f}.")
""")

# =============================================================================== Step 6
variant("local")
md("""
**Step 6: Create the App Files (Optional)**
Optional in the reference, not here: the deployed Space is one of the project's deliverables. The reference writes
`app.py`, `model.py` and `requirements.txt` into a deployment folder. Ours live in `app/`:
`app.py` (the Gradio interface, a `gr.Interface` with a `gr.Label` and the timed `gr.Number("Prediction Time (s)")`
plus token-saliency highlights and rule-based cues), `requirements.txt` pinned for the ZeroGPU runtime (torch 2.13.0,
Python 3.12.12: the highest versions ZeroGPU allows), `README.md` with the Space front matter, and `model.py`, which
`src/export.py` copies **byte-for-byte** from `src/model.py` next to the exported weights.
""")
code("""
APP_DIR = Path(CFG["paths"]["app_dir"])
print(f"{APP_DIR.relative_to(ROOT)}/")
for p in sorted(APP_DIR.iterdir()):
    if p.name == "__pycache__":
        continue
    print(f"  {p.name:<20} {p.stat().st_size:>12,} bytes")
print("\\n--- app/app.py (first 60 lines) ---")
print("\\n".join((APP_DIR / "app.py").read_text().splitlines()[:60]))
""")
code("""
print("--- app/requirements.txt ---"); print((APP_DIR / "requirements.txt").read_text())
readme = (APP_DIR / "README.md").read_text().split("---")
print("--- app/README.md front matter (Space configuration) ---"); print(readme[1].strip())
""")
code("""
# model.py in the Space must be the byte-identical file the model was trained with (export.py asserts the sha1).
src_sha = hashlib.sha1((ROOT / "src/model.py").read_bytes()).hexdigest()
print(f"src/model.py  sha1 {src_sha}")
app_model = artifact("app/model.py", "src/export.py run --checkpoint checkpoints/finetune/<run>/best  (copies src/model.py byte-for-byte)")
if app_model:
    app_sha = hashlib.sha1(app_model.read_bytes()).hexdigest()
    print(f"app/model.py  sha1 {app_sha}  identical: {app_sha == src_sha}")
for f in ("model.safetensors", "config.json", "tokenizer.json"):
    artifact(f"app/{f}", "src/export.py run (copied next to app.py so the Space can also load weights locally)")
""")

variant("colab")
md("""
**Step 6: Create the App Files (Optional)**
Optional in the reference, not here: the Space is one of the project's deliverables. The files the reference writes
(`app.py`, `model.py`, `requirements.txt`) are the ones `scripts/upload_to_hf.py` pushed to the Space, together with the bf16
weights, `config.json` and `tokenizer.json` (bundled, so the Space needs no download and no secret at startup). The Space's
file list comes from the Hub; the first lines of `app.py`, the pinned `requirements.txt` and the Space card's front matter are
read from the bundle's copy of `app/`.
""")
code("""
try:
    space_files = api.list_repo_files(SPACE, repo_type="space")
    print(f"Space https://huggingface.co/spaces/{SPACE} files ({len(space_files)}): " + ", ".join(space_files))
except Exception as e:
    print(f"Space file list not readable ({type(e).__name__}): private Space without a token")
print("\\n--- app/app.py (first 40 lines) ---")
if app_py:
    print("\\n".join(app_py.read_text().splitlines()[:40]))
req = artifact("app/requirements.txt", "app/ on the workstation (pinned for the ZeroGPU runtime)")
if req:
    print("\\n--- app/requirements.txt ---"); print(req.read_text())
card = artifact("app/README.md", "app/ on the workstation (the Space card)")
if card:
    print("--- app/README.md front matter (Space configuration) ---"); print(card.read_text().split("---")[1].strip())
""")

# =============================================================================== Step 7
variant("local")
md("""
**Step 7: Launch the Gradio Interface**
The reference launches the demo with `share=True` to get a public link; `gradio.live` links expire after 72 hours, so the
permanent link for the class is the Hugging Face Space (Step 9). Here the real `app/app.py` is imported and its `predict`
function (the function the Gradio `Interface` calls) is run on two of the app's built-in **synthetic** example emails,
printing the three class probabilities and the prediction time. `demo.launch()` is behind a flag that is off by default
so the notebook executes unattended; flip `LAUNCH_GRADIO = True` to open the interface locally.
""")
code("""
import importlib.util
def stage_model_dir():
    \"\"\"Directory with model.safetensors + config.json + tokenizer.json for app.py (export dir, or a staged dev checkpoint).\"\"\"
    exports = sorted((ROOT / "model_export").glob("*/model.safetensors")) if (ROOT / "model_export").exists() else []
    if exports:
        return exports[-1].parent
    if CLS_DIR is None:
        return None
    stage = NOTEBOOK_TMP / "stage"          # project-local; symlinks only; removed by the last cell
    stage.mkdir(parents=True, exist_ok=True)
    for name, src in (("model.safetensors", CLS_DIR / "model.safetensors"), ("config.json", CLS_DIR / "config.json"),
                      ("tokenizer.json", Path(CFG["paths"]["tokenizer"]))):
        dst = stage / name
        if dst.is_symlink() or dst.exists():
            dst.unlink()
        dst.symlink_to(src)
    return stage
app = None
model_dir = stage_model_dir()
if model_dir is None:
    print("not available yet — app needs a classifier (produced by: src/finetune.py train, then src/export.py run)")
else:
    os.environ["PHISH_MODEL_DIR"] = str(model_dir)        # app.py loads from here; otherwise it would download from the private Hub repo
    print(f"PHISH_MODEL_DIR={model_dir}  ({CLS_KIND})")
    spec = importlib.util.spec_from_file_location("phish_app", APP_DIR / "app.py")
    app = importlib.util.module_from_spec(spec); spec.loader.exec_module(app)   # builds `app.demo` (gr.Interface) without launching
    print(f"\\napp loaded: {app.N_PARAMS:,} params on {app.DEVICE} ({next(app.MODEL.parameters()).dtype}), classes {app.LABELS}, max_len {app.MAX_LEN}")
    for subject, body in (app.EXAMPLES[0], app.EXAMPLES[4]):
        label_probs, secs, highlights, signals = app.predict(subject, body)
        words = [t for t, l in highlights if l]
        print(f"\\n=== {subject}\\n  probabilities: " + ", ".join(f"{k} {100 * v:.1f}%" for k, v in label_probs.items())
              + f"\\n  Prediction Time (s): {secs}   verdict line: {signals.splitlines()[0]}\\n  tokens weighed most: {words[:8]}")
""")
code("""
LAUNCH_GRADIO = False    # share links (gradio.live) expire after 72 h; the permanent link is the Space in Step 9
if LAUNCH_GRADIO and app is not None:
    app.demo.launch(share=False)
else:
    print("Gradio not launched here (LAUNCH_GRADIO = False). Locally: PHISH_MODEL_DIR=model_export/<run> .venv/bin/python app/app.py")
""")

variant("colab")
md("""
**Step 7: Launch the Gradio Interface**
The reference calls `demo.launch(share=True)`. Here the interface *is* the deployed Space: `gr.load(SPACE, src="spaces")`
builds a Gradio app whose every call is proxied to the Space (the model never leaves ZeroGPU), and `launch()` renders it
inline in Colab (Colab shows Gradio through a temporary public link). Gradio 6.29.1 has a bug in `gr.load` for Spaces whose
`gr.Interface` carries an examples table: `Blocks.from_config` rebuilds the `Dataset` component from `{name, class_id}`
dicts and hands `class_id` on to the child `Textbox.__init__`, which rejects it. The cell first calls `gr.load` as is; if
it fails with exactly that `TypeError`, a four-line compatibility shim (drop `class_id` before the child component is
instantiated, the same thing newer Gradio releases do) is installed and `gr.load` is retried, so the interface shown is the
Space's own. Only if `gr.load` still fails does the cell fall back to a hand-built `gr.Interface` around the Step 5
`gradio_client` call. Outside Colab the cell starts and immediately stops a headless server as a smoke test. When the Space
is public, the native page is also embedded in an iframe; while it is private the iframe would only show a login page, so it
is skipped with a note.
""")
code("""
import io, contextlib, warnings
import gradio as gr
def shim_gr_load_dataset_bug():
    \"\"\"gradio 6.29.1: gr.load -> Blocks.from_config rebuilds a Space's examples `Dataset` from {name, class_id} dicts and
    get_component_instance passes `class_id` on to the child component's __init__, which rejects it. Drop that key.\"\"\"
    import gradio.components.dataset as ds
    if getattr(ds.get_component_instance, "_class_id_shim", False):
        return
    original = ds.get_component_instance
    def patched(comp, *args, **kwargs):
        if isinstance(comp, dict) and "class_id" in comp:
            comp = {k: v for k, v in comp.items() if k != "class_id"}
        return original(comp, *args, **kwargs)
    patched._class_id_shim = True
    ds.get_component_instance = patched
def load_space():
    with warnings.catch_warnings(), contextlib.redirect_stdout(io.StringIO()):   # hide gradio's chatter + theme UserWarning
        warnings.simplefilter("ignore")
        return gr.load(SPACE, src="spaces", token=TOKEN)
demo, demo_kind = None, None
try:
    try:
        demo = load_space()
        demo_kind = "gr.load(src='spaces'): the Space's own interface, every call proxied"
    except TypeError as e:
        if "class_id" not in str(e):
            raise
        print(f"gr.load hit the gradio {gr.__version__} Dataset/class_id bug ({str(e)[:90]}); installing the compat shim and retrying")
        shim_gr_load_dataset_bug()
        demo = load_space()
        demo_kind = "gr.load(src='spaces') with the class_id compat shim: the Space's own interface, every call proxied"
except Exception as e:
    print(f"gr.load({SPACE!r}) failed: {type(e).__name__}: {str(e)[:160]}  -> hand-built proxy interface")
if demo is None and client is not None:
    # Fallback: the same interface built by hand around the gradio_client connection of Step 5 (labels mirror app/app.py).
    def proxy_predict(subject, body):
        label, secs, highlights, signals = client.predict(subject, body, api_name="/predict")
        confidences = {c["label"]: c["confidence"] for c in label["confidences"]} if isinstance(label, dict) and "confidences" in label else label
        spans = [(h["token"], h.get("class_or_confidence")) if isinstance(h, dict) else tuple(h) for h in highlights]
        return confidences, secs, spans, signals
    demo = gr.Interface(
        fn=proxy_predict,
        inputs=[gr.Textbox(label="Subject", lines=1), gr.Textbox(label="Email body", lines=14)],
        outputs=[gr.Label(label="Prediction"), gr.Number(label="Prediction Time (s)"),
                 gr.HighlightedText(label="Tokens the model weighed most", color_map={"high": "#d62728", "medium": "#ff9896"},
                                    combine_adjacent=True, show_label=True),
                 gr.Markdown(label="Signals")],
        examples=SAMPLES, cache_examples=False, title="BITS Phishing Email Detector (proxied to the Space)",
        description=f"Every prediction below runs on {SPACE} ({SPACE_HW}); this runtime only forwards the request.")
    demo_kind = "gr.Interface around gradio_client (gr.load fallback), every call proxied"
if demo is None:
    print("no interface: the Space is unreachable from this runtime (private without a token, or not running)")
else:
    print("interface:", demo_kind)
if demo is not None:
    if IN_COLAB:
        demo.launch()                       # inline in Colab; every prediction is proxied to the Space
    else:
        demo.launch(prevent_thread_lock=True, inline=False, share=False, quiet=True)
        print(f"headless smoke test: the proxied interface was served at {demo.local_url} and is closed again")
        demo.close()
""")
code("""
from IPython.display import IFrame
try:
    space_public = api.space_info(SPACE).private is False
except Exception:
    space_public = False
if space_public:
    display(IFrame(SPACE_HOST, "100%", 900))
else:
    print(f"The iframe embed needs a public Space; {SPACE} is private right now. Flip it to public in the Hub settings before class "
          f"and the native page appears here: {SPACE_HOST}")
""")

# =============================================================================== Step 8
variant("local")
md("""
**Step 8: Authenticate with Hugging Face**
The reference calls `notebook_login()` and pastes a write token into a widget. We never put a token in a notebook: the
workstation is logged in once with `hf auth login` (huggingface_hub 1.33.0's CLI), the token lives in the CLI cache, and
every script reads it from there or from the `HF_TOKEN` environment variable without printing it. This cell only asks the
Hub who is logged in; it degrades to "offline" without network access.
""")
code("""
import logging; logging.getLogger("httpx").setLevel(logging.WARNING)
try:
    from huggingface_hub import HfApi
    print("Logged in to Hugging Face as:", HfApi().whoami()["name"])
except Exception as e:   # no network, or not logged in: the notebook still runs
    print(f"offline / not logged in ({type(e).__name__}); run `hf auth login` once on this machine.")
print("The token is never stored in this notebook or in the repository (.gitignore excludes *.token and .env).")
""")

variant("colab")
md("""
**Step 8: Authenticate with Hugging Face**
The reference pastes a write token into `notebook_login()`. Here the token came from the Colab secret (or the environment /
login cache) in Step 1 and every Hub call in this notebook passes it explicitly, so `login()` is only needed when the token is
not yet the one `huggingface_hub` resolves on its own (in huggingface_hub 1.33.0 `login(token=...)` always validates with
`whoami` and re-saves the token to the runtime's HF cache; `skip_if_logged_in` only governs the interactive flows). When the
token already comes from the login cache or the Colab secret, nothing is written. The only thing printed is the account name
the Hub answers with: the token value never appears in the notebook, its outputs or the repository.
""")
code("""
from huggingface_hub import login
if TOKEN and TOKEN != huggingface_hub.get_token():
    login(token=TOKEN, add_to_git_credential=False)     # validates with whoami and saves the token in this runtime's HF cache (Colab: ephemeral disk)
    print("token saved for this runtime's huggingface_hub")
elif TOKEN:
    print("token already active for huggingface_hub (login cache / Colab secret): nothing written")
else:
    print("no token found (anonymous): the private repos stay unreadable until they are public")
try:
    print("Logged in to Hugging Face as:", HfApi(token=TOKEN).whoami()["name"])
except Exception as e:
    print(f"whoami failed ({type(e).__name__}): no token / offline")
print("The token is never stored in this notebook or in the repository.")
""")

# =============================================================================== Step 9
variant("local")
md("""
**Step 9: Upload the App to Hugging Face Spaces**
The reference uploads a folder with `HfApi().upload_folder`. Ours is `scripts/upload_to_hf.py`: it creates the **private**
model repo and the **private** ZeroGPU Space named in `config.yaml`, uploads `model_export/<run>/` (bf16 safetensors, config,
tokenizer, model card) to the model repo and `app/` to the Space, sets the `HF_TOKEN` read secret and `PHISH_MODEL_REPO`
variable so the Space can download the private weights at startup, and prints both URLs. It is a **dry run by default** and
refuses to write anything without `--yes`; the live upload is run by Nathan from the terminal, never from the notebook.
Both repos start private and are switched to public for the presentation in the Hub settings.
""")
code("""
hf = CFG["hf"]
print(f"model repo : https://huggingface.co/{hf['model_repo']}            (private: {hf['private']})")
print(f"Space      : https://huggingface.co/spaces/{hf['space_repo']}      (sdk {hf['space_sdk']} {hf['space_sdk_version']}, "
      f"hardware {hf['space_hardware']}, python {hf['space_python_version']}, torch {hf['space_torch']})\\n")
cmd = [sys.executable, str(ROOT / "scripts/upload_to_hf.py"), "--dry-run"]
print("$", " ".join(Path(c).name if c.startswith(str(ROOT)) else c for c in cmd))
res = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, timeout=180, env={**os.environ, "CUDA_VISIBLE_DEVICES": ""})
print((res.stdout + res.stderr).strip()[-4000:])
print("\\nLive run (Nathan, after approval):  HF_TOKEN_FOR_SPACE=<read token> .venv/bin/python scripts/upload_to_hf.py --model-dir model_export/<run> --yes")
""")

variant("colab")
md("""
**Step 9: Upload the App to Hugging Face Spaces**
The reference uploads a folder with `HfApi().upload_folder`. Ours is `scripts/upload_to_hf.py` (in the bundle under
`scripts/`): it creates the private model repo and the private ZeroGPU Space named in `config.yaml`, uploads
`model_export/ft-focal/` and `app/` (with the weights bundled into the Space, so no secret is needed) and prints both URLs.
It is a dry run unless `--yes` is given, and the live run was done once from the workstation terminal (RUN_LOG entry 24),
never from a notebook. This cell shows the command that was run, the two URLs, the Space's current runtime and the log entry.
""")
code("""
hf = CFG["hf"]
print("Live upload (run once from the workstation terminal, 2026-10-12, RUN_LOG entry 24):")
print("    .venv/bin/python scripts/upload_to_hf.py --yes --model-dir model_export/ft-focal --bundle-weights\\n")
print(f"model repo : https://huggingface.co/{MODEL_REPO}")
print(f"Space      : https://huggingface.co/spaces/{SPACE}   (sdk {hf['space_sdk']} {hf['space_sdk_version']}, hardware {hf['space_hardware']}, "
      f"python {hf['space_python_version']}, torch {hf['space_torch']})")
try:
    rt = api.get_space_runtime(SPACE)
    print(f"Space runtime now: stage {rt.stage}, hardware {rt.hardware}")
except Exception as e:
    print(f"Space runtime not readable ({type(e).__name__})")
log = artifact("RUN_LOG.md", "kept by hand on the workstation and mirrored into the bundle")
if log:
    lines = log.read_text().splitlines()
    start = next((i for i, l in enumerate(lines) if l.startswith("## 24.")), None)
    if start is None:
        print("RUN_LOG entry 24 not found")
    else:
        para = []
        for l in lines[start + 1:]:          # the first paragraph = the first non-empty block after the header
            if l.strip():
                para.append(l)
            elif para:
                break
        display(Markdown(f"> **{lines[start].lstrip('# ')}**\\n>\\n> " + "\\n> ".join(para)))
""")
variant(None)

# =============================================================================== Results
md("""
## Results
Performance on the **unseen** test sets, written by `src/evaluate.py` (the only script allowed to open the holdouts).
The headline is the binary roll-up on TREC-06: P(unwanted) = P(phishing) + P(spam) at threshold 0.5, with recall on the
unwanted class first (a missed phish is the costly error). The three-way holdout gives the 3x3 confusion matrix; the
phishing-vs-spam boundary follows corpus provenance and is expected to be softer than legitimate-vs-unwanted.
""")
code("""
def metric_tables(path, title):
    M = json.loads(path.read_text())
    at, tf = M.get("at_threshold", {}), M.get("threshold_free", {})
    print(f"{title}: checkpoint {M.get('checkpoint')}, {M['split_sizes']['rows']:,} rows, positive rate {M['split_sizes']['positive_rate']}, threshold {M.get('threshold')}")
    binary = {"recall (unwanted)": at.get("recall"), "precision": at.get("precision"), "F1": at.get("f1"), "accuracy": at.get("accuracy"),
              "specificity": at.get("specificity"), "ROC-AUC": tf.get("roc_auc"), "PR-AUC": tf.get("pr_auc"),
              "recall @ 1% FPR": (tf.get("recall_at_fpr_1pct") or {}).get("recall"), "recall @ 0.1% FPR": (tf.get("recall_at_fpr_0_1pct") or {}).get("recall")}
    show(pd.DataFrame(binary, index=["value"]).T, f"{title}: binary roll-up", fmt="{:.4f}")
    cm = at.get("confusion_matrix")
    if cm:
        show(pd.DataFrame([[cm["tn"], cm["fp"]], [cm["fn"], cm["tp"]]], index=["true legitimate", "true unwanted"],
                          columns=["pred legitimate", "pred unwanted"]), "Binary confusion matrix")
    lure = (M.get("credential_lure_slice") or {}).get("metrics")
    if lure:
        show(pd.DataFrame({k: lure.get(k) for k in ("n", "n_positive", "recall", "precision", "f1", "accuracy")}, index=["credential-lure slice"]).T, "Credential-lure slice")
    tw = M.get("threeway")
    if tw and tw.get("confusion_matrix"):
        labs = tw["confusion_matrix"]["labels"]
        show(pd.DataFrame(tw["confusion_matrix"]["matrix"], index=[f"true {l}" for l in labs], columns=[f"pred {l}" for l in labs]), "3x3 confusion matrix")
        show(pd.DataFrame(tw["per_class"]).T, f"Per-class metrics (macro-F1 {tw.get('macro_f1'):.4f})" if tw.get("macro_f1") is not None else "Per-class metrics")
    return M
RESULTS = {}
for rel, title in (("reports/metrics.json", "TREC-06 binary holdout"), ("reports/metrics_test3.json", "Three-way holdout")):
    p = artifact(rel, "src/evaluate.py run --checkpoint checkpoints/finetune/ft-focal/best --split test,test3  (main run ft-focal, selected on validation F1)")
    if p:
        RESULTS[rel] = metric_tables(p, title)
""")
code("""
# Ablations on the SAME holdouts: run_experiments.sh evaluates every queued run into reports/eval_<run>/ (one directory per
# run, so no report overwrites another); the main run (ft-focal: best validation F1 of the unwanted roll-up and best validation
# three-way macro-F1 among the six, chosen before looking at the test numbers) is also written to the top-level reports/ shown above.
abl = []
for d in sorted((ROOT / "reports").glob("eval_*")):
    mj, m3 = d / "metrics.json", d / "metrics_test3.json"
    if mj.exists():
        M = json.loads(mj.read_text()); at, tf = M.get("at_threshold", {}), M.get("threshold_free", {})
        row = {"run": d.name[len("eval_"):], "TREC-06 recall": at.get("recall"), "precision": at.get("precision"), "F1": at.get("f1"),
               "ROC-AUC": tf.get("roc_auc"), "PR-AUC": tf.get("pr_auc"), "recall @ 1% FPR": (tf.get("recall_at_fpr_1pct") or {}).get("recall")}
        if m3.exists():
            tw = json.loads(m3.read_text()).get("threeway") or {}
            row["3-way macro-F1"] = tw.get("macro_f1"); row["3-way accuracy"] = tw.get("accuracy")
        abl.append(row)
if abl:
    show(pd.DataFrame(abl).set_index("run").sort_values("F1", ascending=False), "Ablations on the unseen holdouts (reports/eval_<run>/)", fmt="{:.4f}")
else:
    artifact("reports/eval_<run>/metrics.json", "scripts/run_experiments.sh -> src/evaluate.py run --out-dir reports/eval_<run> (one per queued run)")
""")
code("""
for rel in ("reports/confusion_matrix.png", "reports/roc_curve.png", "reports/pr_curve.png", "reports/confusion_matrix3_test3.png",
            "reports/confusion_matrix_test3.png"):
    p = artifact(rel, "src/evaluate.py run --split test,test3")
    if p and p.stat().st_size < 2_000_000:
        display(Image(filename=str(p), width=520))
for rel in ("reports/error_analysis.md", "reports/error_analysis_test3.md"):
    p = artifact(rel, "src/evaluate.py run --split test,test3")
    if p:
        display(Markdown("\\n".join(p.read_text().splitlines()[:40]) + "\\n\\n*(truncated; full file: " + rel + ")*"))
""")
code("""
# Latency as recorded by evaluate.py (GPU bf16 and CPU fp32 on the same 100 holdout emails) next to the benchmark.
if lat:
    L = json.loads(lat.read_text())
    show(pd.DataFrame({k: v for k, v in L.items() if isinstance(v, dict)}).T, f"reports/latency.json (benchmark {L.get('instructor_benchmark_ms')} ms)")
else:
    SL = SMOKE.get("latency", {})
    g, c = SL.get("gpu_bf16_512_no_mask", {}).get("avg_ms"), SL.get("cpu_fp32_512", {}).get("avg_ms")
    print("latency.json not written yet (produced by: scripts/run_experiments.sh -> src/evaluate.py run); the measured numbers so far are in the "
          f"Step 5 table (smoke test GPU {g} ms bf16, CPU {c} ms fp32 at 512 tokens).")
""")

# =============================================================================== Challenges
md("""
## Challenges and what we learned
Distilled from `RUN_LOG.md` (entries 0-9), which records every decision, number and problem.
- **Python 3.14 wheels.** The workstation runs Python 3.14.7; every pin (torch 2.14.1 cu130, tokenizers, safetensors, pyarrow, ...) had to be checked live for a cp314 wheel before the venv was built. The plain PyPI torch wheel *is* the CUDA 13.0 build, and driver 615 (CUDA 13.4) runs any 13.x build by backward compatibility; `uv --torch-backend=auto` would nevertheless have resolved an unverified cu132 build instead of the verified cu130 wheel (uv 0.12.5 iterates cu132 before cu130 for drivers >= 580), so the install pins `--torch-backend=cu130`.
- **ZeroGPU pins the Space.** ZeroGPU supports only torch 2.8-2.13 on Python 3.12.12, no `torch.compile`, and cannot preload a private model repo, so the Space uses torch 2.13.0 and downloads the weights at startup with an `HF_TOKEN` secret. Training (torch 2.14.1 / 3.14) and serving (2.13.0 / 3.12) are different stacks; safetensors makes that safe.
- **Datasets without body text.** `it4lia/...Cleaned` turned out to hold only subjects and 53 numeric features; `zefang-liu/...` was a lossy 92% copy of the primary set with an "enron" shortcut. Both were excluded before downloading anything.
- **Hub-split leakage.** The primary set's own train/test split has 5.1% exact and ~40% near-duplicate test rows in train; we built our own split and asserted zero leakage with independent recomputation.
- **Enron contamination.** 89% of TREC-05 ham and 86% of Enron-Spam ham are in the pretraining corpus, so neither could be the test set; TREC-06 (5 rows mention Enron) became the holdout, and 1,652 training rows that near-duplicate TREC-06 spam templates were removed.
- **Near-duplicate decontamination.** Exact-key dedup missed quoted/forwarded 419-letter variants of holdout emails in the MLM corpus; the reviewer caught it, and 50-char shingle matching (>= 3 shared) now gates every split. 52.7% of `corbt/enron-emails` was exact duplicates.
- **VRAM finder bug.** The first batch finder measured peak memory with one forward/backward and under-reported the real gradient-accumulation peak by one fp32 gradient copy (1.26 GiB); fixed before the long run. Checkpointing turned out unnecessary (B=16 fits with 8 GiB headroom) and costs ~25% throughput.
- **Effective batch 264, not 256.** Micro-batch 24 does not divide 256; floor division would have reported 256 while training at 240. The trainer now uses ceil(256/24) = 11 accumulation steps and records the realised 264 everywhere, including the LR schedule.
- **CPU latency reality.** The research estimate of ~0.34 s per 512-token email on the 5950X was 2x optimistic: measured 0.70 s (0.32 s at 128 tokens) because CPU inference is per-op-overhead bound. The 0.074 s benchmark is met on GPU (10.7 ms), hence ZeroGPU rather than a free CPU Space.
- **The 300k three-class set was rejected.** Inspection showed it is a union of Kaggle re-packagings of corpora we already hold, 9.9% of it copies of our frozen TREC-06 test set, with a "phish" label that agrees with real provenance only 60% of the time and a stopword-stripping artifact that would be a near-perfect label proxy. Not one row was ingested.
- **Three-way label scarcity.** No public corpus separates spam from phishing at scale: only 3,150 clean phishing training rows (Nazario, Nigerian fraud) plus 1,000 synthetic lures carry the distinction, while 62,805 TREC/CEAS positives are trained as "unwanted" with a partial-label loss. The binary roll-up on TREC-06 is the headline; the three-way split is the weaker decision and is reported as such.
- **Reviewer-caught trainer bugs** (fixed before any real run): an `--lr` override silently discarded on resume but logged as applied; a resumed finished run training one extra step at lr 0; orphan metric rows after a hard crash; 0 occurrences of `[CLS]` in the packed MLM stream (every window now starts with `[CLS]`, so the classifier's pooled position is pretrained).
- **DataLoader migration with an RNG side effect.** The course requires `torch.utils.data.DataLoader` for batching and shuffling; `finetune.py` had its own numpy sampler. Moving every batch through `DataLoader` (row-index Dataset + seeded length-bucketed `batch_sampler` + padding collate, Step 2) was verified loss-identical at every step on the CPU fixture, except that iterating a DataLoader draws its worker base seed from the global torch RNG and shifted the dropout stream (step-1 loss 1.1093 vs 1.1125). Passing an explicit `generator` removed the side effect.
- **Self-test artifacts are not results.** The agents' dev checkpoints (scratch init, a handful of steps on 2,000 rows) and their `experiments.jsonl` lines were deleted before the real queue was armed, so nothing in `reports/` or `checkpoints/` can be mistaken for a result; this notebook labels anything named `dev*` and shows "not available yet" until the queue writes the real artifacts.
""")
variant("local")
code("""
shutil.rmtree(NOTEBOOK_TMP, ignore_errors=True)      # the save round-trip files and the staged symlinks (Step 5 / Step 7)
elapsed = timer() - T_NOTEBOOK_START
print(f"Notebook executed top-to-bottom in {elapsed:.0f} s on {device}; scratch dir {NOTEBOOK_TMP.name} removed: {not NOTEBOOK_TMP.exists()}.")
if MISSING:
    print(f"\\n{len(MISSING)} artifact(s) not available yet (they appear as the pipeline advances):")
    for rel, by in dict(MISSING).items():
        print(f"  - {rel}\\n      produced by: {by}")
else:
    print("Every artifact was present: this is the final run.")
""")
variant("colab")
code("""
elapsed = timer() - T_NOTEBOOK_START
print(f"Notebook executed top-to-bottom in {elapsed:.0f} s (in Colab: {IN_COLAB}); model compute on the Space {SPACE}, "
      f"artifacts from {MODEL_REPO}/{PRESENTATION_PREFIX}.")
if MISSING:
    print(f"\\n{len(MISSING)} artifact(s) not in the bundle:")
    for rel, by in dict(MISSING).items():
        print(f"  - {rel}\\n      produced by: {by}")
else:
    print("Every artifact of the bundle was present.")
""")
variant(None)


def cells_for(colab: bool) -> list[tuple[str, str]]:
    """The (kind, source) cells of one variant, in order."""
    keep = "colab" if colab else "local"
    return [(kind, text) for kind, text, tag in CELLS if tag is None or tag == keep]


def build(colab: bool = False) -> nbformat.NotebookNode:
    nb = new_notebook()
    nb.metadata = {
        "kernelspec": {"display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3"},
        "language_info": {"name": "python"},
    }
    for i, (kind, text) in enumerate(cells_for(colab)):
        cell = new_markdown_cell(text) if kind == "markdown" else new_code_cell(text)
        cell["id"] = f"cell-{i:02d}"   # deterministic ids (nbformat would draw random ones), so rebuilds are byte-identical
        nb.cells.append(cell)
    return nb


def main(argv: list[str] | None = None) -> int:
    ap = argparse.ArgumentParser(description="Build the project notebook (local variant by default).")
    ap.add_argument("--colab", action="store_true", help="build the Colab variant (bundle from the Hub, compute on the Space)")
    ap.add_argument("--out", type=Path, default=None, help="output path (default: FinalProject_PhishingDetector[_Colab].ipynb)")
    args = ap.parse_args(argv)
    out = args.out or (OUT_COLAB if args.colab else OUT)
    nb = build(colab=args.colab)
    nbformat.validate(nb)
    nbformat.write(nb, out)
    cells = cells_for(args.colab)
    n_md = sum(1 for k, _ in cells if k == "markdown")
    print(f"wrote {out} ({len(cells)} cells: {n_md} markdown, {len(cells) - n_md} code; variant: {'colab' if args.colab else 'local'})")
    return 0


if __name__ == "__main__":
    sys.exit(main())