Add decision core: autojev/train.py
Browse files- autojev/train.py +740 -0
autojev/train.py
ADDED
|
@@ -0,0 +1,740 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Full-weight cross-entropy training with periodic, fixed-fold evaluation."""
|
| 2 |
+
|
| 3 |
+
import argparse
|
| 4 |
+
import copy
|
| 5 |
+
import hashlib
|
| 6 |
+
import json
|
| 7 |
+
import math
|
| 8 |
+
import os
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
import random
|
| 11 |
+
import subprocess
|
| 12 |
+
import shutil
|
| 13 |
+
import time
|
| 14 |
+
from collections.abc import Callable, Sequence
|
| 15 |
+
from typing import cast
|
| 16 |
+
|
| 17 |
+
import torch
|
| 18 |
+
import torch.nn.functional as F
|
| 19 |
+
|
| 20 |
+
from autojev.evaluate import (
|
| 21 |
+
Metrics, Prediction, calibration_ok, evaluate_logits, fit_temperature,
|
| 22 |
+
hard_label, label_index, metrics, options, read_predictions, read_rows,
|
| 23 |
+
selection_key, validate_coverage, write_json,
|
| 24 |
+
)
|
| 25 |
+
from autojev.events import record
|
| 26 |
+
from autojev.model import DecisionModel
|
| 27 |
+
from autojev.optim import CPUOffloadAdamW
|
| 28 |
+
from autojev.sft_pipeline import Receipt, ids_sha256, load_schedule, path_at
|
| 29 |
+
from autojev.types import Example, JSONValue
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class Arguments(argparse.Namespace):
|
| 33 |
+
train: str
|
| 34 |
+
schedule: str | None
|
| 35 |
+
development: str
|
| 36 |
+
temperature: str
|
| 37 |
+
reference: str
|
| 38 |
+
public: str | None
|
| 39 |
+
guard_data: str | None
|
| 40 |
+
guard_reference: str | None
|
| 41 |
+
quick_development: str | None
|
| 42 |
+
quick_guard_data: str | None
|
| 43 |
+
quick_temperature: str | None
|
| 44 |
+
quick_eval_every: int
|
| 45 |
+
run: str
|
| 46 |
+
output: str
|
| 47 |
+
base_model: str
|
| 48 |
+
revision: str
|
| 49 |
+
cache_dir: str | None
|
| 50 |
+
epochs: int
|
| 51 |
+
batch_size: int
|
| 52 |
+
effective_batch_size: int
|
| 53 |
+
token_budget: int
|
| 54 |
+
max_length: int
|
| 55 |
+
lr: float
|
| 56 |
+
weight_decay: float
|
| 57 |
+
seed: int
|
| 58 |
+
eval_every: int
|
| 59 |
+
public_eval_every: int
|
| 60 |
+
resume_every: int
|
| 61 |
+
stop_after: int | None
|
| 62 |
+
resume: str | None
|
| 63 |
+
initial_checkpoint: str | None
|
| 64 |
+
cpu_threads: int
|
| 65 |
+
allow_shared_families: bool
|
| 66 |
+
keep_only_selected: bool
|
| 67 |
+
no_resume: bool
|
| 68 |
+
resume_unlink_first: bool
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
KEEP_ONLY_SELECTED = False
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def digest(path: str | Path) -> str:
|
| 75 |
+
with Path(path).open("rb") as stream:
|
| 76 |
+
return hashlib.file_digest(stream, "sha256").hexdigest()
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def initial_identity(path: str, base_model: str, revision: str) -> dict[str, JSONValue]:
|
| 80 |
+
"""Bind a decision artifact used to start a new stage, not an optimizer resume."""
|
| 81 |
+
directory = Path(path).resolve()
|
| 82 |
+
saved = cast(dict[str, JSONValue], json.loads((directory / "decision_config.json").read_text()))
|
| 83 |
+
if saved.get("format_version") != 1 or saved.get("base_model") != base_model or saved.get("revision") != revision:
|
| 84 |
+
raise ValueError("Initial checkpoint format/base model/revision differs from the requested model")
|
| 85 |
+
temperature = saved.get("temperature")
|
| 86 |
+
if not isinstance(temperature, (int, float)) or isinstance(temperature, bool) or not math.isfinite(temperature) or temperature <= 0:
|
| 87 |
+
raise ValueError("Initial checkpoint temperature must be positive and finite")
|
| 88 |
+
index = cast(dict[str, JSONValue], json.loads((directory / "model.safetensors.index.json").read_text()))
|
| 89 |
+
weights = index.get("weight_map")
|
| 90 |
+
if not isinstance(weights, dict) or not weights:
|
| 91 |
+
raise ValueError("Initial checkpoint must have an indexed set of backbone weights")
|
| 92 |
+
names = {"config.json", "decision_config.json", "model.safetensors.index.json", "readout.safetensors",
|
| 93 |
+
"tokenizer.json", "tokenizer_config.json", "processor_config.json", "chat_template.jinja"}
|
| 94 |
+
for name in weights.values():
|
| 95 |
+
if not isinstance(name, str) or Path(name).name != name or not name.endswith(".safetensors"):
|
| 96 |
+
raise ValueError("Checkpoint shard names must be local safetensors filenames")
|
| 97 |
+
names.add(name)
|
| 98 |
+
# team-1: text-only base saves no processor_config.json; hash only files that exist
|
| 99 |
+
hashes: dict[str, JSONValue] = {name: digest(directory / name) for name in sorted(names) if name != "processor_config.json" or (directory / name).exists()}
|
| 100 |
+
result: dict[str, JSONValue] = {
|
| 101 |
+
"path": str(directory), "base_model": base_model, "revision": revision,
|
| 102 |
+
"temperature": temperature, "decision_config": saved, "files_sha256": hashes,
|
| 103 |
+
"artifact_sha256": hashlib.sha256(json.dumps(hashes, sort_keys=True, separators=(",", ":")).encode()).hexdigest(),
|
| 104 |
+
}
|
| 105 |
+
release_manifest = directory / "release-manifest.json"
|
| 106 |
+
if release_manifest.exists():
|
| 107 |
+
result["release_manifest_sha256"] = digest(release_manifest)
|
| 108 |
+
return result
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def append(path: Path, value: object) -> None:
|
| 112 |
+
with path.open("a") as stream:
|
| 113 |
+
stream.write(json.dumps(value, ensure_ascii=False) + "\n")
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
class ScheduledRows:
|
| 117 |
+
"""A frozen ID order whose audited text may arrive in immutable tranches."""
|
| 118 |
+
|
| 119 |
+
def __init__(self, path: Path, calibration: Path, protected: Sequence[Example], run: str) -> None:
|
| 120 |
+
self.path = path.absolute()
|
| 121 |
+
self.schedule = load_schedule(self.path)
|
| 122 |
+
self.sha256 = digest(self.path)
|
| 123 |
+
self.calibration = calibration
|
| 124 |
+
self.calibration_sha256 = digest(calibration)
|
| 125 |
+
self.run = run
|
| 126 |
+
self.ids = self.schedule["ordered_ids"]
|
| 127 |
+
self.protected_ids = {row["id"] for row in protected}
|
| 128 |
+
self.protected_families = {(str(row["source"].get("dataset", row["suite"])), row["family"]) for row in protected}
|
| 129 |
+
if set(self.ids) & self.protected_ids:
|
| 130 |
+
raise ValueError("Scheduled training IDs overlap a held-out fold")
|
| 131 |
+
for source, checksum in ((self.schedule["plan_path"], self.schedule["plan_sha256"]),
|
| 132 |
+
(self.schedule["original_train_path"], self.schedule["original_train_sha256"])):
|
| 133 |
+
if digest(path_at(self.path, source)) != checksum:
|
| 134 |
+
raise ValueError(f"Scheduled source changed: {source}")
|
| 135 |
+
if path_at(self.path, self.schedule["calibration_path"]).resolve() != calibration.resolve():
|
| 136 |
+
raise ValueError("--temperature must be the calibration fold bound by the schedule")
|
| 137 |
+
self.tranche_for_id = {identifier: part["index"] for part in self.schedule["tranches"] for identifier in part["ordered_ids"]}
|
| 138 |
+
self.receipts: list[dict[str, JSONValue]] = []
|
| 139 |
+
self.cache: dict[int, dict[str, Example]] = {}
|
| 140 |
+
|
| 141 |
+
def _read(self, index: int) -> tuple[dict[str, JSONValue], list[Example]]:
|
| 142 |
+
if digest(self.path) != self.sha256 or digest(self.calibration) != self.calibration_sha256:
|
| 143 |
+
raise ValueError("The frozen schedule or calibration fold changed")
|
| 144 |
+
part = self.schedule["tranches"][index]
|
| 145 |
+
manifest = path_at(self.path, part["manifest_path"])
|
| 146 |
+
receipt = cast(Receipt, json.loads(manifest.read_text()))
|
| 147 |
+
expected = {"schema_version": 1, "complete": True, "schedule_sha256": self.sha256,
|
| 148 |
+
"tranche": index, "rows": len(part["ordered_ids"]),
|
| 149 |
+
"ordered_ids_sha256": ids_sha256(part["ordered_ids"]),
|
| 150 |
+
"calibration_sha256": self.calibration_sha256,
|
| 151 |
+
"previous_receipts_sha256": [value["manifest_sha256"] for value in self.receipts[:index]]}
|
| 152 |
+
if any(receipt.get(key) != value for key, value in expected.items()):
|
| 153 |
+
raise ValueError(f"Tranche {index} readiness receipt differs from the frozen schedule")
|
| 154 |
+
data_path = path_at(self.path, part["data_path"])
|
| 155 |
+
audit_path = path_at(self.path, receipt["audit_path"])
|
| 156 |
+
for path, checksum in ((data_path, receipt["data_sha256"]), (audit_path, receipt["audit_sha256"]),
|
| 157 |
+
(path_at(self.path, part["synthetic_path"]), receipt["synthetic_sha256"]),
|
| 158 |
+
(path_at(self.path, self.schedule["controls_audit_path"]), receipt["controls_audit_sha256"])):
|
| 159 |
+
if digest(path) != checksum:
|
| 160 |
+
raise ValueError(f"Tranche {index} bound file changed: {path}")
|
| 161 |
+
audit = cast(dict[str, JSONValue], json.loads(audit_path.read_text()))
|
| 162 |
+
if (audit.get("complete") is not True or audit.get("phase") != "tranche"
|
| 163 |
+
or audit.get("schedule_sha256") != self.sha256 or audit.get("tranche") != index
|
| 164 |
+
or audit.get("accepted_rows") != len(part["synthetic_ids"])):
|
| 165 |
+
raise ValueError(f"Tranche {index} has no complete matching synthetic audit")
|
| 166 |
+
audited_train = cast(dict[str, JSONValue], cast(dict[str, JSONValue], audit["outputs"])["train"])
|
| 167 |
+
if audited_train.get("sha256") != receipt["synthetic_sha256"]:
|
| 168 |
+
raise ValueError(f"Tranche {index} synthetic rows differ from their audit")
|
| 169 |
+
controls = cast(dict[str, JSONValue], json.loads(path_at(self.path, self.schedule["controls_audit_path"]).read_text()))
|
| 170 |
+
if controls.get("complete") is not True or controls.get("phase") != "controls":
|
| 171 |
+
raise ValueError("The fixed synthetic controls must pass their complete audit first")
|
| 172 |
+
rows = read_rows(data_path)
|
| 173 |
+
if [row["id"] for row in rows] != part["ordered_ids"]:
|
| 174 |
+
raise ValueError(f"Tranche {index} text IDs/order differ from the frozen schedule")
|
| 175 |
+
families = {(str(row["source"].get("dataset", row["suite"])), row["family"]) for row in rows}
|
| 176 |
+
if families & self.protected_families:
|
| 177 |
+
raise ValueError(f"Tranche {index} has a family overlapping a held-out fold")
|
| 178 |
+
provenance = cast(dict[str, JSONValue], dict(receipt))
|
| 179 |
+
provenance.update(manifest_path=str(manifest), manifest_sha256=digest(manifest), data_path=str(data_path))
|
| 180 |
+
return provenance, rows
|
| 181 |
+
|
| 182 |
+
def batch(self, step: int, batch_size: int, before_wait: Callable[[], None] | None = None) -> list[Example]:
|
| 183 |
+
selected_ids = self.ids[step * batch_size:(step + 1) * batch_size]
|
| 184 |
+
if not selected_ids:
|
| 185 |
+
raise ValueError("The training cursor is outside the frozen schedule")
|
| 186 |
+
needed = {self.tranche_for_id[identifier] for identifier in selected_ids}
|
| 187 |
+
for index in sorted(needed):
|
| 188 |
+
if index in self.cache:
|
| 189 |
+
continue
|
| 190 |
+
manifest = path_at(self.path, self.schedule["tranches"][index]["manifest_path"])
|
| 191 |
+
waiting_started = time.monotonic()
|
| 192 |
+
waiting = False
|
| 193 |
+
while not manifest.exists():
|
| 194 |
+
if not waiting and before_wait is not None:
|
| 195 |
+
before_wait()
|
| 196 |
+
waiting = True
|
| 197 |
+
record("training_waiting_for_data", run=self.run, step=step, next_step=step + 1,
|
| 198 |
+
tranche=index, schedule_sha256=self.sha256, manifest=str(manifest),
|
| 199 |
+
waiting_seconds=time.monotonic() - waiting_started)
|
| 200 |
+
time.sleep(20)
|
| 201 |
+
provenance, rows = self._read(index)
|
| 202 |
+
if index == len(self.receipts):
|
| 203 |
+
self.receipts.append(provenance)
|
| 204 |
+
record("training_tranche_loaded", run=self.run, step=step, tranche=index,
|
| 205 |
+
receipt=provenance, waiting_seconds=time.monotonic() - waiting_started if waiting else 0)
|
| 206 |
+
elif index >= len(self.receipts) or self.receipts[index] != provenance:
|
| 207 |
+
raise ValueError("A previously validated tranche receipt changed, or a tranche was skipped")
|
| 208 |
+
self.cache[index] = {row["id"]: row for row in rows}
|
| 209 |
+
result = [self.cache[self.tranche_for_id[identifier]][identifier] for identifier in selected_ids]
|
| 210 |
+
self.cache = {index: rows for index, rows in self.cache.items() if index in needed}
|
| 211 |
+
return sorted(result, key=length_estimate)
|
| 212 |
+
|
| 213 |
+
def snapshot(self, examples_seen: int) -> dict[str, JSONValue]:
|
| 214 |
+
if not 0 <= examples_seen <= len(self.ids):
|
| 215 |
+
raise ValueError("Invalid consumed-row cursor")
|
| 216 |
+
return {"schedule_path": str(self.path), "schedule_sha256": self.sha256,
|
| 217 |
+
"scheduled_rows": len(self.ids), "examples_seen": examples_seen,
|
| 218 |
+
"consumed_ids_sha256": ids_sha256(self.ids[:examples_seen]),
|
| 219 |
+
"validated_tranches": cast(JSONValue, copy.deepcopy(self.receipts))}
|
| 220 |
+
|
| 221 |
+
def restore(self, saved: dict[str, JSONValue], examples_seen: int) -> None:
|
| 222 |
+
receipts = saved.get("validated_tranches")
|
| 223 |
+
if not isinstance(receipts, list) or len(receipts) > len(self.schedule["tranches"]):
|
| 224 |
+
raise ValueError("Invalid saved tranche receipt prefix")
|
| 225 |
+
if examples_seen > sum(len(part["ordered_ids"]) for part in self.schedule["tranches"][:len(receipts)]):
|
| 226 |
+
raise ValueError("The saved cursor includes rows without validated tranche receipts")
|
| 227 |
+
for index, previous in enumerate(receipts):
|
| 228 |
+
provenance, _ = self._read(index)
|
| 229 |
+
if provenance != previous:
|
| 230 |
+
raise ValueError(f"Tranche {index} changed since the saved optimizer cursor")
|
| 231 |
+
self.receipts.append(provenance)
|
| 232 |
+
if self.snapshot(examples_seen) != saved:
|
| 233 |
+
raise ValueError("The saved training schedule/cursor differs from this run")
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
def length_estimate(row: Example) -> int:
|
| 237 |
+
measured = row["source"].get("input_tokens")
|
| 238 |
+
if isinstance(measured, int) and not isinstance(measured, bool):
|
| 239 |
+
return measured + 16
|
| 240 |
+
return len(json.dumps([row["state"], row["question"]], ensure_ascii=False)) // 3 + 192 + 512 * len(row.get("images", []))
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
def fixed_subset(path: str | None, full: Sequence[Example]) -> list[Example]:
|
| 244 |
+
"""Keep diagnostic panels identical to their frozen parent rows, including option order."""
|
| 245 |
+
if path is None:
|
| 246 |
+
return []
|
| 247 |
+
rows = read_rows(Path(path))
|
| 248 |
+
originals = {row["id"]: json.dumps(row, ensure_ascii=False) for row in full}
|
| 249 |
+
if (not rows or len({row["id"] for row in rows}) != len(rows)
|
| 250 |
+
or any(originals.get(row["id"]) != json.dumps(row, ensure_ascii=False) for row in rows)):
|
| 251 |
+
raise ValueError(f"Quick evaluation file is not an exact, unique subset of its full fold: {path}")
|
| 252 |
+
return rows
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
def microbatches(rows: Sequence[Example], batch_size: int, token_budget: int) -> list[list[Example]]:
|
| 256 |
+
result: list[list[Example]] = []
|
| 257 |
+
pending: list[Example] = []
|
| 258 |
+
longest = 0
|
| 259 |
+
for row in rows:
|
| 260 |
+
length = length_estimate(row)
|
| 261 |
+
if pending and (len(pending) == batch_size or max(longest, length) * (len(pending) + 1) > token_budget):
|
| 262 |
+
result.append(pending)
|
| 263 |
+
pending, longest = [], 0
|
| 264 |
+
pending.append(row)
|
| 265 |
+
longest = max(longest, length)
|
| 266 |
+
if pending:
|
| 267 |
+
result.append(pending)
|
| 268 |
+
return result
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
def targets(rows: Sequence[Example], device: torch.device) -> torch.Tensor:
|
| 272 |
+
values = torch.zeros((len(rows), 255), dtype=torch.float32, device=device)
|
| 273 |
+
for index, row in enumerate(rows):
|
| 274 |
+
labels, target = options(row["question"]), row["target"]
|
| 275 |
+
if isinstance(target, list):
|
| 276 |
+
distribution = target
|
| 277 |
+
elif row["question"]["type"] == "noul":
|
| 278 |
+
positive = float(cast(float, target))
|
| 279 |
+
distribution = [1.0 - positive, positive]
|
| 280 |
+
else:
|
| 281 |
+
distribution = [float(label == target) for label in labels]
|
| 282 |
+
if len(distribution) != len(labels) or any(not math.isfinite(p) or p < 0 for p in distribution) or abs(sum(distribution) - 1) > 1e-6:
|
| 283 |
+
raise ValueError(f"Invalid training target: {row['id']}")
|
| 284 |
+
values[index, :len(distribution)] = torch.tensor(distribution, device=device)
|
| 285 |
+
return values
|
| 286 |
+
|
| 287 |
+
|
| 288 |
+
def augment(rows: Sequence[Example], rng: random.Random) -> list[Example]:
|
| 289 |
+
result = copy.deepcopy(list(rows))
|
| 290 |
+
for row in result:
|
| 291 |
+
if row["question"]["type"] == "choice":
|
| 292 |
+
criteria = row["question"]["criteria"]
|
| 293 |
+
target = row["target"]
|
| 294 |
+
weights = dict(zip(criteria, target, strict=True)) if isinstance(target, list) else None
|
| 295 |
+
items = list(criteria.items())
|
| 296 |
+
rng.shuffle(items)
|
| 297 |
+
row["question"]["criteria"] = dict(items)
|
| 298 |
+
if weights is not None:
|
| 299 |
+
row["target"] = [weights[key] for key, _ in items]
|
| 300 |
+
return result
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
@torch.inference_mode()
|
| 304 |
+
def infer(model: DecisionModel, rows: Sequence[Example], args: Arguments) -> list[list[float]]:
|
| 305 |
+
was_training = model.training
|
| 306 |
+
model.eval()
|
| 307 |
+
result: list[list[float]] = []
|
| 308 |
+
for batch in microbatches(rows, args.batch_size, args.token_budget):
|
| 309 |
+
logits = model(model.prepare(batch, max_length=args.max_length)).detach().cpu()
|
| 310 |
+
for row, values in zip(batch, logits, strict=True):
|
| 311 |
+
result.append(cast(list[float], values[:len(options(row["question"]))].tolist()))
|
| 312 |
+
model.train(was_training)
|
| 313 |
+
return result
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
def save_predictions(path: Path, predictions: Sequence[Prediction]) -> None:
|
| 317 |
+
with path.open("w") as stream:
|
| 318 |
+
for prediction in predictions:
|
| 319 |
+
stream.write(json.dumps(prediction, ensure_ascii=False) + "\n")
|
| 320 |
+
|
| 321 |
+
|
| 322 |
+
def sync_directory(path: Path) -> None:
|
| 323 |
+
descriptor = os.open(path, os.O_RDONLY | os.O_DIRECTORY)
|
| 324 |
+
try:
|
| 325 |
+
os.fsync(descriptor)
|
| 326 |
+
finally:
|
| 327 |
+
os.close(descriptor)
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
def select_checkpoint(root: Path, step: int) -> None:
|
| 331 |
+
pending = root / "selected.pending"
|
| 332 |
+
pending.unlink(missing_ok=True)
|
| 333 |
+
pending.symlink_to(f"step-{step:05d}", target_is_directory=True)
|
| 334 |
+
os.replace(pending, root / "selected")
|
| 335 |
+
sync_directory(root)
|
| 336 |
+
|
| 337 |
+
|
| 338 |
+
def save_selected(model: DecisionModel, root: Path, temperature: float, step: int, provenance: dict[str, str],
|
| 339 |
+
initial_artifact: dict[str, JSONValue] | None = None,
|
| 340 |
+
training_schedule: dict[str, JSONValue] | None = None) -> None:
|
| 341 |
+
destination = root / f"step-{step:05d}"
|
| 342 |
+
metadata: dict[str, JSONValue] = {"step": step, "provenance": cast(JSONValue, provenance)}
|
| 343 |
+
if initial_artifact is not None:
|
| 344 |
+
metadata["initial_artifact"] = initial_artifact
|
| 345 |
+
if training_schedule is not None:
|
| 346 |
+
metadata["training_schedule"] = training_schedule
|
| 347 |
+
previous = (root / "selected").resolve() if (root / "selected").is_symlink() else None
|
| 348 |
+
model.save(destination, temperature=temperature, **metadata)
|
| 349 |
+
for file in destination.rglob("*"):
|
| 350 |
+
if file.is_file():
|
| 351 |
+
with file.open("rb") as stream:
|
| 352 |
+
os.fsync(stream.fileno())
|
| 353 |
+
sync_directory(destination)
|
| 354 |
+
select_checkpoint(root, step)
|
| 355 |
+
if KEEP_ONLY_SELECTED and previous is not None and previous != destination.resolve() and previous.parent == root.resolve() and previous.name.startswith("step-") and previous.is_dir():
|
| 356 |
+
shutil.rmtree(previous)
|
| 357 |
+
record("checkpoint_pruned", path=str(previous), selected_step=step)
|
| 358 |
+
|
| 359 |
+
|
| 360 |
+
def archive_interrupted_tail(run: Path, output: Path, step: int) -> None:
|
| 361 |
+
"""Keep abandoned work as history while the active trajectory resumes its saved cursor."""
|
| 362 |
+
suffix = str(time.time_ns())
|
| 363 |
+
archive = run / "interrupted" / suffix
|
| 364 |
+
for name in ("training.jsonl", "evaluations.jsonl", "public-evaluations.jsonl", "quick-evaluations.jsonl"):
|
| 365 |
+
path = run / name
|
| 366 |
+
if not path.exists():
|
| 367 |
+
continue
|
| 368 |
+
lines = path.read_text().splitlines()
|
| 369 |
+
tail = [line for line in lines if json.loads(line)["step"] > step]
|
| 370 |
+
if tail:
|
| 371 |
+
archive.mkdir(parents=True, exist_ok=True)
|
| 372 |
+
(archive / name).write_text("\n".join(tail) + "\n")
|
| 373 |
+
path.write_text("\n".join(line for line in lines if json.loads(line)["step"] <= step) + "\n")
|
| 374 |
+
for prefix in ("development", "temperature", "public", "guard", "quick-development", "quick-temperature", "quick-guard"):
|
| 375 |
+
for path in run.glob(f"{prefix}-*.jsonl"):
|
| 376 |
+
suffix_step = path.stem.rsplit("-", 1)[1]
|
| 377 |
+
if suffix_step.isdigit() and int(suffix_step) > step:
|
| 378 |
+
archive.mkdir(parents=True, exist_ok=True)
|
| 379 |
+
path.rename(archive / path.name)
|
| 380 |
+
for path in output.glob("step-*"):
|
| 381 |
+
if path.is_dir() and path.name.rsplit("-", 1)[1].isdigit() and int(path.name.rsplit("-", 1)[1]) > step:
|
| 382 |
+
destination = output / "interrupted" / suffix
|
| 383 |
+
destination.mkdir(parents=True, exist_ok=True)
|
| 384 |
+
path.rename(destination / path.name)
|
| 385 |
+
record("training_resumed", run=run.name, resume_step=step, archived_tail=archive.exists(), attempt=suffix)
|
| 386 |
+
|
| 387 |
+
|
| 388 |
+
def main() -> None:
|
| 389 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 390 |
+
for name, default in [("train", "data/train.jsonl"), ("development", "data/dev.jsonl"),
|
| 391 |
+
("temperature", "data/temperature.jsonl"), ("reference", "progress/baselines/jev-development.predictions.jsonl")]:
|
| 392 |
+
parser.add_argument(f"--{name}", default=default)
|
| 393 |
+
parser.add_argument("--schedule", help="Frozen global ID schedule with immutable audited tranches; replaces --train")
|
| 394 |
+
parser.add_argument("--public")
|
| 395 |
+
parser.add_argument("--guard-data", help="Separate development cohort whose calibration also gates selection")
|
| 396 |
+
parser.add_argument("--guard-reference", help="Complete fixed Jev predictions for --guard-data")
|
| 397 |
+
parser.add_argument("--quick-development", help="Frozen subset of development for diagnostic curves only")
|
| 398 |
+
parser.add_argument("--quick-guard-data", help="Frozen subset of the guard cohort for diagnostic curves only")
|
| 399 |
+
parser.add_argument("--quick-temperature", help="Frozen subset of temperature data, used only to calibrate quick curves")
|
| 400 |
+
parser.add_argument("--quick-eval-every", type=int, default=20)
|
| 401 |
+
parser.add_argument("--run", required=True)
|
| 402 |
+
parser.add_argument("--output", required=True)
|
| 403 |
+
parser.add_argument("--base-model", default="Qwen/Qwen3.8-27B")
|
| 404 |
+
parser.add_argument("--revision", default="1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0")
|
| 405 |
+
parser.add_argument("--cache-dir")
|
| 406 |
+
parser.add_argument("--epochs", type=int, default=1)
|
| 407 |
+
parser.add_argument("--batch-size", type=int, default=32)
|
| 408 |
+
parser.add_argument("--effective-batch-size", type=int, default=256)
|
| 409 |
+
parser.add_argument("--token-budget", type=int, default=8192)
|
| 410 |
+
parser.add_argument("--max-length", type=int, default=8192)
|
| 411 |
+
parser.add_argument("--lr", type=float, default=2e-6)
|
| 412 |
+
parser.add_argument("--weight-decay", type=float, default=0.01)
|
| 413 |
+
parser.add_argument("--seed", type=int, default=20260920)
|
| 414 |
+
parser.add_argument("--eval-every", type=int, default=20)
|
| 415 |
+
parser.add_argument("--public-eval-every", type=int, default=100)
|
| 416 |
+
parser.add_argument("--resume-every", type=int, default=100)
|
| 417 |
+
parser.add_argument("--stop-after", type=int)
|
| 418 |
+
parser.add_argument("--resume")
|
| 419 |
+
parser.add_argument("--initial-checkpoint", help="Start a new full-weight SFT stage from this artifact with a fresh optimizer")
|
| 420 |
+
parser.add_argument("--cpu-threads", type=int, default=32)
|
| 421 |
+
# VIDRAFT team-1 additions (defaults keep upstream behaviour).
|
| 422 |
+
parser.add_argument("--allow-shared-families", action="store_true",
|
| 423 |
+
help="Held-out folds share (dataset, family) with train; IDs and source.group_id must still be disjoint")
|
| 424 |
+
parser.add_argument("--keep-only-selected", action="store_true",
|
| 425 |
+
help="Delete the previously selected step-* artifact after a newer one is selected (disk budget)")
|
| 426 |
+
parser.add_argument("--no-resume", action="store_true", help="Never write resume.pt (optimizer state is ~12 bytes/param)")
|
| 427 |
+
parser.add_argument("--resume-unlink-first", action="store_true",
|
| 428 |
+
help="Remove the old resume.pt before writing the new one (halves peak disk; not atomic)")
|
| 429 |
+
args = parser.parse_args(namespace=Arguments())
|
| 430 |
+
global KEEP_ONLY_SELECTED
|
| 431 |
+
KEEP_ONLY_SELECTED = args.keep_only_selected
|
| 432 |
+
if min(args.epochs, args.batch_size, args.effective_batch_size, args.token_budget, args.eval_every, args.public_eval_every, args.resume_every, args.quick_eval_every) < 1:
|
| 433 |
+
raise ValueError("Batch, epoch and interval settings must be positive")
|
| 434 |
+
if args.stop_after is not None and args.stop_after < 1:
|
| 435 |
+
raise ValueError("A pilot must include at least one training step")
|
| 436 |
+
if bool(args.guard_data) != bool(args.guard_reference):
|
| 437 |
+
raise ValueError("--guard-data and --guard-reference must be supplied together")
|
| 438 |
+
quick_paths = (args.quick_development, args.quick_guard_data, args.quick_temperature)
|
| 439 |
+
if any(quick_paths) and (not all(quick_paths) or not args.guard_data):
|
| 440 |
+
raise ValueError("Quick evaluation needs all three frozen subsets and a full guard cohort/reference")
|
| 441 |
+
if args.initial_checkpoint and not os.environ.get("AUTOJEV_EVENTS"):
|
| 442 |
+
raise ValueError("Set AUTOJEV_EVENTS to the new stage's event log before continuing from a checkpoint")
|
| 443 |
+
if args.schedule and (args.epochs != 1 or not os.environ.get("AUTOJEV_EVENTS")):
|
| 444 |
+
raise ValueError("A streamed schedule requires one epoch and an explicit AUTOJEV_EVENTS path")
|
| 445 |
+
run, output = Path(args.run), Path(args.output)
|
| 446 |
+
if args.initial_checkpoint and any(path.resolve().is_relative_to(Path(args.initial_checkpoint).resolve()) for path in (run, output)):
|
| 447 |
+
raise ValueError("New run/output directories must be outside the initial checkpoint artifact")
|
| 448 |
+
run.mkdir(parents=True, exist_ok=True)
|
| 449 |
+
output.mkdir(parents=True, exist_ok=True)
|
| 450 |
+
if (run / "config.json").exists() and not args.resume:
|
| 451 |
+
raise ValueError("Run already exists; resume explicitly or choose a new run ID")
|
| 452 |
+
train = [] if args.schedule else read_rows(Path(args.train))
|
| 453 |
+
development, temperature_rows = (read_rows(Path(path)) for path in (args.development, args.temperature))
|
| 454 |
+
if (not train and not args.schedule) or not development or not temperature_rows:
|
| 455 |
+
raise ValueError("Training, development and temperature folds must be nonempty")
|
| 456 |
+
reference_predictions = read_predictions(Path(args.reference))
|
| 457 |
+
validate_coverage(development, reference_predictions)
|
| 458 |
+
reference = metrics(reference_predictions)
|
| 459 |
+
public_rows = read_rows(Path(args.public)) if args.public else []
|
| 460 |
+
guard_rows = read_rows(Path(args.guard_data)) if args.guard_data else []
|
| 461 |
+
guard_reference: Metrics | None = None
|
| 462 |
+
guard_predictions: list[Prediction] = []
|
| 463 |
+
if args.guard_data:
|
| 464 |
+
if not guard_rows:
|
| 465 |
+
raise ValueError("The guard cohort must be nonempty")
|
| 466 |
+
guard_predictions = read_predictions(Path(cast(str, args.guard_reference)))
|
| 467 |
+
validate_coverage(guard_rows, guard_predictions)
|
| 468 |
+
guard_reference = metrics(guard_predictions)
|
| 469 |
+
quick_development = fixed_subset(args.quick_development, development)
|
| 470 |
+
quick_guard = fixed_subset(args.quick_guard_data, guard_rows)
|
| 471 |
+
quick_temperature = fixed_subset(args.quick_temperature, temperature_rows)
|
| 472 |
+
quick_ids = {row["id"] for row in quick_development}
|
| 473 |
+
quick_guard_ids = {row["id"] for row in quick_guard}
|
| 474 |
+
quick_reference = metrics([value for value in reference_predictions if value["id"] in quick_ids]) if quick_development else None
|
| 475 |
+
quick_guard_reference = metrics([value for value in guard_predictions if value["id"] in quick_guard_ids]) if quick_guard else None
|
| 476 |
+
partitions = {"train": train, "development": development, "temperature": temperature_rows,
|
| 477 |
+
"public": public_rows, "guard": guard_rows}
|
| 478 |
+
families: dict[str, set[tuple[str, str]]] = {}
|
| 479 |
+
identifiers: dict[str, set[str]] = {}
|
| 480 |
+
source_groups: dict[str, set[tuple[str, str]]] = {}
|
| 481 |
+
for name, rows in partitions.items():
|
| 482 |
+
identifiers[name] = {row["id"] for row in rows}
|
| 483 |
+
if len(identifiers[name]) != len(rows):
|
| 484 |
+
raise ValueError(f"Duplicate IDs in {name}")
|
| 485 |
+
families[name] = {(str(row["source"].get("dataset", row["suite"])), row["family"]) for row in rows}
|
| 486 |
+
source_groups[name] = {(row["family"], str(row["source"].get("group_id", row["id"]))) for row in rows}
|
| 487 |
+
for other in identifiers:
|
| 488 |
+
if name == other:
|
| 489 |
+
continue
|
| 490 |
+
if identifiers[name] & identifiers[other]:
|
| 491 |
+
raise ValueError(f"Partitions overlap: {name}/{other}")
|
| 492 |
+
if args.allow_shared_families:
|
| 493 |
+
if source_groups[name] & source_groups[other]:
|
| 494 |
+
raise ValueError(f"Partitions share source groups: {name}/{other}")
|
| 495 |
+
elif families[name] & families[other]:
|
| 496 |
+
raise ValueError(f"Partitions overlap: {name}/{other}")
|
| 497 |
+
scheduled = ScheduledRows(Path(args.schedule), Path(args.temperature), development + temperature_rows + public_rows + guard_rows, run.name) if args.schedule else None
|
| 498 |
+
if scheduled and scheduled.schedule["seed"] != args.seed:
|
| 499 |
+
raise ValueError("The training seed must match the frozen schedule seed")
|
| 500 |
+
train_rows = len(scheduled.ids) if scheduled else len(train)
|
| 501 |
+
hashes = {name: digest(path) for name, path in [("development", args.development), ("temperature", args.temperature), ("reference", args.reference)]}
|
| 502 |
+
hashes["train_schedule" if scheduled else "train"] = scheduled.sha256 if scheduled else digest(args.train)
|
| 503 |
+
if args.public:
|
| 504 |
+
hashes["public"] = digest(args.public)
|
| 505 |
+
if args.guard_data:
|
| 506 |
+
hashes["guard_data"] = digest(args.guard_data)
|
| 507 |
+
hashes["guard_reference"] = digest(cast(str, args.guard_reference))
|
| 508 |
+
for key, path in (("quick_development", args.quick_development), ("quick_guard_data", args.quick_guard_data), ("quick_temperature", args.quick_temperature)):
|
| 509 |
+
if path is not None:
|
| 510 |
+
hashes[key] = digest(path)
|
| 511 |
+
initial_artifact = initial_identity(args.initial_checkpoint, args.base_model, args.revision) if args.initial_checkpoint else None
|
| 512 |
+
revision = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
|
| 513 |
+
package = Path(__file__).resolve().parent
|
| 514 |
+
code_hashes = {name: digest(package / name) for name in ("train.py", "model.py", "optim.py", "evaluate.py", "types.py", "events.py")}
|
| 515 |
+
if scheduled:
|
| 516 |
+
code_hashes["sft_pipeline.py"] = digest(package / "sft_pipeline.py")
|
| 517 |
+
code_hashes["uv.lock"] = digest(package.parents[1] / "uv.lock")
|
| 518 |
+
config = {**vars(args), "data_sha256": hashes, "code_sha256": code_hashes, "git_commit": revision,
|
| 519 |
+
"initial_artifact": initial_artifact, "events_path": os.getenv("AUTOJEV_EVENTS", "progress/events.jsonl"),
|
| 520 |
+
"train_rows": train_rows, "development_rows": len(development), "temperature_rows": len(temperature_rows),
|
| 521 |
+
"guard_rows": len(guard_rows), "quick_development_rows": len(quick_development),
|
| 522 |
+
"quick_guard_rows": len(quick_guard), "quick_temperature_rows": len(quick_temperature)}
|
| 523 |
+
if not args.resume:
|
| 524 |
+
write_json(run / "config.json", config)
|
| 525 |
+
if scheduled:
|
| 526 |
+
scheduled.batch(0, args.effective_batch_size)
|
| 527 |
+
random.seed(args.seed)
|
| 528 |
+
torch.manual_seed(args.seed)
|
| 529 |
+
torch.cuda.manual_seed_all(args.seed)
|
| 530 |
+
started = time.monotonic()
|
| 531 |
+
model = DecisionModel(checkpoint=args.initial_checkpoint, train=True, base_model=args.base_model,
|
| 532 |
+
revision=args.revision, cache_dir=args.cache_dir, gradient_checkpointing=True,
|
| 533 |
+
cpu_threads=args.cpu_threads)
|
| 534 |
+
optimizer = CPUOffloadAdamW(model.named_parameters(), lr=args.lr, weight_decay=args.weight_decay)
|
| 535 |
+
parameters = [parameter for parameter in model.parameters() if parameter.requires_grad]
|
| 536 |
+
groups: list[list[Example]] = []
|
| 537 |
+
for epoch in range(args.epochs):
|
| 538 |
+
ordered = list(train)
|
| 539 |
+
random.Random(args.seed + epoch).shuffle(ordered)
|
| 540 |
+
for offset in range(0, len(ordered), args.effective_batch_size):
|
| 541 |
+
groups.append(sorted(ordered[offset:offset + args.effective_batch_size], key=length_estimate))
|
| 542 |
+
total_steps = math.ceil(train_rows / args.effective_batch_size) if scheduled else len(groups)
|
| 543 |
+
step, examples_seen = 0, 0
|
| 544 |
+
last_resume_step: int | None = None
|
| 545 |
+
best: Metrics | None = None
|
| 546 |
+
best_guard: Metrics | None = None
|
| 547 |
+
best_step: int | None = None
|
| 548 |
+
selected_temperature: float | None = None
|
| 549 |
+
if args.resume:
|
| 550 |
+
state = torch.load(args.resume, map_location="cpu", weights_only=True)
|
| 551 |
+
if state["data_sha256"] != hashes or state["total_steps"] != total_steps:
|
| 552 |
+
raise ValueError("Resume data/schedule differs from the saved run")
|
| 553 |
+
if state["config"]["code_sha256"] != code_hashes:
|
| 554 |
+
raise ValueError("Training implementation differs from the saved run")
|
| 555 |
+
if state["config"].get("initial_artifact") != initial_artifact:
|
| 556 |
+
raise ValueError("Initial checkpoint content or provenance differs from the saved stage")
|
| 557 |
+
if state["config"].get("events_path") != config["events_path"]:
|
| 558 |
+
raise ValueError("Resume event log differs from the saved stage")
|
| 559 |
+
for key in vars(args):
|
| 560 |
+
if key in {"resume", "stop_after"}:
|
| 561 |
+
continue
|
| 562 |
+
if state["config"][key] != vars(args)[key]:
|
| 563 |
+
raise ValueError(f"Resume configuration differs: {key}")
|
| 564 |
+
step, examples_seen = state["step"], state["examples_seen"]
|
| 565 |
+
if scheduled:
|
| 566 |
+
if examples_seen != min(step * args.effective_batch_size, train_rows):
|
| 567 |
+
raise ValueError("The saved consumed-row cursor differs from its global update")
|
| 568 |
+
scheduled.restore(state["training_schedule"], examples_seen)
|
| 569 |
+
optimizer.load_state_dict(state["optimizer"])
|
| 570 |
+
last_resume_step = step
|
| 571 |
+
if args.stop_after is not None and args.stop_after <= step:
|
| 572 |
+
raise ValueError("The requested stopping step must follow the saved step")
|
| 573 |
+
best, best_step, selected_temperature = state["best"], state["best_step"], state["selected_temperature"]
|
| 574 |
+
best_guard = state.get("best_guard")
|
| 575 |
+
random.setstate(state["python_rng"])
|
| 576 |
+
torch.set_rng_state(state["torch_rng"])
|
| 577 |
+
torch.cuda.set_rng_state_all(state["cuda_rng"])
|
| 578 |
+
del state
|
| 579 |
+
archive_interrupted_tail(run, output, step)
|
| 580 |
+
if best_step is not None:
|
| 581 |
+
selected = output / f"step-{best_step:05d}"
|
| 582 |
+
if not selected.is_dir():
|
| 583 |
+
raise ValueError("The checkpoint selected by the resume state is missing")
|
| 584 |
+
select_checkpoint(output, best_step)
|
| 585 |
+
else:
|
| 586 |
+
(output / "selected").unlink(missing_ok=True)
|
| 587 |
+
record("training_started", run=run.name, git_commit=revision, data_sha256=hashes,
|
| 588 |
+
initial_artifact=initial_artifact, initialization="exact_resume" if args.resume else "checkpoint_fresh_optimizer" if initial_artifact else "base_fresh_optimizer",
|
| 589 |
+
step=step, total_steps=total_steps, training_schedule=scheduled.snapshot(examples_seen) if scheduled else None,
|
| 590 |
+
trainable_parameters=sum(p.numel() for p in parameters), base_model=model.base_model,
|
| 591 |
+
base_revision=model.revision, text_only=model.text_only)
|
| 592 |
+
|
| 593 |
+
def evaluate(include_public: bool = False, diagnostic: bool = False) -> None:
|
| 594 |
+
nonlocal best, best_guard, best_step, selected_temperature
|
| 595 |
+
began = time.monotonic()
|
| 596 |
+
temperature_logits = infer(model, temperature_rows, args)
|
| 597 |
+
fitted = fit_temperature(temperature_logits, [label_index(options(row["question"]), hard_label(row)) for row in temperature_rows])
|
| 598 |
+
logits = infer(model, development, args)
|
| 599 |
+
raw_predictions, fitted_predictions = evaluate_logits(development, logits), evaluate_logits(development, logits, fitted)
|
| 600 |
+
raw, calibrated = metrics(raw_predictions), metrics(fitted_predictions)
|
| 601 |
+
guard_raw: Metrics | None = None
|
| 602 |
+
guard_calibrated: Metrics | None = None
|
| 603 |
+
guard_ok = True
|
| 604 |
+
if guard_rows:
|
| 605 |
+
guard_logits = infer(model, guard_rows, args)
|
| 606 |
+
guard_raw = metrics(evaluate_logits(guard_rows, guard_logits))
|
| 607 |
+
guard_predictions = evaluate_logits(guard_rows, guard_logits, fitted)
|
| 608 |
+
guard_calibrated = metrics(guard_predictions)
|
| 609 |
+
guard_ok = calibration_ok(guard_calibrated, cast(Metrics, guard_reference))
|
| 610 |
+
save_predictions(run / f"guard-{step:05d}.jsonl", guard_predictions)
|
| 611 |
+
public_ok = calibration_ok(calibrated, reference)
|
| 612 |
+
eligible = public_ok and guard_ok
|
| 613 |
+
improved = not diagnostic and eligible and (best is None or selection_key(calibrated, step) < selection_key(best, cast(int, best_step)))
|
| 614 |
+
if improved:
|
| 615 |
+
save_selected(model, output, fitted, step, {"run": run.name, "git_commit": revision, **hashes, **code_hashes},
|
| 616 |
+
initial_artifact, scheduled.snapshot(examples_seen) if scheduled else None)
|
| 617 |
+
best, best_guard, best_step, selected_temperature = calibrated, guard_calibrated, step, fitted
|
| 618 |
+
save_predictions(run / f"development-{step:05d}.jsonl", fitted_predictions)
|
| 619 |
+
save_predictions(run / f"temperature-{step:05d}.jsonl", evaluate_logits(temperature_rows, temperature_logits, fitted))
|
| 620 |
+
guard_target = guard_calibrated is not None and guard_reference is not None and guard_ok and guard_calibrated["accuracy"] > guard_reference["accuracy"]
|
| 621 |
+
value = record("evaluation", run=run.name, step=step, examples_seen=examples_seen, raw=raw, fitted=calibrated,
|
| 622 |
+
temperature=fitted, jev=reference, eligible=eligible, public_calibration_ok=public_ok,
|
| 623 |
+
guard_raw=guard_raw, guard_fitted=guard_calibrated, guard_jev=guard_reference,
|
| 624 |
+
guard_calibration_ok=guard_ok, guard_empirical_target_met=guard_target,
|
| 625 |
+
development_target_met=eligible and calibrated["accuracy"] > reference["accuracy"],
|
| 626 |
+
diagnostic=diagnostic, selected_step=best_step, elapsed_seconds=time.monotonic() - started,
|
| 627 |
+
evaluation_seconds=time.monotonic() - began)
|
| 628 |
+
append(run / "evaluations.jsonl", value)
|
| 629 |
+
print(json.dumps(value), flush=True)
|
| 630 |
+
if include_public and public_rows:
|
| 631 |
+
predictions = evaluate_logits(public_rows, infer(model, public_rows, args), fitted)
|
| 632 |
+
save_predictions(run / f"public-{step:05d}.jsonl", predictions)
|
| 633 |
+
value = record("public_evaluation", run=run.name, step=step, temperature=fitted, metrics=metrics(predictions), status="known_suite_development_check")
|
| 634 |
+
append(run / "public-evaluations.jsonl", value)
|
| 635 |
+
|
| 636 |
+
def save_resume() -> None:
|
| 637 |
+
nonlocal last_resume_step
|
| 638 |
+
began = time.monotonic()
|
| 639 |
+
state = {"optimizer": optimizer.state_dict(), "step": step, "examples_seen": examples_seen, "total_steps": total_steps, "data_sha256": hashes, "config": config,
|
| 640 |
+
"best": best, "best_guard": best_guard, "best_step": best_step, "selected_temperature": selected_temperature,
|
| 641 |
+
"python_rng": random.getstate(), "torch_rng": torch.get_rng_state(), "cuda_rng": torch.cuda.get_rng_state_all()}
|
| 642 |
+
if scheduled:
|
| 643 |
+
state["training_schedule"] = scheduled.snapshot(examples_seen)
|
| 644 |
+
pending = output / "resume.pt.pending"
|
| 645 |
+
if args.resume_unlink_first:
|
| 646 |
+
(output / "resume.pt").unlink(missing_ok=True)
|
| 647 |
+
torch.save(state, pending)
|
| 648 |
+
with pending.open("rb") as stream:
|
| 649 |
+
os.fsync(stream.fileno())
|
| 650 |
+
os.replace(pending, output / "resume.pt")
|
| 651 |
+
sync_directory(output)
|
| 652 |
+
last_resume_step = step
|
| 653 |
+
record("resume_saved", run=run.name, step=step, bytes=(output / "resume.pt").stat().st_size, seconds=time.monotonic() - began)
|
| 654 |
+
|
| 655 |
+
def evaluate_quick() -> None:
|
| 656 |
+
began = time.monotonic()
|
| 657 |
+
temperature_logits = infer(model, quick_temperature, args)
|
| 658 |
+
fitted = fit_temperature(temperature_logits, [label_index(options(row["question"]), hard_label(row)) for row in quick_temperature])
|
| 659 |
+
logits = infer(model, quick_development, args)
|
| 660 |
+
predictions = evaluate_logits(quick_development, logits, fitted)
|
| 661 |
+
guard_logits = infer(model, quick_guard, args)
|
| 662 |
+
guard_predictions = evaluate_logits(quick_guard, guard_logits, fitted)
|
| 663 |
+
save_predictions(run / f"quick-development-{step:05d}.jsonl", predictions)
|
| 664 |
+
save_predictions(run / f"quick-guard-{step:05d}.jsonl", guard_predictions)
|
| 665 |
+
save_predictions(run / f"quick-temperature-{step:05d}.jsonl", evaluate_logits(quick_temperature, temperature_logits, fitted))
|
| 666 |
+
value = record("quick_evaluation", run=run.name, step=step, examples_seen=examples_seen,
|
| 667 |
+
raw=metrics(evaluate_logits(quick_development, logits)), fitted=metrics(predictions),
|
| 668 |
+
guard_raw=metrics(evaluate_logits(quick_guard, guard_logits)), guard_fitted=metrics(guard_predictions),
|
| 669 |
+
temperature=fitted, jev=quick_reference, guard_jev=quick_guard_reference,
|
| 670 |
+
diagnostic=True, selection_candidate=False, selected_step=best_step,
|
| 671 |
+
temperature_rows=len(quick_temperature), development_rows=len(quick_development), guard_rows=len(quick_guard),
|
| 672 |
+
elapsed_seconds=time.monotonic() - started, evaluation_seconds=time.monotonic() - began)
|
| 673 |
+
append(run / "quick-evaluations.jsonl", value)
|
| 674 |
+
print(json.dumps(value), flush=True)
|
| 675 |
+
|
| 676 |
+
def save_before_wait() -> None:
|
| 677 |
+
if step > 0 and last_resume_step != step:
|
| 678 |
+
save_resume()
|
| 679 |
+
|
| 680 |
+
if step == 0:
|
| 681 |
+
evaluate(include_public=bool(public_rows))
|
| 682 |
+
stop = min(total_steps, args.stop_after) if args.stop_after is not None else total_steps
|
| 683 |
+
while step < stop:
|
| 684 |
+
rows = scheduled.batch(step, args.effective_batch_size, save_before_wait) if scheduled else groups[step]
|
| 685 |
+
began = time.monotonic()
|
| 686 |
+
model.train()
|
| 687 |
+
group = augment(rows, random.Random(args.seed + 100003 * (step + 1)))
|
| 688 |
+
optimizer.zero_grad()
|
| 689 |
+
total_loss = 0.0
|
| 690 |
+
input_tokens = 0
|
| 691 |
+
for batch in microbatches(group, args.batch_size, args.token_budget):
|
| 692 |
+
prepared = model.prepare(batch, max_length=args.max_length)
|
| 693 |
+
logits = model(prepared)
|
| 694 |
+
target = targets(batch, logits.device)
|
| 695 |
+
loss = -(target * F.log_softmax(logits, dim=-1)).sum(-1).mean()
|
| 696 |
+
if not torch.isfinite(loss):
|
| 697 |
+
raise RuntimeError(f"Nonfinite loss at step {step + 1}")
|
| 698 |
+
(loss * len(batch) / len(group)).backward()
|
| 699 |
+
total_loss += float(loss.detach()) * len(batch)
|
| 700 |
+
input_tokens += prepared.input_tokens
|
| 701 |
+
del logits, loss, target, prepared
|
| 702 |
+
gradient_norm = float(torch.nn.utils.clip_grad_norm_(parameters, 1.0))
|
| 703 |
+
if not math.isfinite(gradient_norm):
|
| 704 |
+
raise RuntimeError(f"Nonfinite gradient at step {step + 1}")
|
| 705 |
+
step += 1
|
| 706 |
+
warmup = max(1, int(0.05 * total_steps))
|
| 707 |
+
factor = step / warmup if step <= warmup else 0.1 + 0.45 * (1 + math.cos(math.pi * (step - warmup) / max(1, total_steps - warmup)))
|
| 708 |
+
for group_parameters in optimizer.param_groups:
|
| 709 |
+
group_parameters["lr"] = args.lr * factor
|
| 710 |
+
optimizer_started = time.monotonic()
|
| 711 |
+
optimizer.step()
|
| 712 |
+
torch.cuda.synchronize()
|
| 713 |
+
examples_seen += len(group)
|
| 714 |
+
value = record("training_step", run=run.name, step=step, loss=total_loss / len(group), learning_rate=args.lr * factor,
|
| 715 |
+
examples_seen=examples_seen, group_examples=len(group), input_tokens=input_tokens, gradient_norm=gradient_norm,
|
| 716 |
+
step_seconds=time.monotonic() - began, optimizer_seconds=time.monotonic() - optimizer_started,
|
| 717 |
+
elapsed_seconds=time.monotonic() - started, gpu_peak_gb=torch.cuda.max_memory_allocated() / 1e9)
|
| 718 |
+
append(run / "training.jsonl", value)
|
| 719 |
+
print(json.dumps(value), flush=True)
|
| 720 |
+
if step % args.eval_every == 0 or step == stop:
|
| 721 |
+
evaluate(include_public=step % args.public_eval_every == 0 or step == total_steps,
|
| 722 |
+
diagnostic=step % args.eval_every != 0 and step != total_steps)
|
| 723 |
+
elif quick_development and step % args.quick_eval_every == 0:
|
| 724 |
+
evaluate_quick()
|
| 725 |
+
if not args.no_resume and (step % args.resume_every == 0 or step == stop):
|
| 726 |
+
save_resume()
|
| 727 |
+
summary = {"run": run.name, "steps": step, "planned_steps": total_steps, "complete": step == total_steps,
|
| 728 |
+
"examples_seen": examples_seen, "best_step": best_step, "selected_temperature": selected_temperature,
|
| 729 |
+
"selected_metrics": best, "selected_guard_metrics": best_guard, "jev": reference, "guard_jev": guard_reference,
|
| 730 |
+
"initial_artifact": initial_artifact, "data_sha256": hashes, "git_commit": revision,
|
| 731 |
+
"elapsed_seconds": time.monotonic() - started, "checkpoint": str(output / "selected") if best else None}
|
| 732 |
+
if scheduled:
|
| 733 |
+
summary["training_schedule"] = scheduled.snapshot(examples_seen)
|
| 734 |
+
write_json(run / "summary.json", summary)
|
| 735 |
+
record("training_finished" if step == total_steps else "training_paused", **summary)
|
| 736 |
+
print(json.dumps(summary, indent=2), flush=True)
|
| 737 |
+
|
| 738 |
+
|
| 739 |
+
if __name__ == "__main__":
|
| 740 |
+
main()
|