File size: 13,608 Bytes
55e0599 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 | #!/usr/bin/env python3
"""Disposable GPU0 throughput diagnosis; never save or resume model weights.
The candidate changes only detached metric formatting in this process. Core
source files, optimizer mathematics, samples, and the global batch stay intact.
"""
from __future__ import annotations
import argparse
import hashlib
import inspect
import json
import math
import os
from pathlib import Path
import statistics
import subprocess
import sys
import textwrap
import time
PROJECT = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(PROJECT), str(PROJECT / "src")]
from scripts.third_party_strategy_plan import write_json
VARIANTS = ("baseline", "metrics_cpu", "baseline_repeat")
UPDATES, WARMUP = 800, 100
def metric_candidate_source(source):
"""One guarded experimental edit after backward, only to detached logs."""
source = textwrap.dedent(source)
matches = [line for line in source.splitlines() if line.strip() == "count = metric_row[17].clamp_min(1.0)"]
if len(matches) != 1 or "self.optimizer.step()" not in source:
raise ValueError("Training implementation changed; inspect metric experiment again")
anchor = matches[0]
if source.index(anchor) < source.index("self.optimizer.step()"):
raise ValueError("Metric transformation must follow the optimizer update")
indent = anchor[:len(anchor) - len(anchor.lstrip())]
return source.replace(anchor, indent + "metric_row = metric_row.detach().cpu()\n" + anchor)
def choose_group(selection):
from scripts.third_party_scale_plan import coverage_plan
groups = coverage_plan(selection)["epochs"][0]["source_indices"]
candidates = [g for g in groups if len(g) == 16
and min(selection["sessions"][i]["dense_steps"] for i in g) > UPDATES]
if not candidates:
raise ValueError("No complete length-group with sixteen live sources for the entire test")
candidates.sort(key=lambda g: max(selection["sessions"][i]["dense_steps"] for i in g))
return candidates[len(candidates) // 2]
def tensor_fingerprint(model):
digest = hashlib.sha256()
for name, parameter in sorted(model.named_parameters()):
if parameter.requires_grad:
value = parameter.detach().cpu().contiguous()
digest.update(name.encode())
digest.update(value.view(-1).view(__import__("torch").uint8).numpy().tobytes())
return digest.hexdigest()
def prepare(args):
from scripts.train_third_party import contained_output
from scripts.data.prepare_third_party_scale import validate
args.run_dir = contained_output(args.run_dir)
args.manifest = args.manifest.resolve()
selection, _, _ = validate(args.manifest)
group = choose_group(selection)
core = PROJECT / "src/psi/trainers/posttrain_full_duplex.py"
return selection, group, hashlib.sha256(core.read_bytes()).hexdigest()
def run_case(args, selection, group, core_hash):
import torch
from torch.utils.data import DataLoader
from accelerate.utils import set_seed
import psi.trainers.posttrain_full_duplex as core
from psi.trainers.posttrain_third_party import PosttrainThirdPartyTrainer
from psi.utils import flatten, seed_everything
from scripts.train import _initialize_accelerator
from scripts.train_third_party import build_config
from scripts.train_third_party_strategy import source_batches
variant = args.variant
output = args.run_dir / variant
if output.exists():
raise ValueError("Refusing to overwrite an existing diagnostic case")
output.mkdir()
cfg = build_config(args.manifest, output, "train")
cfg.train.train_batch_size = cfg.train.target_effective_sources_per_gpu = 16
cfg.train.num_workers = 8
cfg.train.max_training_steps = 3266088
cfg.train.warmup_steps = 500
if cfg.train.resume_from_checkpoint is not None or cfg.train.resume_load_full_state:
raise ValueError("Diagnostic must initialize independently; no saved training state")
os.environ["HFD_CONTINUOUS_DATA_ROOT"] = cfg.data.continuous_audio_root
os.environ["HFD_BACKGROUND_ASSIGNMENT_PATH"] = cfg.data.background_assignment_path
os.environ["HFD_CONTINUOUS_DATA_CONTRACT_PATH"] = str(
Path(cfg.data.continuous_audio_root) / "reports/production_complete.json")
if variant == "metrics_cpu":
code = metric_candidate_source(inspect.getsource(core.PosttrainFullDuplexTrainer.iter_optimizer_steps))
namespace = {}
exec(compile(code, "<detached_metric_cpu_diagnostic>", "exec"), core.__dict__, namespace)
candidate = namespace["iter_optimizer_steps"]
else:
candidate = core.PosttrainFullDuplexTrainer.iter_optimizer_steps
class BenchmarkTrainer(PosttrainThirdPartyTrainer):
iter_optimizer_steps = candidate
@property
def num_steps_per_epoch(self):
return self.train_cfg.max_training_steps
def create_datasets(self):
self.train_dataset = self.data_cfg(split="train", transform_kwargs={"vlm_processor": self.vlm_processor})
self.val_dataset = None
if len(self.train_dataset) != len(selection["sessions"]):
raise ValueError("Frozen training count changed")
def save_checkpoint(self, *args, **kwargs):
raise RuntimeError("Performance diagnosis must never save model weights")
seed_everything(cfg.seed)
torch.cuda.set_device(0)
trainer = BenchmarkTrainer(cfg, 0)
trainer.init_models()
trainer.global_step = 0
accelerator = _initialize_accelerator(trainer)
trainer.create_datasets()
trainer.train_dataloader = DataLoader(trainer.train_dataset, batch_size=1, num_workers=0,
collate_fn=core._source_collator)
trainer.val_dataloader = None
trainer.create_optimizer_and_scheduler(cfg.train.max_training_steps)
trainer.prepare(accelerator)
set_seed(cfg.seed, device_specific=True)
initial_hash = tensor_fingerprint(accelerator.unwrap_model(trainer.model))
batch = next(source_batches(trainer, [group], workers=8))
trainer.set_train()
torch.cuda.reset_peak_memory_stats()
records = []
torch.cuda.synchronize()
total_started = time.perf_counter()
measured_start = None
for i, (_, values, _) in enumerate(trainer.iter_optimizer_steps(
batch, global_step_start=0, local_step=0, max_optimizer_events=UPDATES), 1):
if values["effective_sources_per_gpu"] != 16 or values["dense_windows_per_optimizer_step"] != 1:
raise ValueError("Benchmark changed the effective batch or grouped optimizer updates")
record = {"step": i, **{k: float(values[k]) for k in
("loss", "loss_action", "control_update_samples", "semantic_update")}}
if "loss_control" in values:
record["loss_control"] = float(values["loss_control"])
if not all(math.isfinite(v) for v in record.values()):
raise FloatingPointError("Nonfinite diagnostic loss")
records.append(record)
trainer.log(flatten(values, parent_key="train"))
if i == WARMUP:
torch.cuda.synchronize()
measured_start = time.perf_counter()
if i % 100 == 0:
print(f"[fixed16] {variant} {i}/{UPDATES}", flush=True)
torch.cuda.synchronize()
finished = time.perf_counter()
if len(records) != UPDATES or measured_start is None:
raise ValueError("Incomplete bounded test")
final_hash = tensor_fingerprint(accelerator.unwrap_model(trainer.model))
result = {"variant": variant, "status": "passed", "global_batch": 16,
"optimizer_updates": UPDATES, "warmup_updates": WARMUP,
"measured_seconds": finished - measured_start,
"updates_per_second": (UPDATES - WARMUP) / (finished - measured_start),
"total_compute_seconds": finished - total_started,
"peak_reserved_gib": torch.cuda.max_memory_reserved() / 2**30,
"initial_trainable_sha256": initial_hash, "final_trainable_sha256": final_hash,
"core_sha256": core_hash, "session_ids": [selection["session_ids"][i] for i in group],
"saved_weights": False, "loaded_checkpoint": None, "formal_training_updates": 0}
write_json(output / "benchmark.json", result)
write_json(output / "losses.json", records)
print(json.dumps(result), flush=True)
if variant == "baseline":
write_json(args.run_dir / "workflow_state.json", {"phase": "profiling_baseline", "time": time.time()})
with torch.profiler.profile(activities=[torch.profiler.ProfilerActivity.CPU,
torch.profiler.ProfilerActivity.CUDA]) as profiler:
for _ in trainer.iter_optimizer_steps(batch, global_step_start=UPDATES,
local_step=1, max_optimizer_events=40):
profiler.step()
(output / "profile_cpu.txt").write_text(profiler.key_averages().table(sort_by="self_cpu_time_total", row_limit=30))
(output / "profile_gpu.txt").write_text(profiler.key_averages().table(sort_by="self_cuda_time_total", row_limit=30))
trainer.finalize()
def summarize(run):
results = {v: json.loads((run / v / "benchmark.json").read_text()) for v in VARIANTS}
losses = {v: json.loads((run / v / "losses.json").read_text()) for v in VARIANTS}
original = results["baseline"]
comparison = {}
for variant in VARIANTS[1:]:
candidate = results[variant]
comparable = (candidate["global_batch"] == original["global_batch"] == 16
and candidate["initial_trainable_sha256"] == original["initial_trainable_sha256"]
and candidate["session_ids"] == original["session_ids"]
and candidate["core_sha256"] == original["core_sha256"])
delta = max(abs(a["loss"] - b["loss"]) for a, b in zip(losses["baseline"], losses[variant]))
comparison[variant] = {"same_inputs_and_initialization": comparable,
"final_parameters_bitwise_equal": candidate["final_trainable_sha256"] == original["final_trainable_sha256"],
"max_logged_loss_difference": delta}
baseline_speed = statistics.mean(results[v]["updates_per_second"] for v in ("baseline", "baseline_repeat"))
speed = results["metrics_cpu"]["updates_per_second"]
return {"status": "complete", "results": results, "comparison": comparison,
"candidate_speedup_over_mean_baseline": speed / baseline_speed,
"candidate_pure_epoch_hours_short_test_estimate": 3266088 / speed / 3600,
"formal_training_started": False, "production_implementation_modified": False,
"saved_weights": False, "batch": 16,
"note": "One representative group, not proof of long-run speed or training convergence"}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--manifest", type=Path, default=PROJECT / "data/hfd_third_party_multi_v5/training_scale_all_commands/selection.json")
parser.add_argument("--run-dir", type=Path, required=True)
parser.add_argument("--variant", choices=VARIANTS)
mode = parser.add_mutually_exclusive_group(required=True)
for name in ("check-only", "launch", "worker", "case"):
mode.add_argument(f"--{name}", action="store_true")
args = parser.parse_args()
os.chdir(PROJECT)
for key in ("RANK", "LOCAL_RANK", "WORLD_SIZE", "LOCAL_WORLD_SIZE", "MASTER_ADDR", "MASTER_PORT",
"ACCELERATE_USE_DEEPSPEED", "ACCELERATE_USE_FSDP", "ACCELERATE_USE_CPU"):
os.environ.pop(key, None)
from scripts.train_third_party_strategy import setup_environment
setup_environment()
os.environ["PYTHONDONTWRITEBYTECODE"] = "1"
selection, group, core_hash = prepare(args)
if args.check_only:
print(json.dumps({"passed": True, "gpu": 0, "batch": 16, "updates_per_case": UPDATES,
"cases": VARIANTS, "saved_weights": False, "core_sha256": core_hash})); return
argv = [sys.executable, str(Path(__file__).resolve()), "--manifest", str(args.manifest), "--run-dir", str(args.run_dir)]
if args.launch:
path = args.manifest.parent / f"{args.run_dir.name}.launch.json"
if path.exists() or args.run_dir.exists():
raise ValueError("Use a fresh diagnostic output")
write_json(path, {"schema": 1, "gpu": 0, "scope": "full_third_party", "project_root": str(PROJECT),
"run_dir": str(args.run_dir), "manifest_path": str(args.manifest),
"readiness_file": str(args.manifest.parent / "acceptance.json"),
"argv": [*argv, "--worker"], "preflight_command": [*argv, "--check-only"]})
raise SystemExit(subprocess.call([sys.executable, str(PROJECT / "scripts/train/third_party_run.py"), "start", "--spec", str(path)]))
if os.environ.get("CUDA_VISIBLE_DEVICES") != "0" or os.environ.get("HFD_NPROC_PER_NODE") != "1":
raise RuntimeError("Use the GPU0 managed --launch entry")
if args.case:
if args.variant is None:
raise ValueError("A diagnostic case requires --variant")
run_case(args, selection, group, core_hash); return
for variant in VARIANTS:
write_json(args.run_dir / "workflow_state.json", {"phase": variant, "time": time.time()})
subprocess.run([*argv, "--case", "--variant", variant], check=True, timeout=900)
write_json(args.run_dir / "report.json", summarize(args.run_dir))
write_json(args.run_dir / "workflow_state.json", {"phase": "completed", "time": time.time()})
if __name__ == "__main__":
main()
|