#!/usr/bin/env python3 """Disposable GPU0 throughput diagnosis; never save or resume model weights. The candidate changes only detached metric formatting in this process. Core source files, optimizer mathematics, samples, and the global batch stay intact. """ from __future__ import annotations import argparse import hashlib import inspect import json import math import os from pathlib import Path import statistics import subprocess import sys import textwrap import time PROJECT = Path(__file__).resolve().parents[1] sys.path[:0] = [str(PROJECT), str(PROJECT / "src")] from scripts.third_party_strategy_plan import write_json VARIANTS = ("baseline", "metrics_cpu", "baseline_repeat") UPDATES, WARMUP = 800, 100 def metric_candidate_source(source): """One guarded experimental edit after backward, only to detached logs.""" source = textwrap.dedent(source) matches = [line for line in source.splitlines() if line.strip() == "count = metric_row[17].clamp_min(1.0)"] if len(matches) != 1 or "self.optimizer.step()" not in source: raise ValueError("Training implementation changed; inspect metric experiment again") anchor = matches[0] if source.index(anchor) < source.index("self.optimizer.step()"): raise ValueError("Metric transformation must follow the optimizer update") indent = anchor[:len(anchor) - len(anchor.lstrip())] return source.replace(anchor, indent + "metric_row = metric_row.detach().cpu()\n" + anchor) def choose_group(selection): from scripts.third_party_scale_plan import coverage_plan groups = coverage_plan(selection)["epochs"][0]["source_indices"] candidates = [g for g in groups if len(g) == 16 and min(selection["sessions"][i]["dense_steps"] for i in g) > UPDATES] if not candidates: raise ValueError("No complete length-group with sixteen live sources for the entire test") candidates.sort(key=lambda g: max(selection["sessions"][i]["dense_steps"] for i in g)) return candidates[len(candidates) // 2] def tensor_fingerprint(model): digest = hashlib.sha256() for name, parameter in sorted(model.named_parameters()): if parameter.requires_grad: value = parameter.detach().cpu().contiguous() digest.update(name.encode()) digest.update(value.view(-1).view(__import__("torch").uint8).numpy().tobytes()) return digest.hexdigest() def prepare(args): from scripts.train_third_party import contained_output from scripts.data.prepare_third_party_scale import validate args.run_dir = contained_output(args.run_dir) args.manifest = args.manifest.resolve() selection, _, _ = validate(args.manifest) group = choose_group(selection) core = PROJECT / "src/psi/trainers/posttrain_full_duplex.py" return selection, group, hashlib.sha256(core.read_bytes()).hexdigest() def run_case(args, selection, group, core_hash): import torch from torch.utils.data import DataLoader from accelerate.utils import set_seed import psi.trainers.posttrain_full_duplex as core from psi.trainers.posttrain_third_party import PosttrainThirdPartyTrainer from psi.utils import flatten, seed_everything from scripts.train import _initialize_accelerator from scripts.train_third_party import build_config from scripts.train_third_party_strategy import source_batches variant = args.variant output = args.run_dir / variant if output.exists(): raise ValueError("Refusing to overwrite an existing diagnostic case") output.mkdir() cfg = build_config(args.manifest, output, "train") cfg.train.train_batch_size = cfg.train.target_effective_sources_per_gpu = 16 cfg.train.num_workers = 8 cfg.train.max_training_steps = 3266088 cfg.train.warmup_steps = 500 if cfg.train.resume_from_checkpoint is not None or cfg.train.resume_load_full_state: raise ValueError("Diagnostic must initialize independently; no saved training state") os.environ["HFD_CONTINUOUS_DATA_ROOT"] = cfg.data.continuous_audio_root os.environ["HFD_BACKGROUND_ASSIGNMENT_PATH"] = cfg.data.background_assignment_path os.environ["HFD_CONTINUOUS_DATA_CONTRACT_PATH"] = str( Path(cfg.data.continuous_audio_root) / "reports/production_complete.json") if variant == "metrics_cpu": code = metric_candidate_source(inspect.getsource(core.PosttrainFullDuplexTrainer.iter_optimizer_steps)) namespace = {} exec(compile(code, "", "exec"), core.__dict__, namespace) candidate = namespace["iter_optimizer_steps"] else: candidate = core.PosttrainFullDuplexTrainer.iter_optimizer_steps class BenchmarkTrainer(PosttrainThirdPartyTrainer): iter_optimizer_steps = candidate @property def num_steps_per_epoch(self): return self.train_cfg.max_training_steps def create_datasets(self): self.train_dataset = self.data_cfg(split="train", transform_kwargs={"vlm_processor": self.vlm_processor}) self.val_dataset = None if len(self.train_dataset) != len(selection["sessions"]): raise ValueError("Frozen training count changed") def save_checkpoint(self, *args, **kwargs): raise RuntimeError("Performance diagnosis must never save model weights") seed_everything(cfg.seed) torch.cuda.set_device(0) trainer = BenchmarkTrainer(cfg, 0) trainer.init_models() trainer.global_step = 0 accelerator = _initialize_accelerator(trainer) trainer.create_datasets() trainer.train_dataloader = DataLoader(trainer.train_dataset, batch_size=1, num_workers=0, collate_fn=core._source_collator) trainer.val_dataloader = None trainer.create_optimizer_and_scheduler(cfg.train.max_training_steps) trainer.prepare(accelerator) set_seed(cfg.seed, device_specific=True) initial_hash = tensor_fingerprint(accelerator.unwrap_model(trainer.model)) batch = next(source_batches(trainer, [group], workers=8)) trainer.set_train() torch.cuda.reset_peak_memory_stats() records = [] torch.cuda.synchronize() total_started = time.perf_counter() measured_start = None for i, (_, values, _) in enumerate(trainer.iter_optimizer_steps( batch, global_step_start=0, local_step=0, max_optimizer_events=UPDATES), 1): if values["effective_sources_per_gpu"] != 16 or values["dense_windows_per_optimizer_step"] != 1: raise ValueError("Benchmark changed the effective batch or grouped optimizer updates") record = {"step": i, **{k: float(values[k]) for k in ("loss", "loss_action", "control_update_samples", "semantic_update")}} if "loss_control" in values: record["loss_control"] = float(values["loss_control"]) if not all(math.isfinite(v) for v in record.values()): raise FloatingPointError("Nonfinite diagnostic loss") records.append(record) trainer.log(flatten(values, parent_key="train")) if i == WARMUP: torch.cuda.synchronize() measured_start = time.perf_counter() if i % 100 == 0: print(f"[fixed16] {variant} {i}/{UPDATES}", flush=True) torch.cuda.synchronize() finished = time.perf_counter() if len(records) != UPDATES or measured_start is None: raise ValueError("Incomplete bounded test") final_hash = tensor_fingerprint(accelerator.unwrap_model(trainer.model)) result = {"variant": variant, "status": "passed", "global_batch": 16, "optimizer_updates": UPDATES, "warmup_updates": WARMUP, "measured_seconds": finished - measured_start, "updates_per_second": (UPDATES - WARMUP) / (finished - measured_start), "total_compute_seconds": finished - total_started, "peak_reserved_gib": torch.cuda.max_memory_reserved() / 2**30, "initial_trainable_sha256": initial_hash, "final_trainable_sha256": final_hash, "core_sha256": core_hash, "session_ids": [selection["session_ids"][i] for i in group], "saved_weights": False, "loaded_checkpoint": None, "formal_training_updates": 0} write_json(output / "benchmark.json", result) write_json(output / "losses.json", records) print(json.dumps(result), flush=True) if variant == "baseline": write_json(args.run_dir / "workflow_state.json", {"phase": "profiling_baseline", "time": time.time()}) with torch.profiler.profile(activities=[torch.profiler.ProfilerActivity.CPU, torch.profiler.ProfilerActivity.CUDA]) as profiler: for _ in trainer.iter_optimizer_steps(batch, global_step_start=UPDATES, local_step=1, max_optimizer_events=40): profiler.step() (output / "profile_cpu.txt").write_text(profiler.key_averages().table(sort_by="self_cpu_time_total", row_limit=30)) (output / "profile_gpu.txt").write_text(profiler.key_averages().table(sort_by="self_cuda_time_total", row_limit=30)) trainer.finalize() def summarize(run): results = {v: json.loads((run / v / "benchmark.json").read_text()) for v in VARIANTS} losses = {v: json.loads((run / v / "losses.json").read_text()) for v in VARIANTS} original = results["baseline"] comparison = {} for variant in VARIANTS[1:]: candidate = results[variant] comparable = (candidate["global_batch"] == original["global_batch"] == 16 and candidate["initial_trainable_sha256"] == original["initial_trainable_sha256"] and candidate["session_ids"] == original["session_ids"] and candidate["core_sha256"] == original["core_sha256"]) delta = max(abs(a["loss"] - b["loss"]) for a, b in zip(losses["baseline"], losses[variant])) comparison[variant] = {"same_inputs_and_initialization": comparable, "final_parameters_bitwise_equal": candidate["final_trainable_sha256"] == original["final_trainable_sha256"], "max_logged_loss_difference": delta} baseline_speed = statistics.mean(results[v]["updates_per_second"] for v in ("baseline", "baseline_repeat")) speed = results["metrics_cpu"]["updates_per_second"] return {"status": "complete", "results": results, "comparison": comparison, "candidate_speedup_over_mean_baseline": speed / baseline_speed, "candidate_pure_epoch_hours_short_test_estimate": 3266088 / speed / 3600, "formal_training_started": False, "production_implementation_modified": False, "saved_weights": False, "batch": 16, "note": "One representative group, not proof of long-run speed or training convergence"} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--manifest", type=Path, default=PROJECT / "data/hfd_third_party_multi_v5/training_scale_all_commands/selection.json") parser.add_argument("--run-dir", type=Path, required=True) parser.add_argument("--variant", choices=VARIANTS) mode = parser.add_mutually_exclusive_group(required=True) for name in ("check-only", "launch", "worker", "case"): mode.add_argument(f"--{name}", action="store_true") args = parser.parse_args() os.chdir(PROJECT) for key in ("RANK", "LOCAL_RANK", "WORLD_SIZE", "LOCAL_WORLD_SIZE", "MASTER_ADDR", "MASTER_PORT", "ACCELERATE_USE_DEEPSPEED", "ACCELERATE_USE_FSDP", "ACCELERATE_USE_CPU"): os.environ.pop(key, None) from scripts.train_third_party_strategy import setup_environment setup_environment() os.environ["PYTHONDONTWRITEBYTECODE"] = "1" selection, group, core_hash = prepare(args) if args.check_only: print(json.dumps({"passed": True, "gpu": 0, "batch": 16, "updates_per_case": UPDATES, "cases": VARIANTS, "saved_weights": False, "core_sha256": core_hash})); return argv = [sys.executable, str(Path(__file__).resolve()), "--manifest", str(args.manifest), "--run-dir", str(args.run_dir)] if args.launch: path = args.manifest.parent / f"{args.run_dir.name}.launch.json" if path.exists() or args.run_dir.exists(): raise ValueError("Use a fresh diagnostic output") write_json(path, {"schema": 1, "gpu": 0, "scope": "full_third_party", "project_root": str(PROJECT), "run_dir": str(args.run_dir), "manifest_path": str(args.manifest), "readiness_file": str(args.manifest.parent / "acceptance.json"), "argv": [*argv, "--worker"], "preflight_command": [*argv, "--check-only"]}) raise SystemExit(subprocess.call([sys.executable, str(PROJECT / "scripts/train/third_party_run.py"), "start", "--spec", str(path)])) if os.environ.get("CUDA_VISIBLE_DEVICES") != "0" or os.environ.get("HFD_NPROC_PER_NODE") != "1": raise RuntimeError("Use the GPU0 managed --launch entry") if args.case: if args.variant is None: raise ValueError("A diagnostic case requires --variant") run_case(args, selection, group, core_hash); return for variant in VARIANTS: write_json(args.run_dir / "workflow_state.json", {"phase": variant, "time": time.time()}) subprocess.run([*argv, "--case", "--variant", variant], check=True, timeout=900) write_json(args.run_dir / "report.json", summarize(args.run_dir)) write_json(args.run_dir / "workflow_state.json", {"phase": "completed", "time": time.time()}) if __name__ == "__main__": main()