code-HFD_lzk / scripts /benchmark_third_party_fixed16.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
55e0599 verified
Raw History Blame Contribute Delete
13.6 kB
#!/usr/bin/env python3
"""Disposable GPU0 throughput diagnosis; never save or resume model weights.
The candidate changes only detached metric formatting in this process. Core
source files, optimizer mathematics, samples, and the global batch stay intact.
"""
from __future__ import annotations
import argparse
import hashlib
import inspect
import json
import math
import os
from pathlib import Path
import statistics
import subprocess
import sys
import textwrap
import time
PROJECT = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(PROJECT), str(PROJECT / "src")]
from scripts.third_party_strategy_plan import write_json
VARIANTS = ("baseline", "metrics_cpu", "baseline_repeat")
UPDATES, WARMUP = 800, 100
def metric_candidate_source(source):
"""One guarded experimental edit after backward, only to detached logs."""
source = textwrap.dedent(source)
matches = [line for line in source.splitlines() if line.strip() == "count = metric_row[17].clamp_min(1.0)"]
if len(matches) != 1 or "self.optimizer.step()" not in source:
raise ValueError("Training implementation changed; inspect metric experiment again")
anchor = matches[0]
if source.index(anchor) < source.index("self.optimizer.step()"):
raise ValueError("Metric transformation must follow the optimizer update")
indent = anchor[:len(anchor) - len(anchor.lstrip())]
return source.replace(anchor, indent + "metric_row = metric_row.detach().cpu()\n" + anchor)
def choose_group(selection):
from scripts.third_party_scale_plan import coverage_plan
groups = coverage_plan(selection)["epochs"][0]["source_indices"]
candidates = [g for g in groups if len(g) == 16
and min(selection["sessions"][i]["dense_steps"] for i in g) > UPDATES]
if not candidates:
raise ValueError("No complete length-group with sixteen live sources for the entire test")
candidates.sort(key=lambda g: max(selection["sessions"][i]["dense_steps"] for i in g))
return candidates[len(candidates) // 2]
def tensor_fingerprint(model):
digest = hashlib.sha256()
for name, parameter in sorted(model.named_parameters()):
if parameter.requires_grad:
value = parameter.detach().cpu().contiguous()
digest.update(name.encode())
digest.update(value.view(-1).view(__import__("torch").uint8).numpy().tobytes())
return digest.hexdigest()
def prepare(args):
from scripts.train_third_party import contained_output
from scripts.data.prepare_third_party_scale import validate
args.run_dir = contained_output(args.run_dir)
args.manifest = args.manifest.resolve()
selection, _, _ = validate(args.manifest)
group = choose_group(selection)
core = PROJECT / "src/psi/trainers/posttrain_full_duplex.py"
return selection, group, hashlib.sha256(core.read_bytes()).hexdigest()
def run_case(args, selection, group, core_hash):
import torch
from torch.utils.data import DataLoader
from accelerate.utils import set_seed
import psi.trainers.posttrain_full_duplex as core
from psi.trainers.posttrain_third_party import PosttrainThirdPartyTrainer
from psi.utils import flatten, seed_everything
from scripts.train import _initialize_accelerator
from scripts.train_third_party import build_config
from scripts.train_third_party_strategy import source_batches
variant = args.variant
output = args.run_dir / variant
if output.exists():
raise ValueError("Refusing to overwrite an existing diagnostic case")
output.mkdir()
cfg = build_config(args.manifest, output, "train")
cfg.train.train_batch_size = cfg.train.target_effective_sources_per_gpu = 16
cfg.train.num_workers = 8
cfg.train.max_training_steps = 3266088
cfg.train.warmup_steps = 500
if cfg.train.resume_from_checkpoint is not None or cfg.train.resume_load_full_state:
raise ValueError("Diagnostic must initialize independently; no saved training state")
os.environ["HFD_CONTINUOUS_DATA_ROOT"] = cfg.data.continuous_audio_root
os.environ["HFD_BACKGROUND_ASSIGNMENT_PATH"] = cfg.data.background_assignment_path
os.environ["HFD_CONTINUOUS_DATA_CONTRACT_PATH"] = str(
Path(cfg.data.continuous_audio_root) / "reports/production_complete.json")
if variant == "metrics_cpu":
code = metric_candidate_source(inspect.getsource(core.PosttrainFullDuplexTrainer.iter_optimizer_steps))
namespace = {}
exec(compile(code, "<detached_metric_cpu_diagnostic>", "exec"), core.__dict__, namespace)
candidate = namespace["iter_optimizer_steps"]
else:
candidate = core.PosttrainFullDuplexTrainer.iter_optimizer_steps
class BenchmarkTrainer(PosttrainThirdPartyTrainer):
iter_optimizer_steps = candidate
@property
def num_steps_per_epoch(self):
return self.train_cfg.max_training_steps
def create_datasets(self):
self.train_dataset = self.data_cfg(split="train", transform_kwargs={"vlm_processor": self.vlm_processor})
self.val_dataset = None
if len(self.train_dataset) != len(selection["sessions"]):
raise ValueError("Frozen training count changed")
def save_checkpoint(self, *args, **kwargs):
raise RuntimeError("Performance diagnosis must never save model weights")
seed_everything(cfg.seed)
torch.cuda.set_device(0)
trainer = BenchmarkTrainer(cfg, 0)
trainer.init_models()
trainer.global_step = 0
accelerator = _initialize_accelerator(trainer)
trainer.create_datasets()
trainer.train_dataloader = DataLoader(trainer.train_dataset, batch_size=1, num_workers=0,
collate_fn=core._source_collator)
trainer.val_dataloader = None
trainer.create_optimizer_and_scheduler(cfg.train.max_training_steps)
trainer.prepare(accelerator)
set_seed(cfg.seed, device_specific=True)
initial_hash = tensor_fingerprint(accelerator.unwrap_model(trainer.model))
batch = next(source_batches(trainer, [group], workers=8))
trainer.set_train()
torch.cuda.reset_peak_memory_stats()
records = []
torch.cuda.synchronize()
total_started = time.perf_counter()
measured_start = None
for i, (_, values, _) in enumerate(trainer.iter_optimizer_steps(
batch, global_step_start=0, local_step=0, max_optimizer_events=UPDATES), 1):
if values["effective_sources_per_gpu"] != 16 or values["dense_windows_per_optimizer_step"] != 1:
raise ValueError("Benchmark changed the effective batch or grouped optimizer updates")
record = {"step": i, **{k: float(values[k]) for k in
("loss", "loss_action", "control_update_samples", "semantic_update")}}
if "loss_control" in values:
record["loss_control"] = float(values["loss_control"])
if not all(math.isfinite(v) for v in record.values()):
raise FloatingPointError("Nonfinite diagnostic loss")
records.append(record)
trainer.log(flatten(values, parent_key="train"))
if i == WARMUP:
torch.cuda.synchronize()
measured_start = time.perf_counter()
if i % 100 == 0:
print(f"[fixed16] {variant} {i}/{UPDATES}", flush=True)
torch.cuda.synchronize()
finished = time.perf_counter()
if len(records) != UPDATES or measured_start is None:
raise ValueError("Incomplete bounded test")
final_hash = tensor_fingerprint(accelerator.unwrap_model(trainer.model))
result = {"variant": variant, "status": "passed", "global_batch": 16,
"optimizer_updates": UPDATES, "warmup_updates": WARMUP,
"measured_seconds": finished - measured_start,
"updates_per_second": (UPDATES - WARMUP) / (finished - measured_start),
"total_compute_seconds": finished - total_started,
"peak_reserved_gib": torch.cuda.max_memory_reserved() / 2**30,
"initial_trainable_sha256": initial_hash, "final_trainable_sha256": final_hash,
"core_sha256": core_hash, "session_ids": [selection["session_ids"][i] for i in group],
"saved_weights": False, "loaded_checkpoint": None, "formal_training_updates": 0}
write_json(output / "benchmark.json", result)
write_json(output / "losses.json", records)
print(json.dumps(result), flush=True)
if variant == "baseline":
write_json(args.run_dir / "workflow_state.json", {"phase": "profiling_baseline", "time": time.time()})
with torch.profiler.profile(activities=[torch.profiler.ProfilerActivity.CPU,
torch.profiler.ProfilerActivity.CUDA]) as profiler:
for _ in trainer.iter_optimizer_steps(batch, global_step_start=UPDATES,
local_step=1, max_optimizer_events=40):
profiler.step()
(output / "profile_cpu.txt").write_text(profiler.key_averages().table(sort_by="self_cpu_time_total", row_limit=30))
(output / "profile_gpu.txt").write_text(profiler.key_averages().table(sort_by="self_cuda_time_total", row_limit=30))
trainer.finalize()
def summarize(run):
results = {v: json.loads((run / v / "benchmark.json").read_text()) for v in VARIANTS}
losses = {v: json.loads((run / v / "losses.json").read_text()) for v in VARIANTS}
original = results["baseline"]
comparison = {}
for variant in VARIANTS[1:]:
candidate = results[variant]
comparable = (candidate["global_batch"] == original["global_batch"] == 16
and candidate["initial_trainable_sha256"] == original["initial_trainable_sha256"]
and candidate["session_ids"] == original["session_ids"]
and candidate["core_sha256"] == original["core_sha256"])
delta = max(abs(a["loss"] - b["loss"]) for a, b in zip(losses["baseline"], losses[variant]))
comparison[variant] = {"same_inputs_and_initialization": comparable,
"final_parameters_bitwise_equal": candidate["final_trainable_sha256"] == original["final_trainable_sha256"],
"max_logged_loss_difference": delta}
baseline_speed = statistics.mean(results[v]["updates_per_second"] for v in ("baseline", "baseline_repeat"))
speed = results["metrics_cpu"]["updates_per_second"]
return {"status": "complete", "results": results, "comparison": comparison,
"candidate_speedup_over_mean_baseline": speed / baseline_speed,
"candidate_pure_epoch_hours_short_test_estimate": 3266088 / speed / 3600,
"formal_training_started": False, "production_implementation_modified": False,
"saved_weights": False, "batch": 16,
"note": "One representative group, not proof of long-run speed or training convergence"}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--manifest", type=Path, default=PROJECT / "data/hfd_third_party_multi_v5/training_scale_all_commands/selection.json")
parser.add_argument("--run-dir", type=Path, required=True)
parser.add_argument("--variant", choices=VARIANTS)
mode = parser.add_mutually_exclusive_group(required=True)
for name in ("check-only", "launch", "worker", "case"):
mode.add_argument(f"--{name}", action="store_true")
args = parser.parse_args()
os.chdir(PROJECT)
for key in ("RANK", "LOCAL_RANK", "WORLD_SIZE", "LOCAL_WORLD_SIZE", "MASTER_ADDR", "MASTER_PORT",
"ACCELERATE_USE_DEEPSPEED", "ACCELERATE_USE_FSDP", "ACCELERATE_USE_CPU"):
os.environ.pop(key, None)
from scripts.train_third_party_strategy import setup_environment
setup_environment()
os.environ["PYTHONDONTWRITEBYTECODE"] = "1"
selection, group, core_hash = prepare(args)
if args.check_only:
print(json.dumps({"passed": True, "gpu": 0, "batch": 16, "updates_per_case": UPDATES,
"cases": VARIANTS, "saved_weights": False, "core_sha256": core_hash})); return
argv = [sys.executable, str(Path(__file__).resolve()), "--manifest", str(args.manifest), "--run-dir", str(args.run_dir)]
if args.launch:
path = args.manifest.parent / f"{args.run_dir.name}.launch.json"
if path.exists() or args.run_dir.exists():
raise ValueError("Use a fresh diagnostic output")
write_json(path, {"schema": 1, "gpu": 0, "scope": "full_third_party", "project_root": str(PROJECT),
"run_dir": str(args.run_dir), "manifest_path": str(args.manifest),
"readiness_file": str(args.manifest.parent / "acceptance.json"),
"argv": [*argv, "--worker"], "preflight_command": [*argv, "--check-only"]})
raise SystemExit(subprocess.call([sys.executable, str(PROJECT / "scripts/train/third_party_run.py"), "start", "--spec", str(path)]))
if os.environ.get("CUDA_VISIBLE_DEVICES") != "0" or os.environ.get("HFD_NPROC_PER_NODE") != "1":
raise RuntimeError("Use the GPU0 managed --launch entry")
if args.case:
if args.variant is None:
raise ValueError("A diagnostic case requires --variant")
run_case(args, selection, group, core_hash); return
for variant in VARIANTS:
write_json(args.run_dir / "workflow_state.json", {"phase": variant, "time": time.time()})
subprocess.run([*argv, "--case", "--variant", variant], check=True, timeout=900)
write_json(args.run_dir / "report.json", summarize(args.run_dir))
write_json(args.run_dir / "workflow_state.json", {"phase": "completed", "time": time.time()})
if __name__ == "__main__":
main()