Download scripts/benchmark_third_party_fixed16.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 13.6 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/benchmark_third_party_fixed16.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/benchmark_third_party_fixed16.py
-
curl -L -o benchmark_third_party_fixed16.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/benchmark_third_party_fixed16.py
13.6 kB
| #!/usr/bin/env python3 | |
| """Disposable GPU0 throughput diagnosis; never save or resume model weights. | |
| The candidate changes only detached metric formatting in this process. Core | |
| source files, optimizer mathematics, samples, and the global batch stay intact. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import inspect | |
| import json | |
| import math | |
| import os | |
| from pathlib import Path | |
| import statistics | |
| import subprocess | |
| import sys | |
| import textwrap | |
| import time | |
| PROJECT = Path(__file__).resolve().parents[1] | |
| sys.path[:0] = [str(PROJECT), str(PROJECT / "src")] | |
| from scripts.third_party_strategy_plan import write_json | |
| VARIANTS = ("baseline", "metrics_cpu", "baseline_repeat") | |
| UPDATES, WARMUP = 800, 100 | |
| def metric_candidate_source(source): | |
| """One guarded experimental edit after backward, only to detached logs.""" | |
| source = textwrap.dedent(source) | |
| matches = [line for line in source.splitlines() if line.strip() == "count = metric_row[17].clamp_min(1.0)"] | |
| if len(matches) != 1 or "self.optimizer.step()" not in source: | |
| raise ValueError("Training implementation changed; inspect metric experiment again") | |
| anchor = matches[0] | |
| if source.index(anchor) < source.index("self.optimizer.step()"): | |
| raise ValueError("Metric transformation must follow the optimizer update") | |
| indent = anchor[:len(anchor) - len(anchor.lstrip())] | |
| return source.replace(anchor, indent + "metric_row = metric_row.detach().cpu()\n" + anchor) | |
| def choose_group(selection): | |
| from scripts.third_party_scale_plan import coverage_plan | |
| groups = coverage_plan(selection)["epochs"][0]["source_indices"] | |
| candidates = [g for g in groups if len(g) == 16 | |
| and min(selection["sessions"][i]["dense_steps"] for i in g) > UPDATES] | |
| if not candidates: | |
| raise ValueError("No complete length-group with sixteen live sources for the entire test") | |
| candidates.sort(key=lambda g: max(selection["sessions"][i]["dense_steps"] for i in g)) | |
| return candidates[len(candidates) // 2] | |
| def tensor_fingerprint(model): | |
| digest = hashlib.sha256() | |
| for name, parameter in sorted(model.named_parameters()): | |
| if parameter.requires_grad: | |
| value = parameter.detach().cpu().contiguous() | |
| digest.update(name.encode()) | |
| digest.update(value.view(-1).view(__import__("torch").uint8).numpy().tobytes()) | |
| return digest.hexdigest() | |
| def prepare(args): | |
| from scripts.train_third_party import contained_output | |
| from scripts.data.prepare_third_party_scale import validate | |
| args.run_dir = contained_output(args.run_dir) | |
| args.manifest = args.manifest.resolve() | |
| selection, _, _ = validate(args.manifest) | |
| group = choose_group(selection) | |
| core = PROJECT / "src/psi/trainers/posttrain_full_duplex.py" | |
| return selection, group, hashlib.sha256(core.read_bytes()).hexdigest() | |
| def run_case(args, selection, group, core_hash): | |
| import torch | |
| from torch.utils.data import DataLoader | |
| from accelerate.utils import set_seed | |
| import psi.trainers.posttrain_full_duplex as core | |
| from psi.trainers.posttrain_third_party import PosttrainThirdPartyTrainer | |
| from psi.utils import flatten, seed_everything | |
| from scripts.train import _initialize_accelerator | |
| from scripts.train_third_party import build_config | |
| from scripts.train_third_party_strategy import source_batches | |
| variant = args.variant | |
| output = args.run_dir / variant | |
| if output.exists(): | |
| raise ValueError("Refusing to overwrite an existing diagnostic case") | |
| output.mkdir() | |
| cfg = build_config(args.manifest, output, "train") | |
| cfg.train.train_batch_size = cfg.train.target_effective_sources_per_gpu = 16 | |
| cfg.train.num_workers = 8 | |
| cfg.train.max_training_steps = 3266088 | |
| cfg.train.warmup_steps = 500 | |
| if cfg.train.resume_from_checkpoint is not None or cfg.train.resume_load_full_state: | |
| raise ValueError("Diagnostic must initialize independently; no saved training state") | |
| os.environ["HFD_CONTINUOUS_DATA_ROOT"] = cfg.data.continuous_audio_root | |
| os.environ["HFD_BACKGROUND_ASSIGNMENT_PATH"] = cfg.data.background_assignment_path | |
| os.environ["HFD_CONTINUOUS_DATA_CONTRACT_PATH"] = str( | |
| Path(cfg.data.continuous_audio_root) / "reports/production_complete.json") | |
| if variant == "metrics_cpu": | |
| code = metric_candidate_source(inspect.getsource(core.PosttrainFullDuplexTrainer.iter_optimizer_steps)) | |
| namespace = {} | |
| exec(compile(code, "<detached_metric_cpu_diagnostic>", "exec"), core.__dict__, namespace) | |
| candidate = namespace["iter_optimizer_steps"] | |
| else: | |
| candidate = core.PosttrainFullDuplexTrainer.iter_optimizer_steps | |
| class BenchmarkTrainer(PosttrainThirdPartyTrainer): | |
| iter_optimizer_steps = candidate | |
| def num_steps_per_epoch(self): | |
| return self.train_cfg.max_training_steps | |
| def create_datasets(self): | |
| self.train_dataset = self.data_cfg(split="train", transform_kwargs={"vlm_processor": self.vlm_processor}) | |
| self.val_dataset = None | |
| if len(self.train_dataset) != len(selection["sessions"]): | |
| raise ValueError("Frozen training count changed") | |
| def save_checkpoint(self, *args, **kwargs): | |
| raise RuntimeError("Performance diagnosis must never save model weights") | |
| seed_everything(cfg.seed) | |
| torch.cuda.set_device(0) | |
| trainer = BenchmarkTrainer(cfg, 0) | |
| trainer.init_models() | |
| trainer.global_step = 0 | |
| accelerator = _initialize_accelerator(trainer) | |
| trainer.create_datasets() | |
| trainer.train_dataloader = DataLoader(trainer.train_dataset, batch_size=1, num_workers=0, | |
| collate_fn=core._source_collator) | |
| trainer.val_dataloader = None | |
| trainer.create_optimizer_and_scheduler(cfg.train.max_training_steps) | |
| trainer.prepare(accelerator) | |
| set_seed(cfg.seed, device_specific=True) | |
| initial_hash = tensor_fingerprint(accelerator.unwrap_model(trainer.model)) | |
| batch = next(source_batches(trainer, [group], workers=8)) | |
| trainer.set_train() | |
| torch.cuda.reset_peak_memory_stats() | |
| records = [] | |
| torch.cuda.synchronize() | |
| total_started = time.perf_counter() | |
| measured_start = None | |
| for i, (_, values, _) in enumerate(trainer.iter_optimizer_steps( | |
| batch, global_step_start=0, local_step=0, max_optimizer_events=UPDATES), 1): | |
| if values["effective_sources_per_gpu"] != 16 or values["dense_windows_per_optimizer_step"] != 1: | |
| raise ValueError("Benchmark changed the effective batch or grouped optimizer updates") | |
| record = {"step": i, **{k: float(values[k]) for k in | |
| ("loss", "loss_action", "control_update_samples", "semantic_update")}} | |
| if "loss_control" in values: | |
| record["loss_control"] = float(values["loss_control"]) | |
| if not all(math.isfinite(v) for v in record.values()): | |
| raise FloatingPointError("Nonfinite diagnostic loss") | |
| records.append(record) | |
| trainer.log(flatten(values, parent_key="train")) | |
| if i == WARMUP: | |
| torch.cuda.synchronize() | |
| measured_start = time.perf_counter() | |
| if i % 100 == 0: | |
| print(f"[fixed16] {variant} {i}/{UPDATES}", flush=True) | |
| torch.cuda.synchronize() | |
| finished = time.perf_counter() | |
| if len(records) != UPDATES or measured_start is None: | |
| raise ValueError("Incomplete bounded test") | |
| final_hash = tensor_fingerprint(accelerator.unwrap_model(trainer.model)) | |
| result = {"variant": variant, "status": "passed", "global_batch": 16, | |
| "optimizer_updates": UPDATES, "warmup_updates": WARMUP, | |
| "measured_seconds": finished - measured_start, | |
| "updates_per_second": (UPDATES - WARMUP) / (finished - measured_start), | |
| "total_compute_seconds": finished - total_started, | |
| "peak_reserved_gib": torch.cuda.max_memory_reserved() / 2**30, | |
| "initial_trainable_sha256": initial_hash, "final_trainable_sha256": final_hash, | |
| "core_sha256": core_hash, "session_ids": [selection["session_ids"][i] for i in group], | |
| "saved_weights": False, "loaded_checkpoint": None, "formal_training_updates": 0} | |
| write_json(output / "benchmark.json", result) | |
| write_json(output / "losses.json", records) | |
| print(json.dumps(result), flush=True) | |
| if variant == "baseline": | |
| write_json(args.run_dir / "workflow_state.json", {"phase": "profiling_baseline", "time": time.time()}) | |
| with torch.profiler.profile(activities=[torch.profiler.ProfilerActivity.CPU, | |
| torch.profiler.ProfilerActivity.CUDA]) as profiler: | |
| for _ in trainer.iter_optimizer_steps(batch, global_step_start=UPDATES, | |
| local_step=1, max_optimizer_events=40): | |
| profiler.step() | |
| (output / "profile_cpu.txt").write_text(profiler.key_averages().table(sort_by="self_cpu_time_total", row_limit=30)) | |
| (output / "profile_gpu.txt").write_text(profiler.key_averages().table(sort_by="self_cuda_time_total", row_limit=30)) | |
| trainer.finalize() | |
| def summarize(run): | |
| results = {v: json.loads((run / v / "benchmark.json").read_text()) for v in VARIANTS} | |
| losses = {v: json.loads((run / v / "losses.json").read_text()) for v in VARIANTS} | |
| original = results["baseline"] | |
| comparison = {} | |
| for variant in VARIANTS[1:]: | |
| candidate = results[variant] | |
| comparable = (candidate["global_batch"] == original["global_batch"] == 16 | |
| and candidate["initial_trainable_sha256"] == original["initial_trainable_sha256"] | |
| and candidate["session_ids"] == original["session_ids"] | |
| and candidate["core_sha256"] == original["core_sha256"]) | |
| delta = max(abs(a["loss"] - b["loss"]) for a, b in zip(losses["baseline"], losses[variant])) | |
| comparison[variant] = {"same_inputs_and_initialization": comparable, | |
| "final_parameters_bitwise_equal": candidate["final_trainable_sha256"] == original["final_trainable_sha256"], | |
| "max_logged_loss_difference": delta} | |
| baseline_speed = statistics.mean(results[v]["updates_per_second"] for v in ("baseline", "baseline_repeat")) | |
| speed = results["metrics_cpu"]["updates_per_second"] | |
| return {"status": "complete", "results": results, "comparison": comparison, | |
| "candidate_speedup_over_mean_baseline": speed / baseline_speed, | |
| "candidate_pure_epoch_hours_short_test_estimate": 3266088 / speed / 3600, | |
| "formal_training_started": False, "production_implementation_modified": False, | |
| "saved_weights": False, "batch": 16, | |
| "note": "One representative group, not proof of long-run speed or training convergence"} | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--manifest", type=Path, default=PROJECT / "data/hfd_third_party_multi_v5/training_scale_all_commands/selection.json") | |
| parser.add_argument("--run-dir", type=Path, required=True) | |
| parser.add_argument("--variant", choices=VARIANTS) | |
| mode = parser.add_mutually_exclusive_group(required=True) | |
| for name in ("check-only", "launch", "worker", "case"): | |
| mode.add_argument(f"--{name}", action="store_true") | |
| args = parser.parse_args() | |
| os.chdir(PROJECT) | |
| for key in ("RANK", "LOCAL_RANK", "WORLD_SIZE", "LOCAL_WORLD_SIZE", "MASTER_ADDR", "MASTER_PORT", | |
| "ACCELERATE_USE_DEEPSPEED", "ACCELERATE_USE_FSDP", "ACCELERATE_USE_CPU"): | |
| os.environ.pop(key, None) | |
| from scripts.train_third_party_strategy import setup_environment | |
| setup_environment() | |
| os.environ["PYTHONDONTWRITEBYTECODE"] = "1" | |
| selection, group, core_hash = prepare(args) | |
| if args.check_only: | |
| print(json.dumps({"passed": True, "gpu": 0, "batch": 16, "updates_per_case": UPDATES, | |
| "cases": VARIANTS, "saved_weights": False, "core_sha256": core_hash})); return | |
| argv = [sys.executable, str(Path(__file__).resolve()), "--manifest", str(args.manifest), "--run-dir", str(args.run_dir)] | |
| if args.launch: | |
| path = args.manifest.parent / f"{args.run_dir.name}.launch.json" | |
| if path.exists() or args.run_dir.exists(): | |
| raise ValueError("Use a fresh diagnostic output") | |
| write_json(path, {"schema": 1, "gpu": 0, "scope": "full_third_party", "project_root": str(PROJECT), | |
| "run_dir": str(args.run_dir), "manifest_path": str(args.manifest), | |
| "readiness_file": str(args.manifest.parent / "acceptance.json"), | |
| "argv": [*argv, "--worker"], "preflight_command": [*argv, "--check-only"]}) | |
| raise SystemExit(subprocess.call([sys.executable, str(PROJECT / "scripts/train/third_party_run.py"), "start", "--spec", str(path)])) | |
| if os.environ.get("CUDA_VISIBLE_DEVICES") != "0" or os.environ.get("HFD_NPROC_PER_NODE") != "1": | |
| raise RuntimeError("Use the GPU0 managed --launch entry") | |
| if args.case: | |
| if args.variant is None: | |
| raise ValueError("A diagnostic case requires --variant") | |
| run_case(args, selection, group, core_hash); return | |
| for variant in VARIANTS: | |
| write_json(args.run_dir / "workflow_state.json", {"phase": variant, "time": time.time()}) | |
| subprocess.run([*argv, "--case", "--variant", variant], check=True, timeout=900) | |
| write_json(args.run_dir / "report.json", summarize(args.run_dir)) | |
| write_json(args.run_dir / "workflow_state.json", {"phase": "completed", "time": time.time()}) | |
| if __name__ == "__main__": | |
| main() | |