#!/usr/bin/env python3 """Managed GPU0 full eligible-data stage, with held-out metrics and final videos.""" from __future__ import annotations import argparse import hashlib import importlib.util import json import math import os from pathlib import Path import shutil import subprocess import sys import time PROJECT = Path(__file__).resolve().parents[1] sys.path[:0] = [str(PROJECT), str(PROJECT / "src")] from scripts.third_party_strategy_plan import write_json from scripts.third_party_scale_plan import coverage_plan ORIGINAL_PILOT_CONFIG = PROJECT / "runs/third_party_train_20261004_183932/run_config.json" CODE_FILES = ("scripts/train_third_party_scale.py", "scripts/third_party_scale_plan.py", "scripts/train_third_party.py", "scripts/train.py", "scripts/third_party_joint_init.py", "scripts/data/prepare_third_party_scale.py", "scripts/infer/third_party_scale_metrics.py", "scripts/infer/render_third_party_scale.py", "scripts/train/third_party_run.py", "src/psi/trainers/posttrain_full_duplex.py", "src/psi/trainers/posttrain_v1.py", "src/psi/data/seed/qwen_streaming_audio.py", "src/psi/data/seed/third_party_frozen_dataset.py") def read(path): return json.loads(Path(path).read_text()) def digest(path): h = hashlib.sha256() with Path(path).open("rb") as f: for data in iter(lambda: f.read(4 * 1024 * 1024), b""): h.update(data) return h.hexdigest() def validate_original_definition(current, original): """Compare configuration, never read the old experiment's learned tensors.""" fields = {k for k in original["train"] if "loss" in k} | { "learning_rate", "control_head_lr_multiplier", "cosine_min_lr_rate", "max_grad_norm", "lr_scheduler_kwargs", "full_duplex_semantics_version", "qwen_kv_gradient_horizon_windows"} if (current["model"] != original["model"] or current["seed"] != original["seed"] or any(current["train"].get(k) != original["train"].get(k) for k in fields) or current["train"].get("resume_from_checkpoint") is not None or current["train"].get("resume_load_full_state") is not False or current["train"].get("resume_checkpoint_step", 0) != 0 or current["model"].get("pretrained_action_header_path") or current["model"].get("vlm_ckpt_step")): raise ValueError("Fresh scale initialization/model/loss must match the original fresh pilot") def configure(args): from scripts.data.prepare_third_party_scale import validate from scripts.train_third_party import contained_output, build_config from scripts.third_party_joint_init import _contract args.run_dir = contained_output(args.run_dir) args.manifest = args.manifest.resolve() selection, validation, readiness = validate(args.manifest) plan = coverage_plan(selection, epochs=args.epochs) if getattr(args, "parent", None) is not None: raise ValueError("Fresh scale training must not specify any parent weights") cfg = build_config(args.manifest, args.run_dir, "train") cfg.train.train_batch_size = cfg.train.target_effective_sources_per_gpu = 16 cfg.train.num_workers = 8 cfg.train.coverage_epochs = args.epochs cfg.train.max_training_steps = plan["max_optimizer_updates"] cfg.train.warmup_steps = 500 # Original fresh train300, not the later warm-start stage. cfg.train.checkpointing_steps = 20000 cfg.train.max_checkpoints_to_keep = 3 original = read(ORIGINAL_PILOT_CONFIG) validate_original_definition(cfg.model_dump(mode="json"), original) contract = _contract(cfg) identity = {"schema": 2, "initialization_policy": "original_pilot_fresh_v1", "parent": None, "parent_step": None, "qwen_initialization": "pretrained", "hfd_initialization": "fresh", "reference_pilot_config": str(ORIGINAL_PILOT_CONFIG), "reference_pilot_config_sha256": digest(ORIGINAL_PILOT_CONFIG), "train_infer_contract": contract, "initialization_seed": cfg.seed, "manifest_sha256": readiness["manifest_sha256"], "validation_sha256": readiness["validation_sha256"], "code_hashes": {name: digest(PROJECT / name) for name in CODE_FILES}, "initialization": "original pretrained Qwen plus fresh HFD modules; no pilot checkpoint or adapter", "train_count": len(selection["session_ids"]), "validation_count": len(validation["session_ids"]), "optimizer_heldout_inputs": 0, "visual_inputs": 0, "other_task_inputs": 0, "gpu": 0, "base_step_not_added_to_new_stage": True} resume = None if args.resume_checkpoint: checkpoint = args.resume_checkpoint.resolve(); old_run = checkpoint.parent.parent if (not checkpoint.is_relative_to(PROJECT / "runs") or old_run == args.run_dir or old_run.is_relative_to(args.run_dir) or args.run_dir.is_relative_to(old_run) or not (checkpoint / "optimizer.bin").is_file()): raise ValueError("Use a complete scale checkpoint and a fresh output run") old_identity = read(old_run / "scale_identity.json") if old_identity != identity or read(old_run / "coverage_plan.json") != plan: raise ValueError("Scale resume cannot change inputs, implementation, initialization or budget") module_spec = importlib.util.spec_from_file_location("scale_resume_runner", PROJECT / "scripts/train/third_party_run.py") manager = importlib.util.module_from_spec(module_spec) sys.modules[module_spec.name] = manager module_spec.loader.exec_module(manager) process_identity = manager.identity import socket previous = read(old_run / "run_status.json") if previous["host"] != socket.gethostname() and not args.allow_host_change: raise ValueError("Host changed: review GPU allocation and use --allow-host-change") process = process_identity(previous["launcher_pid"]) if process and previous["host"] == socket.gethostname() and process.start_ticks == previous["launcher_start_ticks"]: raise ValueError("Original scale launcher is still alive") cursor = read(checkpoint / "scale_cursor.json") step = int(checkpoint.name.removeprefix("ckpt_")) epoch = plan["epochs"][cursor["epoch"] - 1] expected = sum(e["optimizer_updates"] for e in plan["epochs"][:cursor["epoch"] - 1]) + sum(epoch["group_steps"][:cursor["next_group"]]) if cursor["step"] != step or step != expected or not 0 < cursor["next_group"] <= len(epoch["source_indices"]): raise ValueError("Scale checkpoint coverage cursor mismatch") marker = read(checkpoint / "full_duplex_source_boundary.json") if marker.get("global_step") != step or not marker.get("source_group_complete"): raise ValueError("Scale resume is not at a complete source boundary") resume = {"checkpoint": str(checkpoint), "old_run": str(old_run), "cursor": cursor} return cfg, selection, validation, plan, identity, resume def publish(run): from scripts.third_party_dashboard import render_curves state = read(run / "workflow_state.json") identity, plan = read(run / "scale_identity.json"), read(run / "coverage_plan.json") evaluations = read(run / "evaluations.json") if (run / "evaluations.json").exists() else [] if (run / "metrics.jsonl").exists(): points = [json.loads(l) for l in (run / "metrics.jsonl").read_text().splitlines() if l.strip()] (run / "curves.svg").write_text(render_curves(points)) lines = ["# Third-party 全量训练与验证", "", f"GPU 0;训练 {identity['train_count']} 条/独立验证 {identity['validation_count']} 条;只输入语音,不混入其他任务。", f"当前阶段:`{state['phase']}`;新阶段 step {state.get('step',0)}/{plan['max_optimizer_updates']};最多 {plan['max_epochs']} 遍。", "与最初300条实验相同:原版Qwen预训练+新初始化HFD模块,不加载890620或控制头adapter。新优化器/学习率,step从0计;预算完成不代表收敛。", "", "![训练 loss 曲线](curves.svg)", "", "[实时状态](workflow_state.json) · [原始 loss](metrics.jsonl) · [覆盖游标](coverage_state.json) · [GPU/占卡状态](run_status.json)", "", "中途固定训练/验证面板按类型与语言抽取,不能把均衡面板均值当作整个验证集分布。最终另测全部可用验证样本。", "", "| 新阶段 step | 范围 | 条数 | 控制准确率 | 背景误动率 | 漏动作率 | 关节 MAE | 报告 |", "|---:|---|---:|---:|---:|---:|---:|---|"] for item in evaluations: for split in ("train", "validation"): if split not in item: continue r = item[split]; a = r["aggregate"] label = "训练" if split == "train" else "独立验证" if r.get("panel") == "all_eligible": label += "全量" def fmt(key): return "不适用" if a.get(key) is None else f"{a[key]:.2%}" link = os.path.relpath(r["report_path"], run) lines.append(f"| {item['step']} | {label} | {r['count']} | {fmt('control_accuracy')} | " f"{fmt('background_only_false_motion_rate')} | {fmt('missed_motion_rate')} | {a['joint_mae_radians']:.5f} | [逐条与分组]({link}) |") lines += ["", "训练结束后自动从保存的预测生成审核视频,不把 GT 当作模型效果。", "[最终视频索引](prediction_videos/README.md) · [最终结果](result.json)", ""] (run / "review.md").write_text("\n".join(lines)) def verify_fresh_initialization(trainer, identity): """Read-only audit of the original initializer; never install old weights.""" from safetensors import safe_open import torch from scripts.third_party_joint_init import _trainability if identity.get("initialization_policy") != "original_pilot_fresh_v1" or identity.get("parent") is not None: raise ValueError("Expected fresh initialization without a learned parent") if trainer.global_step != 0 or trainer.cfg.train.resume_from_checkpoint is not None: raise ValueError("Fresh initialization must precede every optimizer update/load") model = trainer.model parameters, qwen, expected = _trainability(model, trainer.cfg) root = Path(trainer.cfg.model.model_name_or_path) weights = read(root / "model.safetensors.index.json")["weight_map"] state = model.state_dict() for name in sorted(qwen): key = name.removeprefix("vlm_model.") if key not in weights: key = "thinker." + key with safe_open(str(root / weights[key]), framework="pt", device="cpu") as tensors: original = tensors.get_tensor(key).to(dtype=state[name].dtype) if not torch.equal(state[name].detach().cpu(), original): raise ValueError(f"Qwen trainable tail is not the original pretrained tensor: {name}") if trainer._hfdv2_checkpoint_contract() != identity["train_infer_contract"]: raise ValueError("Fresh runtime checkpoint contract differs") return {"initialization_policy": identity["initialization_policy"], "loaded_hfd_checkpoints": [], "loaded_control_adapters": [], "qwen_tail_verified_against_pretrained": len(qwen), "compact_tensor_count": len(expected), "trainable_parameters": sum(p.numel() for p in parameters.values() if p.requires_grad), "new_step": 0, "optimizer_reset": True, "scheduler_reset": True, "control_head_replaced": False} def validation_regressed(candidate, reference): # An untrained all-idle predictor is not a trained quality target. Start # this safety comparison at the first actual trained fixed-panel result. if reference is None: return False return (candidate["control_accuracy"] < reference["control_accuracy"] - .03 or candidate["missed_motion_rate"] > reference["missed_motion_rate"] + .03 or candidate["joint_mae_radians"] > reference["joint_mae_radians"] * 1.20) def worker(args, prepared): import torch from accelerate.utils import set_seed from torch.utils.data import DataLoader from scripts.train import _initialize_accelerator from scripts.train_third_party_strategy import source_batches from scripts.third_party_convergence_plan import EpochLossAccumulator from scripts.infer.third_party_scale_metrics import evaluate from psi.trainers.posttrain_third_party import PosttrainThirdPartyTrainer, append_record from psi.trainers.posttrain_full_duplex import _source_collator from psi.utils import flatten, seed_everything cfg, selection, validation, plan, identity, resume = prepared os.environ["HFD_CONTINUOUS_DATA_ROOT"] = cfg.data.continuous_audio_root os.environ["HFD_BACKGROUND_ASSIGNMENT_PATH"] = cfg.data.background_assignment_path os.environ["HFD_CONTINUOUS_DATA_CONTRACT_PATH"] = str(Path(cfg.data.continuous_audio_root) / "reports/production_complete.json") run = args.run_dir if (run / "scale_identity.json").exists(): raise ValueError("Refusing to overwrite an existing scale stage") run.mkdir(parents=True, exist_ok=True) for name, value in (("scale_identity.json", identity), ("coverage_plan.json", plan), ("input_identity.json", identity), ("evaluations.json", [])): write_json(run / name, value) (run / "run_config.json").write_text(cfg.model_dump_json(indent=2)) evaluations = [] step = 0 def phase(name, **fields): write_json(run / "workflow_state.json", {"phase": name, "step": step, "time": time.time(), **fields}) def evaluated(source, checkpoint_step, label, stage_step, final=False): record = {"step": stage_step, "source_step": checkpoint_step, "label": label} for split, manifest in (("validation", args.manifest.parent / "validation.json"), ("train", args.manifest)): phase(f"{label}_{split}_metrics") output = run / "evaluations" / label / split result = evaluate(source, checkpoint_step, manifest, output, full=final and split == "validation", include_videos=final, workers=4) record[split] = {"aggregate": result["aggregate"], "count": result["count"], "report_path": str(output / "report.json"), "panel": result["panel"], "fixed_panel_comparison": result["fixed_panel_comparison"]} write_json(run / "current_evaluation.json", record) evaluations.append(record); write_json(run / "evaluations.json", evaluations) publish(run) return record phase("initializing_full_training"); publish(run) if resume: old = Path(resume["old_run"]) step = resume["cursor"]["step"] evaluations = [r for r in read(old / "evaluations.json") if r["step"] <= step] write_json(run / "evaluations.json", evaluations) write_json(run / "resume_identity.json", resume) # Loss records are references, not regenerated observations. (run / "previous_run.txt").write_text(str(old) + "\n") class ScaleTrainer(PosttrainThirdPartyTrainer): @property def num_steps_per_epoch(self): return plan["epochs"][0]["optimizer_updates"] def create_datasets(self): self.train_dataset = self.data_cfg(split="train", transform_kwargs={"vlm_processor": self.vlm_processor}) if len(self.train_dataset) != len(selection["sessions"]): raise ValueError("Full frozen coverage count changed") self.val_dataset = None return self.train_dataset, None def _checkpoint_probe(self, global_step): return None def log(self, metrics, start_time=None): self._dashboard_step = self.global_step # publish full-scale review separately super().log(metrics, start_time) seed_everything(cfg.seed); torch.cuda.set_device(0) trainer = ScaleTrainer(cfg, 0); trainer.init_models(); trainer.global_step = 0 if not resume: write_json(run / "initialization_verified.json", verify_fresh_initialization(trainer, identity)) accelerator = _initialize_accelerator(trainer) trainer.create_datasets() trainer.train_dataloader = DataLoader(trainer.train_dataset, batch_size=1, shuffle=False, num_workers=0, collate_fn=_source_collator) trainer.val_dataloader = None trainer.create_optimizer_and_scheduler(plan["max_optimizer_updates"]) trainer.prepare(accelerator) set_seed(cfg.seed, device_specific=True) if resume: accelerator.load_state(resume["checkpoint"]) if trainer.lr_scheduler.state_dict()["last_epoch"] != step or not trainer.optimizer.state_dict()["state"]: raise ValueError("Full-scale optimizer/schedule restore failed") write_json(run / "full_state_resume_verified.json", {"step": step, "passed": True}) # Publish a local complete checkpoint even if resuming at the last # boundary solely to finish interrupted final evaluations. trainer.global_step = step trainer._at_source_group_boundary = True trainer._last_source_group_complete = True trainer._last_boundary_reason = "restored_complete_source_boundary" restored = Path(trainer.save_checkpoint(step)) write_json(restored / "scale_cursor.json", resume["cursor"]) elif trainer.optimizer.state_dict()["state"]: raise ValueError("New scale stage inherited optimizer moments") else: if trainer.lr_scheduler.state_dict()["last_epoch"] != 0: raise ValueError("Fresh learning-rate schedule is not at step zero") # No source/KV has been opened: this is a genuine cache-free boundary. trainer._at_source_group_boundary = True trainer._last_source_group_complete = True trainer._last_boundary_reason = "fresh_before_first_source" checkpoint = Path(trainer.save_checkpoint(0)) write_json(run / "initial_checkpoint.json", {"step": 0, "checkpoint": str(checkpoint), "weights_sha256": digest(checkpoint / "model.safetensors"), "optimizer_state_entries": 0, "training_updates": 0, "not_a_pilot_checkpoint": True}) torch.cuda.empty_cache() evaluated(run, 0, "baseline", 0) trainer.global_step = step (run / "run_config.json").write_text(cfg.model_dump_json(indent=2)) append_record(run / "events.jsonl", {"event": "scale_training_initialized", "step": step, "train_count": len(selection["sessions"]), "batch": 16, "time": time.time(), "full_state_resume": bool(resume)}) last_saved, reload_passed = step, bool(resume) best_bg = float("inf"); regressions = 0 reference = None for record in evaluations: if record["step"] <= 0: continue candidate = record["validation"]["fixed_panel_comparison"]["aggregate"] bad = validation_regressed(candidate, reference) regressions = regressions + 1 if bad else 0 if reference is None: reference = candidate if not bad and candidate["background_only_false_motion_rate"] is not None: best_bg = min(best_bg, candidate["background_only_false_motion_rate"]) decision = {"action": "stop_budget", "converged": False, "reason": "bounded complete-coverage stage"} try: for epoch_plan in plan["epochs"]: epoch = epoch_plan["epoch"] if resume and epoch < resume["cursor"]["epoch"]: continue offset = resume["cursor"]["next_group"] if resume and epoch == resume["cursor"]["epoch"] else 0 groups = epoch_plan["source_indices"] acc = EpochLossAccumulator() if resume and epoch == resume["cursor"]["epoch"]: c = resume["cursor"]["accumulator"] for attr, key in (("action_sum", "action_loss_sum"), ("action_samples", "action_samples"), ("control_sum", "control_ce_sum"), ("control_samples", "control_samples"), ("optimizer_updates", "optimizer_updates"), ("control_updates", "control_updates")): setattr(acc, attr, c[key]) completed = sum(len(g) for g in groups[:offset]) interval = max(1, math.ceil(len(groups) / 10)) trainer.set_train() for group_index, (indices, batch) in enumerate(zip(groups[offset:], source_batches(trainer, groups[offset:], workers=8)), offset): phase("training", epoch=epoch, completed_sources=completed, train_count=len(selection["sessions"])) cfg.train.train_batch_size = cfg.train.target_effective_sources_per_gpu = len(indices) before, boundary = step, False for _, values, boundary in trainer.iter_optimizer_steps(batch, global_step_start=step, local_step=group_index, max_optimizer_events=epoch_plan["group_steps"][group_index]): step += 1; trainer.global_step = step acc.add(action_loss=values["loss_action"], action_samples=values["effective_sources_per_gpu"] * values["dense_windows_per_optimizer_step"], control_ce=values.get("loss_control"), control_samples=values["control_update_samples"]) trainer.log(flatten({**values, "epoch": epoch, "completed_sources": completed}, parent_key="train")) if step % 1000 == 0: print(f"[scale] epoch={epoch}/{args.epochs} step={step}/{plan['max_optimizer_updates']} sources={completed}/{len(selection['sessions'])} action={values['loss_action']:.5f}", flush=True) phase("training", epoch=epoch, completed_sources=completed, train_count=len(selection["sessions"])) if step % 50000 == 0: publish(run) if torch.cuda.max_memory_reserved() / 2**30 > 120: raise RuntimeError("GPU reserved memory exceeded 120 GiB safety ceiling") if not boundary or step - before != epoch_plan["group_steps"][group_index]: raise RuntimeError("Incomplete scale source group") completed += len(indices) cfg.train.train_batch_size = cfg.train.target_effective_sources_per_gpu = 16 del batch cursor = {"epoch": epoch, "step": step, "next_group": group_index + 1, "completed_sources": completed, "accumulator": acc.summary()} write_json(run / "coverage_state.json", cursor) last_group = group_index + 1 == len(groups) needs_eval = (group_index + 1) % interval == 0 or last_group if not reload_passed or step - last_saved >= 20000 or needs_eval: phase("saving_checkpoint", epoch=epoch, completed_sources=completed) checkpoint = Path(trainer.save_checkpoint(step)); last_saved = step write_json(checkpoint / "scale_cursor.json", cursor) if not reload_passed: accelerator.load_state(str(checkpoint)) if trainer.lr_scheduler.state_dict()["last_epoch"] != step or not trainer.optimizer.state_dict()["state"]: raise RuntimeError("First real full-scale checkpoint did not reload") reload_passed = True write_json(run / "first_checkpoint_verified.json", {"passed": True, "step": step, "checkpoint": str(checkpoint)}) publish(run) if needs_eval: torch.cuda.synchronize(); torch.cuda.empty_cache() final = last_group and epoch == args.epochs record = evaluated(run, step, "final" if final else f"step_{step}", step, final=final) candidate = record["validation"]["fixed_panel_comparison"]["aggregate"] bad = validation_regressed(candidate, reference) if reference is None: reference = candidate regressions = regressions + 1 if bad else 0 bg = candidate["background_only_false_motion_rate"] if not bad and bg is not None and bg < best_bg: dest = run / "best_model"; dest.mkdir(exist_ok=True) for name in ("model.safetensors", "hfd_fused_delta_manifest.json"): temp = dest / (name + ".tmp"); shutil.copy2(checkpoint / name, temp); temp.replace(dest / name) write_json(dest / "selection.json", {"step": step, "validation_fixed_panel": candidate, "policy": "lowest BG error passing first-trained-panel accuracy/miss/joint protection; weights only"}) best_bg = bg if regressions >= 2: decision = {"action": "stop_regression", "converged": False, "reason": "two fixed validation checks materially regressed"} break if completed == len(selection["sessions"]): summary = acc.summary() if summary["action_samples"] != epoch_plan["action_samples"] or summary["control_samples"] != epoch_plan["control_samples"]: raise RuntimeError("Complete epoch lost or duplicated supervision") append_record(run / "epoch_metrics.jsonl", {"epoch": epoch, "step": step, **summary}) if decision["action"] == "stop_regression": break if not (run / "evaluations/final/validation/report.json").exists(): evaluated(run, step, "final", step, final=True) write_json(run / "result.json", {"status": "complete", "step": step, "decision": decision, "train_count": len(selection["sessions"]), "validation_count": len(validation["sessions"]), "evaluation_file": str(run / "evaluations.json"), "final_checkpoint": "last, not necessarily best", "original_data_and_parent_weights_modified": False, "video_status": "see prediction_videos/status.json"}) phase("completed", decision=decision) except Exception as error: phase("failed", error=repr(error)); raise finally: trainer.finalize() publish(run) with (run / "video_render.log").open("x") as log: child = subprocess.Popen([sys.executable, str(PROJECT / "scripts/infer/render_third_party_scale.py"), "--run-dir", str(run), "--manifest", str(args.manifest)], cwd=PROJECT, env={**os.environ, "CUDA_VISIBLE_DEVICES": "", "OMP_NUM_THREADS": "1", "MKL_NUM_THREADS": "1"}, stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) write_json(run / "video_render_job.json", {"pid": child.pid, "gpu_used": False, "inference_repeated": False}) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--manifest", type=Path, default=PROJECT / "data/hfd_third_party_multi_v5/training_scale_all_commands/selection.json") parser.add_argument("--run-dir", type=Path, required=True) parser.add_argument("--epochs", type=int, default=3) parser.add_argument("--resume-checkpoint", type=Path) parser.add_argument("--allow-host-change", action="store_true") mode = parser.add_mutually_exclusive_group(required=True) for name in ("check-only", "launch", "worker"): mode.add_argument(f"--{name}", action="store_true") args = parser.parse_args(); os.chdir(PROJECT) for key in ("RANK", "LOCAL_RANK", "WORLD_SIZE", "LOCAL_WORLD_SIZE", "MASTER_ADDR", "MASTER_PORT", "ACCELERATE_USE_DEEPSPEED", "ACCELERATE_USE_FSDP", "ACCELERATE_USE_CPU", "SLURM_PROCID", "SLURM_LOCALID", "SLURM_NTASKS"): os.environ.pop(key, None) os.environ.update(HFD_SKIP_DOTENV="1", HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1", PSI_SKIP_VLM_MODEL="0", PSI_FLOW_SIGMA_MIN="0.05", PSI_FLOW_SIGMA_MAX="1.0", PSI_WORKER_TORCH_THREADS="1", PSI_DATALOADER_PIN_MEMORY="1", PSI_FOREACH_GRAD_NORM="1", OMP_NUM_THREADS="4", PYTHONDONTWRITEBYTECODE="1") if args.worker: if os.environ.get("CUDA_VISIBLE_DEVICES") != "0" or os.environ.get("HFD_NPROC_PER_NODE") != "1": raise RuntimeError("Use --launch for GPU0 management") from scripts.third_party_temp import configure_project_temp configure_project_temp(PROJECT) for key, name in (("TORCHINDUCTOR_CACHE_DIR", "inductor"), ("TRITON_CACHE_DIR", "triton"), ("CUDA_CACHE_PATH", "cuda")): path = PROJECT / "cache/third_party_scale" / name; path.mkdir(parents=True, exist_ok=True); os.environ[key] = str(path) prepared = configure(args) if args.check_only: print(json.dumps({"passed": True, "gpu": 0, "train_count": len(prepared[1]["sessions"]), "validation_count": len(prepared[2]["sessions"]), "max_steps": prepared[3]["max_optimizer_updates"], "epochs": args.epochs, "initialization": prepared[4]["initialization_policy"], "parent": prepared[4]["parent"], "full_state_resume": bool(prepared[5])})); return if args.worker: try: worker(args, prepared) except Exception as error: state_path = args.run_dir / "workflow_state.json" if state_path.exists(): state = read(state_path) write_json(state_path, {**state, "phase": "failed", "error": repr(error), "time": time.time()}) publish(args.run_dir) raise return argv = [sys.executable, str(Path(__file__).resolve()), "--manifest", str(args.manifest), "--run-dir", str(args.run_dir), "--epochs", str(args.epochs)] if args.resume_checkpoint: argv += ["--resume-checkpoint", str(args.resume_checkpoint.resolve())] if args.allow_host_change: argv += ["--allow-host-change"] spec = {"schema": 1, "gpu": 0, "scope": "full_third_party", "project_root": str(PROJECT), "run_dir": str(args.run_dir), "manifest_path": str(args.manifest), "readiness_file": str(args.manifest.parent / "acceptance.json"), "argv": [*argv, "--worker"], "preflight_command": [*argv, "--check-only"]} path = args.manifest.parent / f"{args.run_dir.name}.launch.json" if path.exists(): raise ValueError("Use a fresh launch specification") write_json(path, spec) raise SystemExit(subprocess.call([sys.executable, str(PROJECT / "scripts/train/third_party_run.py"), "start", "--spec", str(path)])) if __name__ == "__main__": main()