code-HFD_lzk / scripts /train_third_party_scale.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
55e0599 verified
Raw History Blame Contribute Delete
30.8 kB
#!/usr/bin/env python3
"""Managed GPU0 full eligible-data stage, with held-out metrics and final videos."""
from __future__ import annotations
import argparse
import hashlib
import importlib.util
import json
import math
import os
from pathlib import Path
import shutil
import subprocess
import sys
import time
PROJECT = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(PROJECT), str(PROJECT / "src")]
from scripts.third_party_strategy_plan import write_json
from scripts.third_party_scale_plan import coverage_plan
ORIGINAL_PILOT_CONFIG = PROJECT / "runs/third_party_train_20261004_183932/run_config.json"
CODE_FILES = ("scripts/train_third_party_scale.py", "scripts/third_party_scale_plan.py",
"scripts/train_third_party.py", "scripts/train.py", "scripts/third_party_joint_init.py",
"scripts/data/prepare_third_party_scale.py", "scripts/infer/third_party_scale_metrics.py",
"scripts/infer/render_third_party_scale.py", "scripts/train/third_party_run.py",
"src/psi/trainers/posttrain_full_duplex.py", "src/psi/trainers/posttrain_v1.py",
"src/psi/data/seed/qwen_streaming_audio.py",
"src/psi/data/seed/third_party_frozen_dataset.py")
def read(path):
return json.loads(Path(path).read_text())
def digest(path):
h = hashlib.sha256()
with Path(path).open("rb") as f:
for data in iter(lambda: f.read(4 * 1024 * 1024), b""):
h.update(data)
return h.hexdigest()
def validate_original_definition(current, original):
"""Compare configuration, never read the old experiment's learned tensors."""
fields = {k for k in original["train"] if "loss" in k} | {
"learning_rate", "control_head_lr_multiplier", "cosine_min_lr_rate", "max_grad_norm",
"lr_scheduler_kwargs", "full_duplex_semantics_version", "qwen_kv_gradient_horizon_windows"}
if (current["model"] != original["model"] or current["seed"] != original["seed"]
or any(current["train"].get(k) != original["train"].get(k) for k in fields)
or current["train"].get("resume_from_checkpoint") is not None
or current["train"].get("resume_load_full_state") is not False
or current["train"].get("resume_checkpoint_step", 0) != 0
or current["model"].get("pretrained_action_header_path")
or current["model"].get("vlm_ckpt_step")):
raise ValueError("Fresh scale initialization/model/loss must match the original fresh pilot")
def configure(args):
from scripts.data.prepare_third_party_scale import validate
from scripts.train_third_party import contained_output, build_config
from scripts.third_party_joint_init import _contract
args.run_dir = contained_output(args.run_dir)
args.manifest = args.manifest.resolve()
selection, validation, readiness = validate(args.manifest)
plan = coverage_plan(selection, epochs=args.epochs)
if getattr(args, "parent", None) is not None:
raise ValueError("Fresh scale training must not specify any parent weights")
cfg = build_config(args.manifest, args.run_dir, "train")
cfg.train.train_batch_size = cfg.train.target_effective_sources_per_gpu = 16
cfg.train.num_workers = 8
cfg.train.coverage_epochs = args.epochs
cfg.train.max_training_steps = plan["max_optimizer_updates"]
cfg.train.warmup_steps = 500 # Original fresh train300, not the later warm-start stage.
cfg.train.checkpointing_steps = 20000
cfg.train.max_checkpoints_to_keep = 3
original = read(ORIGINAL_PILOT_CONFIG)
validate_original_definition(cfg.model_dump(mode="json"), original)
contract = _contract(cfg)
identity = {"schema": 2, "initialization_policy": "original_pilot_fresh_v1",
"parent": None, "parent_step": None, "qwen_initialization": "pretrained", "hfd_initialization": "fresh",
"reference_pilot_config": str(ORIGINAL_PILOT_CONFIG), "reference_pilot_config_sha256": digest(ORIGINAL_PILOT_CONFIG),
"train_infer_contract": contract, "initialization_seed": cfg.seed,
"manifest_sha256": readiness["manifest_sha256"], "validation_sha256": readiness["validation_sha256"],
"code_hashes": {name: digest(PROJECT / name) for name in CODE_FILES},
"initialization": "original pretrained Qwen plus fresh HFD modules; no pilot checkpoint or adapter",
"train_count": len(selection["session_ids"]), "validation_count": len(validation["session_ids"]),
"optimizer_heldout_inputs": 0, "visual_inputs": 0, "other_task_inputs": 0,
"gpu": 0, "base_step_not_added_to_new_stage": True}
resume = None
if args.resume_checkpoint:
checkpoint = args.resume_checkpoint.resolve(); old_run = checkpoint.parent.parent
if (not checkpoint.is_relative_to(PROJECT / "runs") or old_run == args.run_dir
or old_run.is_relative_to(args.run_dir) or args.run_dir.is_relative_to(old_run)
or not (checkpoint / "optimizer.bin").is_file()):
raise ValueError("Use a complete scale checkpoint and a fresh output run")
old_identity = read(old_run / "scale_identity.json")
if old_identity != identity or read(old_run / "coverage_plan.json") != plan:
raise ValueError("Scale resume cannot change inputs, implementation, initialization or budget")
module_spec = importlib.util.spec_from_file_location("scale_resume_runner", PROJECT / "scripts/train/third_party_run.py")
manager = importlib.util.module_from_spec(module_spec)
sys.modules[module_spec.name] = manager
module_spec.loader.exec_module(manager)
process_identity = manager.identity
import socket
previous = read(old_run / "run_status.json")
if previous["host"] != socket.gethostname() and not args.allow_host_change:
raise ValueError("Host changed: review GPU allocation and use --allow-host-change")
process = process_identity(previous["launcher_pid"])
if process and previous["host"] == socket.gethostname() and process.start_ticks == previous["launcher_start_ticks"]:
raise ValueError("Original scale launcher is still alive")
cursor = read(checkpoint / "scale_cursor.json")
step = int(checkpoint.name.removeprefix("ckpt_"))
epoch = plan["epochs"][cursor["epoch"] - 1]
expected = sum(e["optimizer_updates"] for e in plan["epochs"][:cursor["epoch"] - 1]) + sum(epoch["group_steps"][:cursor["next_group"]])
if cursor["step"] != step or step != expected or not 0 < cursor["next_group"] <= len(epoch["source_indices"]):
raise ValueError("Scale checkpoint coverage cursor mismatch")
marker = read(checkpoint / "full_duplex_source_boundary.json")
if marker.get("global_step") != step or not marker.get("source_group_complete"):
raise ValueError("Scale resume is not at a complete source boundary")
resume = {"checkpoint": str(checkpoint), "old_run": str(old_run), "cursor": cursor}
return cfg, selection, validation, plan, identity, resume
def publish(run):
from scripts.third_party_dashboard import render_curves
state = read(run / "workflow_state.json")
identity, plan = read(run / "scale_identity.json"), read(run / "coverage_plan.json")
evaluations = read(run / "evaluations.json") if (run / "evaluations.json").exists() else []
if (run / "metrics.jsonl").exists():
points = [json.loads(l) for l in (run / "metrics.jsonl").read_text().splitlines() if l.strip()]
(run / "curves.svg").write_text(render_curves(points))
lines = ["# Third-party 全量训练与验证", "",
f"GPU 0;训练 {identity['train_count']} 条/独立验证 {identity['validation_count']} 条;只输入语音,不混入其他任务。",
f"当前阶段:`{state['phase']}`;新阶段 step {state.get('step',0)}/{plan['max_optimizer_updates']};最多 {plan['max_epochs']} 遍。",
"与最初300条实验相同:原版Qwen预训练+新初始化HFD模块,不加载890620或控制头adapter。新优化器/学习率,step从0计;预算完成不代表收敛。", "",
"![训练 loss 曲线](curves.svg)", "",
"[实时状态](workflow_state.json) · [原始 loss](metrics.jsonl) · [覆盖游标](coverage_state.json) · [GPU/占卡状态](run_status.json)", "",
"中途固定训练/验证面板按类型与语言抽取,不能把均衡面板均值当作整个验证集分布。最终另测全部可用验证样本。", "",
"| 新阶段 step | 范围 | 条数 | 控制准确率 | 背景误动率 | 漏动作率 | 关节 MAE | 报告 |",
"|---:|---|---:|---:|---:|---:|---:|---|"]
for item in evaluations:
for split in ("train", "validation"):
if split not in item:
continue
r = item[split]; a = r["aggregate"]
label = "训练" if split == "train" else "独立验证"
if r.get("panel") == "all_eligible": label += "全量"
def fmt(key):
return "不适用" if a.get(key) is None else f"{a[key]:.2%}"
link = os.path.relpath(r["report_path"], run)
lines.append(f"| {item['step']} | {label} | {r['count']} | {fmt('control_accuracy')} | "
f"{fmt('background_only_false_motion_rate')} | {fmt('missed_motion_rate')} | {a['joint_mae_radians']:.5f} | [逐条与分组]({link}) |")
lines += ["", "训练结束后自动从保存的预测生成审核视频,不把 GT 当作模型效果。",
"[最终视频索引](prediction_videos/README.md) · [最终结果](result.json)", ""]
(run / "review.md").write_text("\n".join(lines))
def verify_fresh_initialization(trainer, identity):
"""Read-only audit of the original initializer; never install old weights."""
from safetensors import safe_open
import torch
from scripts.third_party_joint_init import _trainability
if identity.get("initialization_policy") != "original_pilot_fresh_v1" or identity.get("parent") is not None:
raise ValueError("Expected fresh initialization without a learned parent")
if trainer.global_step != 0 or trainer.cfg.train.resume_from_checkpoint is not None:
raise ValueError("Fresh initialization must precede every optimizer update/load")
model = trainer.model
parameters, qwen, expected = _trainability(model, trainer.cfg)
root = Path(trainer.cfg.model.model_name_or_path)
weights = read(root / "model.safetensors.index.json")["weight_map"]
state = model.state_dict()
for name in sorted(qwen):
key = name.removeprefix("vlm_model.")
if key not in weights:
key = "thinker." + key
with safe_open(str(root / weights[key]), framework="pt", device="cpu") as tensors:
original = tensors.get_tensor(key).to(dtype=state[name].dtype)
if not torch.equal(state[name].detach().cpu(), original):
raise ValueError(f"Qwen trainable tail is not the original pretrained tensor: {name}")
if trainer._hfdv2_checkpoint_contract() != identity["train_infer_contract"]:
raise ValueError("Fresh runtime checkpoint contract differs")
return {"initialization_policy": identity["initialization_policy"], "loaded_hfd_checkpoints": [],
"loaded_control_adapters": [], "qwen_tail_verified_against_pretrained": len(qwen),
"compact_tensor_count": len(expected),
"trainable_parameters": sum(p.numel() for p in parameters.values() if p.requires_grad),
"new_step": 0, "optimizer_reset": True, "scheduler_reset": True, "control_head_replaced": False}
def validation_regressed(candidate, reference):
# An untrained all-idle predictor is not a trained quality target. Start
# this safety comparison at the first actual trained fixed-panel result.
if reference is None:
return False
return (candidate["control_accuracy"] < reference["control_accuracy"] - .03
or candidate["missed_motion_rate"] > reference["missed_motion_rate"] + .03
or candidate["joint_mae_radians"] > reference["joint_mae_radians"] * 1.20)
def worker(args, prepared):
import torch
from accelerate.utils import set_seed
from torch.utils.data import DataLoader
from scripts.train import _initialize_accelerator
from scripts.train_third_party_strategy import source_batches
from scripts.third_party_convergence_plan import EpochLossAccumulator
from scripts.infer.third_party_scale_metrics import evaluate
from psi.trainers.posttrain_third_party import PosttrainThirdPartyTrainer, append_record
from psi.trainers.posttrain_full_duplex import _source_collator
from psi.utils import flatten, seed_everything
cfg, selection, validation, plan, identity, resume = prepared
os.environ["HFD_CONTINUOUS_DATA_ROOT"] = cfg.data.continuous_audio_root
os.environ["HFD_BACKGROUND_ASSIGNMENT_PATH"] = cfg.data.background_assignment_path
os.environ["HFD_CONTINUOUS_DATA_CONTRACT_PATH"] = str(Path(cfg.data.continuous_audio_root) / "reports/production_complete.json")
run = args.run_dir
if (run / "scale_identity.json").exists():
raise ValueError("Refusing to overwrite an existing scale stage")
run.mkdir(parents=True, exist_ok=True)
for name, value in (("scale_identity.json", identity), ("coverage_plan.json", plan),
("input_identity.json", identity), ("evaluations.json", [])):
write_json(run / name, value)
(run / "run_config.json").write_text(cfg.model_dump_json(indent=2))
evaluations = []
step = 0
def phase(name, **fields):
write_json(run / "workflow_state.json", {"phase": name, "step": step, "time": time.time(), **fields})
def evaluated(source, checkpoint_step, label, stage_step, final=False):
record = {"step": stage_step, "source_step": checkpoint_step, "label": label}
for split, manifest in (("validation", args.manifest.parent / "validation.json"), ("train", args.manifest)):
phase(f"{label}_{split}_metrics")
output = run / "evaluations" / label / split
result = evaluate(source, checkpoint_step, manifest, output,
full=final and split == "validation", include_videos=final, workers=4)
record[split] = {"aggregate": result["aggregate"], "count": result["count"],
"report_path": str(output / "report.json"), "panel": result["panel"],
"fixed_panel_comparison": result["fixed_panel_comparison"]}
write_json(run / "current_evaluation.json", record)
evaluations.append(record); write_json(run / "evaluations.json", evaluations)
publish(run)
return record
phase("initializing_full_training"); publish(run)
if resume:
old = Path(resume["old_run"])
step = resume["cursor"]["step"]
evaluations = [r for r in read(old / "evaluations.json") if r["step"] <= step]
write_json(run / "evaluations.json", evaluations)
write_json(run / "resume_identity.json", resume)
# Loss records are references, not regenerated observations.
(run / "previous_run.txt").write_text(str(old) + "\n")
class ScaleTrainer(PosttrainThirdPartyTrainer):
@property
def num_steps_per_epoch(self):
return plan["epochs"][0]["optimizer_updates"]
def create_datasets(self):
self.train_dataset = self.data_cfg(split="train", transform_kwargs={"vlm_processor": self.vlm_processor})
if len(self.train_dataset) != len(selection["sessions"]):
raise ValueError("Full frozen coverage count changed")
self.val_dataset = None
return self.train_dataset, None
def _checkpoint_probe(self, global_step):
return None
def log(self, metrics, start_time=None):
self._dashboard_step = self.global_step # publish full-scale review separately
super().log(metrics, start_time)
seed_everything(cfg.seed); torch.cuda.set_device(0)
trainer = ScaleTrainer(cfg, 0); trainer.init_models(); trainer.global_step = 0
if not resume:
write_json(run / "initialization_verified.json", verify_fresh_initialization(trainer, identity))
accelerator = _initialize_accelerator(trainer)
trainer.create_datasets()
trainer.train_dataloader = DataLoader(trainer.train_dataset, batch_size=1, shuffle=False,
num_workers=0, collate_fn=_source_collator)
trainer.val_dataloader = None
trainer.create_optimizer_and_scheduler(plan["max_optimizer_updates"])
trainer.prepare(accelerator)
set_seed(cfg.seed, device_specific=True)
if resume:
accelerator.load_state(resume["checkpoint"])
if trainer.lr_scheduler.state_dict()["last_epoch"] != step or not trainer.optimizer.state_dict()["state"]:
raise ValueError("Full-scale optimizer/schedule restore failed")
write_json(run / "full_state_resume_verified.json", {"step": step, "passed": True})
# Publish a local complete checkpoint even if resuming at the last
# boundary solely to finish interrupted final evaluations.
trainer.global_step = step
trainer._at_source_group_boundary = True
trainer._last_source_group_complete = True
trainer._last_boundary_reason = "restored_complete_source_boundary"
restored = Path(trainer.save_checkpoint(step))
write_json(restored / "scale_cursor.json", resume["cursor"])
elif trainer.optimizer.state_dict()["state"]:
raise ValueError("New scale stage inherited optimizer moments")
else:
if trainer.lr_scheduler.state_dict()["last_epoch"] != 0:
raise ValueError("Fresh learning-rate schedule is not at step zero")
# No source/KV has been opened: this is a genuine cache-free boundary.
trainer._at_source_group_boundary = True
trainer._last_source_group_complete = True
trainer._last_boundary_reason = "fresh_before_first_source"
checkpoint = Path(trainer.save_checkpoint(0))
write_json(run / "initial_checkpoint.json", {"step": 0, "checkpoint": str(checkpoint),
"weights_sha256": digest(checkpoint / "model.safetensors"),
"optimizer_state_entries": 0, "training_updates": 0, "not_a_pilot_checkpoint": True})
torch.cuda.empty_cache()
evaluated(run, 0, "baseline", 0)
trainer.global_step = step
(run / "run_config.json").write_text(cfg.model_dump_json(indent=2))
append_record(run / "events.jsonl", {"event": "scale_training_initialized", "step": step,
"train_count": len(selection["sessions"]), "batch": 16, "time": time.time(), "full_state_resume": bool(resume)})
last_saved, reload_passed = step, bool(resume)
best_bg = float("inf"); regressions = 0
reference = None
for record in evaluations:
if record["step"] <= 0:
continue
candidate = record["validation"]["fixed_panel_comparison"]["aggregate"]
bad = validation_regressed(candidate, reference)
regressions = regressions + 1 if bad else 0
if reference is None:
reference = candidate
if not bad and candidate["background_only_false_motion_rate"] is not None:
best_bg = min(best_bg, candidate["background_only_false_motion_rate"])
decision = {"action": "stop_budget", "converged": False, "reason": "bounded complete-coverage stage"}
try:
for epoch_plan in plan["epochs"]:
epoch = epoch_plan["epoch"]
if resume and epoch < resume["cursor"]["epoch"]:
continue
offset = resume["cursor"]["next_group"] if resume and epoch == resume["cursor"]["epoch"] else 0
groups = epoch_plan["source_indices"]
acc = EpochLossAccumulator()
if resume and epoch == resume["cursor"]["epoch"]:
c = resume["cursor"]["accumulator"]
for attr, key in (("action_sum", "action_loss_sum"), ("action_samples", "action_samples"),
("control_sum", "control_ce_sum"), ("control_samples", "control_samples"),
("optimizer_updates", "optimizer_updates"), ("control_updates", "control_updates")):
setattr(acc, attr, c[key])
completed = sum(len(g) for g in groups[:offset])
interval = max(1, math.ceil(len(groups) / 10))
trainer.set_train()
for group_index, (indices, batch) in enumerate(zip(groups[offset:], source_batches(trainer, groups[offset:], workers=8)), offset):
phase("training", epoch=epoch, completed_sources=completed, train_count=len(selection["sessions"]))
cfg.train.train_batch_size = cfg.train.target_effective_sources_per_gpu = len(indices)
before, boundary = step, False
for _, values, boundary in trainer.iter_optimizer_steps(batch, global_step_start=step,
local_step=group_index, max_optimizer_events=epoch_plan["group_steps"][group_index]):
step += 1; trainer.global_step = step
acc.add(action_loss=values["loss_action"], action_samples=values["effective_sources_per_gpu"] * values["dense_windows_per_optimizer_step"],
control_ce=values.get("loss_control"), control_samples=values["control_update_samples"])
trainer.log(flatten({**values, "epoch": epoch, "completed_sources": completed}, parent_key="train"))
if step % 1000 == 0:
print(f"[scale] epoch={epoch}/{args.epochs} step={step}/{plan['max_optimizer_updates']} sources={completed}/{len(selection['sessions'])} action={values['loss_action']:.5f}", flush=True)
phase("training", epoch=epoch, completed_sources=completed, train_count=len(selection["sessions"]))
if step % 50000 == 0:
publish(run)
if torch.cuda.max_memory_reserved() / 2**30 > 120:
raise RuntimeError("GPU reserved memory exceeded 120 GiB safety ceiling")
if not boundary or step - before != epoch_plan["group_steps"][group_index]:
raise RuntimeError("Incomplete scale source group")
completed += len(indices)
cfg.train.train_batch_size = cfg.train.target_effective_sources_per_gpu = 16
del batch
cursor = {"epoch": epoch, "step": step, "next_group": group_index + 1,
"completed_sources": completed, "accumulator": acc.summary()}
write_json(run / "coverage_state.json", cursor)
last_group = group_index + 1 == len(groups)
needs_eval = (group_index + 1) % interval == 0 or last_group
if not reload_passed or step - last_saved >= 20000 or needs_eval:
phase("saving_checkpoint", epoch=epoch, completed_sources=completed)
checkpoint = Path(trainer.save_checkpoint(step)); last_saved = step
write_json(checkpoint / "scale_cursor.json", cursor)
if not reload_passed:
accelerator.load_state(str(checkpoint))
if trainer.lr_scheduler.state_dict()["last_epoch"] != step or not trainer.optimizer.state_dict()["state"]:
raise RuntimeError("First real full-scale checkpoint did not reload")
reload_passed = True
write_json(run / "first_checkpoint_verified.json", {"passed": True, "step": step, "checkpoint": str(checkpoint)})
publish(run)
if needs_eval:
torch.cuda.synchronize(); torch.cuda.empty_cache()
final = last_group and epoch == args.epochs
record = evaluated(run, step, "final" if final else f"step_{step}", step, final=final)
candidate = record["validation"]["fixed_panel_comparison"]["aggregate"]
bad = validation_regressed(candidate, reference)
if reference is None:
reference = candidate
regressions = regressions + 1 if bad else 0
bg = candidate["background_only_false_motion_rate"]
if not bad and bg is not None and bg < best_bg:
dest = run / "best_model"; dest.mkdir(exist_ok=True)
for name in ("model.safetensors", "hfd_fused_delta_manifest.json"):
temp = dest / (name + ".tmp"); shutil.copy2(checkpoint / name, temp); temp.replace(dest / name)
write_json(dest / "selection.json", {"step": step, "validation_fixed_panel": candidate,
"policy": "lowest BG error passing first-trained-panel accuracy/miss/joint protection; weights only"})
best_bg = bg
if regressions >= 2:
decision = {"action": "stop_regression", "converged": False, "reason": "two fixed validation checks materially regressed"}
break
if completed == len(selection["sessions"]):
summary = acc.summary()
if summary["action_samples"] != epoch_plan["action_samples"] or summary["control_samples"] != epoch_plan["control_samples"]:
raise RuntimeError("Complete epoch lost or duplicated supervision")
append_record(run / "epoch_metrics.jsonl", {"epoch": epoch, "step": step, **summary})
if decision["action"] == "stop_regression":
break
if not (run / "evaluations/final/validation/report.json").exists():
evaluated(run, step, "final", step, final=True)
write_json(run / "result.json", {"status": "complete", "step": step, "decision": decision,
"train_count": len(selection["sessions"]), "validation_count": len(validation["sessions"]),
"evaluation_file": str(run / "evaluations.json"), "final_checkpoint": "last, not necessarily best",
"original_data_and_parent_weights_modified": False, "video_status": "see prediction_videos/status.json"})
phase("completed", decision=decision)
except Exception as error:
phase("failed", error=repr(error)); raise
finally:
trainer.finalize()
publish(run)
with (run / "video_render.log").open("x") as log:
child = subprocess.Popen([sys.executable, str(PROJECT / "scripts/infer/render_third_party_scale.py"),
"--run-dir", str(run), "--manifest", str(args.manifest)], cwd=PROJECT,
env={**os.environ, "CUDA_VISIBLE_DEVICES": "", "OMP_NUM_THREADS": "1", "MKL_NUM_THREADS": "1"},
stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
write_json(run / "video_render_job.json", {"pid": child.pid, "gpu_used": False, "inference_repeated": False})
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--manifest", type=Path, default=PROJECT / "data/hfd_third_party_multi_v5/training_scale_all_commands/selection.json")
parser.add_argument("--run-dir", type=Path, required=True)
parser.add_argument("--epochs", type=int, default=3)
parser.add_argument("--resume-checkpoint", type=Path)
parser.add_argument("--allow-host-change", action="store_true")
mode = parser.add_mutually_exclusive_group(required=True)
for name in ("check-only", "launch", "worker"):
mode.add_argument(f"--{name}", action="store_true")
args = parser.parse_args(); os.chdir(PROJECT)
for key in ("RANK", "LOCAL_RANK", "WORLD_SIZE", "LOCAL_WORLD_SIZE", "MASTER_ADDR", "MASTER_PORT",
"ACCELERATE_USE_DEEPSPEED", "ACCELERATE_USE_FSDP", "ACCELERATE_USE_CPU", "SLURM_PROCID", "SLURM_LOCALID", "SLURM_NTASKS"):
os.environ.pop(key, None)
os.environ.update(HFD_SKIP_DOTENV="1", HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1", PSI_SKIP_VLM_MODEL="0",
PSI_FLOW_SIGMA_MIN="0.05", PSI_FLOW_SIGMA_MAX="1.0", PSI_WORKER_TORCH_THREADS="1",
PSI_DATALOADER_PIN_MEMORY="1", PSI_FOREACH_GRAD_NORM="1", OMP_NUM_THREADS="4", PYTHONDONTWRITEBYTECODE="1")
if args.worker:
if os.environ.get("CUDA_VISIBLE_DEVICES") != "0" or os.environ.get("HFD_NPROC_PER_NODE") != "1":
raise RuntimeError("Use --launch for GPU0 management")
from scripts.third_party_temp import configure_project_temp
configure_project_temp(PROJECT)
for key, name in (("TORCHINDUCTOR_CACHE_DIR", "inductor"), ("TRITON_CACHE_DIR", "triton"), ("CUDA_CACHE_PATH", "cuda")):
path = PROJECT / "cache/third_party_scale" / name; path.mkdir(parents=True, exist_ok=True); os.environ[key] = str(path)
prepared = configure(args)
if args.check_only:
print(json.dumps({"passed": True, "gpu": 0, "train_count": len(prepared[1]["sessions"]),
"validation_count": len(prepared[2]["sessions"]), "max_steps": prepared[3]["max_optimizer_updates"],
"epochs": args.epochs, "initialization": prepared[4]["initialization_policy"],
"parent": prepared[4]["parent"], "full_state_resume": bool(prepared[5])})); return
if args.worker:
try:
worker(args, prepared)
except Exception as error:
state_path = args.run_dir / "workflow_state.json"
if state_path.exists():
state = read(state_path)
write_json(state_path, {**state, "phase": "failed", "error": repr(error), "time": time.time()})
publish(args.run_dir)
raise
return
argv = [sys.executable, str(Path(__file__).resolve()), "--manifest", str(args.manifest),
"--run-dir", str(args.run_dir), "--epochs", str(args.epochs)]
if args.resume_checkpoint:
argv += ["--resume-checkpoint", str(args.resume_checkpoint.resolve())]
if args.allow_host_change:
argv += ["--allow-host-change"]
spec = {"schema": 1, "gpu": 0, "scope": "full_third_party", "project_root": str(PROJECT),
"run_dir": str(args.run_dir), "manifest_path": str(args.manifest),
"readiness_file": str(args.manifest.parent / "acceptance.json"),
"argv": [*argv, "--worker"], "preflight_command": [*argv, "--check-only"]}
path = args.manifest.parent / f"{args.run_dir.name}.launch.json"
if path.exists():
raise ValueError("Use a fresh launch specification")
write_json(path, spec)
raise SystemExit(subprocess.call([sys.executable, str(PROJECT / "scripts/train/third_party_run.py"), "start", "--spec", str(path)]))
if __name__ == "__main__":
main()