Download scripts/train_third_party_strategy.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 25.4 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/train_third_party_strategy.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/train_third_party_strategy.py
-
curl -L -o train_third_party_strategy.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/train_third_party_strategy.py
25.4 kB
| #!/usr/bin/env python3 | |
| """Measured batch migration, historical train-only metrics, then continuation. | |
| The original model/loss/optimizer implementation is reused unchanged. The new | |
| driver changes source grouping and explicitly maps the remaining LR schedule. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import json | |
| import math | |
| import os | |
| from pathlib import Path | |
| import subprocess | |
| import sys | |
| import time | |
| PROJECT = Path(__file__).resolve().parents[1] | |
| sys.path[:0] = [str(PROJECT), str(PROJECT / "src")] | |
| from scripts.third_party_strategy_plan import ( | |
| evaluation_ids, remaining_plan, mapped_scheduler_step, choose_batch, write_json, | |
| ) | |
| def setup_environment(): | |
| os.environ.update(HFD_SKIP_DOTENV="1", HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1", | |
| PSI_SKIP_VLM_MODEL="0", PSI_FLOW_SIGMA_MIN="0.05", PSI_FLOW_SIGMA_MAX="1.0", | |
| PSI_WORKER_TORCH_THREADS="1", PSI_DATALOADER_PIN_MEMORY="1", | |
| PSI_FOREACH_GRAD_NORM="1", OMP_NUM_THREADS="4", OPENBLAS_NUM_THREADS="1") | |
| os.environ.pop("_ENV_INFO_JSON", None) | |
| for variable, name in (("TORCHINDUCTOR_CACHE_DIR", "inductor"), | |
| ("TRITON_CACHE_DIR", "triton"), ("CUDA_CACHE_PATH", "cuda")): | |
| directory = PROJECT / "cache" / "third_party_training" / name | |
| directory.mkdir(parents=True, exist_ok=True) | |
| os.environ[variable] = str(directory) | |
| from scripts.third_party_temp import configure_project_temp | |
| configure_project_temp(PROJECT) | |
| def prepare(args, output=None): | |
| from scripts.train_third_party import build_config, contained_output, PRODUCTION | |
| from scripts.third_party_resume import prepare_resume, frozen_resume_cursor | |
| from scripts.data.prepare_third_party_training import validate_acceptance | |
| from psi.trainers.posttrain_full_duplex import SourceLengthGroupedSampler | |
| manifest = args.manifest.resolve() | |
| run = contained_output(output or args.run_dir) | |
| validate_acceptance(manifest, PRODUCTION, deep_payload=False) | |
| cfg, identity = prepare_resume(build_config(manifest, run, "train"), args.resume_checkpoint, | |
| manifest=manifest, run_dir=run) | |
| selection = json.loads(manifest.read_text()) | |
| rows = selection["sessions"] | |
| class Lengths: | |
| def __len__(self): | |
| return len(rows) | |
| def estimated_stream_steps(self, i): | |
| return rows[i]["dense_steps"] | |
| sampler = SourceLengthGroupedSampler(Lengths(), world_size=1, batch_size=2, seed=cfg.seed) | |
| cursor = identity["source_cursor"] | |
| if cfg.train.train_batch_size != 2 or cursor["epoch"] != 2: | |
| raise ValueError("This switch requires an original batch-2 third-epoch checkpoint") | |
| plans = {str(b): remaining_plan(selection, cursor, sampler._ordered_indices(2), b) | |
| for b in (2, 4, 8, 16)} | |
| return cfg, identity, selection, plans | |
| def restore_trainer(args, output): | |
| import torch | |
| from accelerate.utils import set_seed | |
| from scripts.train import _initialize_accelerator | |
| from scripts.third_party_resume import code_identity | |
| from psi.trainers.posttrain_third_party import PosttrainThirdPartyTrainer, append_record | |
| from psi.trainers.posttrain_full_duplex import PosttrainFullDuplexTrainer | |
| cfg, identity, selection, plans = prepare(args, output) | |
| os.environ["HFD_CONTINUOUS_DATA_ROOT"] = cfg.data.continuous_audio_root | |
| os.environ["HFD_BACKGROUND_ASSIGNMENT_PATH"] = cfg.data.background_assignment_path | |
| os.environ["HFD_CONTINUOUS_DATA_CONTRACT_PATH"] = str( | |
| Path(cfg.data.continuous_audio_root) / "reports/production_complete.json") | |
| output.mkdir(parents=True, exist_ok=True) | |
| implementation = ("scripts/train_third_party_strategy.py", "scripts/third_party_strategy_plan.py", | |
| "scripts/infer/third_party_metrics.py") | |
| write_json(output / "strategy_runtime_code_identity.json", { | |
| "time": time.time(), "files": {p: hashlib.sha256((PROJECT / p).read_bytes()).hexdigest() for p in implementation}}) | |
| write_json(output / "resume_identity.json", identity) | |
| write_json(output / "code_identity.json", code_identity()) | |
| write_json(output / "input_identity.json", {"manifest_sha256": identity["manifest_sha256"], | |
| "train_count": 300, "visual_inputs": 0, "other_task_inputs": 0, "heldout_inputs": 0}) | |
| class StrategyTrainer(PosttrainThirdPartyTrainer): | |
| def _checkpoint_probe(self, global_step): | |
| # Metrics are explicitly invoked by this driver after a complete | |
| # checkpoint; never run the inherited 12-video pause. | |
| return None | |
| set_seed(int(cfg.seed)) | |
| torch.cuda.set_device(0) | |
| trainer = StrategyTrainer(cfg, 0) | |
| trainer.init_models() | |
| accelerator = _initialize_accelerator(trainer) | |
| trainer.create_datasets() | |
| trainer.create_dataloaders(trainer.train_dataset, None) | |
| trainer.create_optimizer_and_scheduler(trainer.max_training_steps) | |
| trainer.prepare(accelerator) | |
| trainer.resume_from_checkpoint() | |
| assert trainer.global_step == identity["step"] | |
| return trainer, identity, selection, plans | |
| def source_batches(trainer, groups, workers=8): | |
| import torch | |
| from torch.utils.data import DataLoader | |
| from psi.trainers.posttrain_full_duplex import _source_collator | |
| from psi.trainers.trainer import worker_init_fn | |
| from psi.utils import batch_str_to_tensor | |
| flattened = [i for group in groups for i in group] | |
| loader = DataLoader(trainer.train_dataset, batch_size=1, sampler=flattened, | |
| num_workers=min(workers, len(flattened)), prefetch_factor=2, persistent_workers=False, | |
| pin_memory=True, collate_fn=_source_collator, worker_init_fn=worker_init_fn, | |
| generator=torch.Generator().manual_seed(26100601)) | |
| iterator = iter(loader) | |
| for group in groups: | |
| sources = [next(iterator)["sources"][0] for _ in group] | |
| yield batch_str_to_tensor({"sources": sources}) | |
| def configure_training_batch(trainer, size): | |
| trainer.train_cfg.train_batch_size = size | |
| trainer.train_cfg.target_effective_sources_per_gpu = size | |
| # Full-state restore must use the old contract; subsequent saves must use | |
| # the new batch. The base trainer caches this metadata at initialization. | |
| trainer._hfdv2_contract_cache = None | |
| trainer._hfdv2_checkpoint_contract() | |
| def benchmark(args): | |
| import torch | |
| trainer, identity, selection, plans = restore_trainer(args, args.run_dir) | |
| size = args.batch_size | |
| # Same long-source pool for all candidates. Benchmark weights are discarded. | |
| pending = [i for g in plans["2"]["source_indices"] for i in g] | |
| pending.sort(key=lambda i: selection["sessions"][i]["dense_steps"], reverse=True) | |
| group = pending[:size] | |
| configure_training_batch(trainer, size) | |
| trainer.set_train() | |
| batch = next(source_batches(trainer, [group])) | |
| torch.cuda.reset_peak_memory_stats() | |
| torch.cuda.synchronize() | |
| measured_start = None | |
| windows = 0 | |
| losses = [] | |
| started = time.monotonic() | |
| for i, (_, values, _) in enumerate(trainer.iter_optimizer_steps( | |
| batch, global_step_start=identity["step"], local_step=0, max_optimizer_events=240)): | |
| loss = float(values["loss"]) | |
| if not math.isfinite(loss): | |
| raise FloatingPointError("Non-finite benchmark loss") | |
| losses.append(loss) | |
| if i == 39: | |
| torch.cuda.synchronize() | |
| measured_start = time.monotonic() | |
| elif i >= 40: | |
| windows += size | |
| torch.cuda.synchronize() | |
| duration = time.monotonic() - measured_start | |
| result = {"status": "passed", "batch_size": size, "updates": len(losses), | |
| "measured_updates": len(losses) - 40, "active_windows_per_second": windows / duration, | |
| "measured_seconds": duration, "total_compute_seconds": time.monotonic() - started, | |
| "peak_allocated_gib": torch.cuda.max_memory_allocated() / 2**30, | |
| "peak_reserved_gib": torch.cuda.max_memory_reserved() / 2**30, | |
| "last_loss": losses[-1], "saved_weights": False} | |
| write_json(args.run_dir / "benchmark.json", result) | |
| print(json.dumps(result), flush=True) | |
| def continue_training(args): | |
| import torch | |
| from scripts.infer.third_party_metrics import evaluate_checkpoint | |
| from psi.trainers.posttrain_third_party import append_record | |
| from psi.utils import flatten | |
| from scripts.third_party_dashboard import update_dashboard | |
| trainer, identity, selection, plans = restore_trainer(args, args.run_dir) | |
| plan = plans[str(args.batch_size)] | |
| write_json(args.run_dir / "strategy_plan.json", plan) | |
| base = identity["step"] | |
| cfg = trainer.train_cfg | |
| configure_training_batch(trainer, args.batch_size) | |
| cfg.max_training_steps = plan["final_optimizer_step"] | |
| cfg.max_checkpoints_to_keep = 8 | |
| scheduler = getattr(trainer.lr_scheduler, "scheduler", trainer.lr_scheduler) | |
| old_lambdas = list(scheduler.lr_lambdas) | |
| scheduler.lr_lambdas = [ | |
| (lambda step, fn=fn: fn(mapped_scheduler_step( | |
| step, base, plan["remaining_updates"], plan["original_final_step"]))) | |
| for fn in old_lambdas] | |
| for group, fn in zip(trainer.optimizer.param_groups, scheduler.lr_lambdas): | |
| expected = group["initial_lr"] * fn(base) | |
| if not math.isclose(group["lr"], expected, rel_tol=1e-8, abs_tol=1e-12): | |
| raise RuntimeError("LR mapping would jump at the migration boundary") | |
| (args.run_dir / "run_config.json").write_text(trainer.cfg.model_dump_json(indent=2)) | |
| append_record(args.run_dir / "events.jsonl", {"event": "batch_migration_started", "time": time.time(), | |
| "step": base, "batch_size": args.batch_size, "remaining_sources": plan["source_count"], | |
| "remaining_updates": plan["remaining_updates"], "final_optimizer_step": plan["final_optimizer_step"], | |
| "original_final_step": plan["original_final_step"], "learning_rates": scheduler.get_last_lr()}) | |
| trainer.set_train() | |
| step = base | |
| completed_sources = [] | |
| halfway_saved = False | |
| for group_index, (indices, batch) in enumerate(zip(plan["source_indices"], source_batches(trainer, plan["source_indices"]))): | |
| # An incomplete final source group is real, smaller training—not a | |
| # dropped tail or padded duplicate. Checkpoints describe the max batch. | |
| cfg.train_batch_size = len(indices) | |
| cfg.target_effective_sources_per_gpu = len(indices) | |
| group_start = step | |
| boundary = False | |
| for _, values, boundary in trainer.iter_optimizer_steps( | |
| batch, global_step_start=step, local_step=group_index, | |
| max_optimizer_events=plan["group_steps"][group_index]): | |
| step += 1 | |
| trainer.global_step = step | |
| metrics = flatten({**values, "epoch": 2}, parent_key="train") | |
| metrics.update({"strategy/completed_sources": len(completed_sources), | |
| "strategy/target_sources": plan["source_count"], | |
| "strategy/source_batch_size": len(indices)}) | |
| trainer.log(metrics) | |
| if step % 100 == 0: | |
| print(f"[strategy_train] step={step}/{cfg.max_training_steps} loss={float(values['loss']):.5g} batch={len(indices)}", flush=True) | |
| if not boundary or step - group_start != plan["group_steps"][group_index]: | |
| raise RuntimeError("Source group ended outside the planned natural boundary") | |
| completed_sources.extend(indices) | |
| cfg.train_batch_size = args.batch_size | |
| cfg.target_effective_sources_per_gpu = args.batch_size | |
| state = {"step": step, "completed_group_count": group_index + 1, | |
| "completed_source_indices": completed_sources, | |
| "remaining_source_indices": [i for g in plan["source_indices"][group_index + 1:] for i in g], | |
| "parent_checkpoint": identity["checkpoint"], "plan": plan} | |
| write_json(args.run_dir / "strategy_state.json", state) | |
| final = group_index == len(plan["source_indices"]) - 1 | |
| halfway = not halfway_saved and len(completed_sources) >= plan["source_count"] / 2 | |
| if halfway or final: | |
| checkpoint = Path(trainer.save_checkpoint(step)) | |
| write_json(checkpoint / "strategy_state.json", state) | |
| append_record(args.run_dir / "events.jsonl", {"event": "metrics_evaluation_start", "step": step, | |
| "count": 300 if final else 60, "time": time.time()}) | |
| torch.cuda.empty_cache() | |
| result = evaluate_checkpoint(args.run_dir, step, args.manifest, | |
| args.run_dir / "metrics_evaluations" / f"step_{step}", | |
| workers=args.eval_workers, count=300 if final else 60, reuse=False) | |
| append_record(args.run_dir / "events.jsonl", {"event": "metrics_evaluation_complete", "step": step, | |
| "count": result["count"], "aggregate": result["aggregate"], "time": time.time()}) | |
| halfway_saved = True | |
| update_strategy_review(args.run_dir) | |
| if step != plan["final_optimizer_step"] or len(set(completed_sources)) != plan["source_count"]: | |
| raise RuntimeError("Remaining coverage incomplete") | |
| append_record(args.run_dir / "events.jsonl", {"event": "strategy_training_complete", "step": step, | |
| "remaining_sources_completed": len(completed_sources), "time": time.time()}) | |
| trainer.finalize() | |
| update_strategy_review(args.run_dir) | |
| def update_strategy_review(run): | |
| lines = ["# 并行策略训练与训练集复查", "", "仅使用固定 300 条训练数据;中途无视频渲染。", "", | |
| "| Checkpoint | 样例数 | 控制准确率 ↑ | 背景误触发率 ↓(逐条/逐帧) | 关节误差 ↓(弧度) | 指标 |", | |
| "|---|---:|---:|---:|---:|---|"] | |
| reports = list(run.glob("**/report.json")) | |
| origin = run / "strategy_origin.json" | |
| if origin.is_file(): | |
| reports.extend(Path(json.loads(origin.read_text())["previous_run"]).glob("history/*/report.json")) | |
| for path in sorted(reports): | |
| if "samples" in path.parts: | |
| continue | |
| result = json.loads(path.read_text()) | |
| if result.get("evaluation_split") != "train_only" or result.get("status") != "complete": | |
| continue | |
| if "inference_benchmark" in str(path): | |
| continue | |
| views = [(str(result["count"]), result["aggregate"])] | |
| if result["count"] == 300 and "fixed_sixty_comparison" in result: | |
| views.append(("其中固定 60", result["fixed_sixty_comparison"]["aggregate"])) | |
| for label, a in views: | |
| def percent(value): | |
| return "无有效时段" if value is None else f"{value:.1%}" | |
| bg = percent(a["background_only_false_motion_rate"]) | |
| weighted = percent(a["background_frame_weighted_false_motion_rate"]) | |
| lines.append(f"| {result['step']} | {label} | {a['control_accuracy']:.1%} | " | |
| f"{bg}/{weighted} | {a['joint_mae_radians']:.3f} | [报告]({os.path.relpath(path, run)}) |") | |
| lines.extend(["", "305990、459330 的权重已在本次工作前被轮换删除,不能补做;原 12 条报告保留。", | |
| "背景误触发仅统计有背景且 GT 不要求动作的时段;分母见报告。控制准确率提高不等于背景误触发已降低。", | |
| "最终 300 条同时单列相同随机种子的固定 60 条;不同样本数的整体均值不直接比较。", | |
| "数值与视频分离;全量指标不代表测试集泛化。改变 batch 后更新步数不与旧步数等价,覆盖以 strategy_plan.json 为准。"]) | |
| (run / "strategy_review.md").write_text("\n".join(lines) + "\n") | |
| def verified_prior_strategy(args, identity, selection, plans): | |
| prior = args.strategy_from.resolve() | |
| if not prior.is_relative_to(PROJECT / "runs") or prior == args.run_dir: | |
| raise ValueError("Use a distinct existing local strategy run") | |
| status = json.loads((prior / "run_status.json").read_text()) | |
| if status["phase"] not in ("failed", "completed") or status.get("keeper_restore_verified") is not True: | |
| raise ValueError("Previous strategy manager has not safely exited") | |
| chosen = json.loads((prior / "chosen_strategy.json").read_text()) | |
| measured = choose_batch(json.loads((prior / "training_benchmarks.json").read_text())) | |
| if (chosen["training"] != measured or chosen["eval_workers"] not in (1, 2, 4) | |
| or chosen["plan"] != plans[str(measured["batch_size"])]): | |
| raise ValueError("Measured strategy differs from this continuation plan") | |
| for step in (153530, 611980, identity["step"]): | |
| directory = prior / "history" / f"step_{step}" | |
| report = json.loads((directory / "report.json").read_text()) | |
| previous_identity = json.loads((directory / "identity.json").read_text()) | |
| if (report["status"] != "complete" or report["count"] != 60 | |
| or report["evaluation_split"] != "train_only" | |
| or [row["session_id"] for row in report["samples"]] != evaluation_ids(selection) | |
| or previous_identity["manifest_sha256"] != identity["manifest_sha256"]): | |
| raise ValueError("Historical fixed-sixty evaluation incomplete or incompatible") | |
| return chosen | |
| def run_continuation(args, plans, selected, eval_workers): | |
| command = [sys.executable, str(Path(__file__).resolve()), "--manifest", str(args.manifest), | |
| "--run-dir", str(args.run_dir), "--resume-checkpoint", str(args.resume_checkpoint), | |
| "--continue-training", "--batch-size", str(selected["batch_size"]), | |
| "--eval-workers", str(eval_workers)] | |
| write_json(args.run_dir / "workflow_state.json", {"phase": "continuing_training", "batch_size": selected["batch_size"], | |
| "remaining_sources": plans[str(selected["batch_size"])]["source_count"], "time": time.time()}) | |
| subprocess.run(command, check=True) | |
| write_json(args.run_dir / "workflow_state.json", {"phase": "completed", "time": time.time()}) | |
| def workflow(args): | |
| from scripts.infer.third_party_metrics import evaluate_checkpoint | |
| cfg, identity, selection, plans = prepare(args) | |
| write_json(args.run_dir / "evaluation_selection.json", {"ids": evaluation_ids(selection), | |
| "training_manifest": str(args.manifest), "seed": 26100601, "split": "train_only"}) | |
| if args.strategy_from: | |
| chosen = verified_prior_strategy(args, identity, selection, plans) | |
| for name in ("chosen_strategy.json", "training_benchmarks.json", "inference_benchmarks.json"): | |
| write_json(args.run_dir / name, json.loads((args.strategy_from / name).read_text())) | |
| write_json(args.run_dir / "strategy_origin.json", {"previous_run": str(args.strategy_from), | |
| "reuse": "completed benchmarks and three fixed-sixty evaluations; no benchmark weights", "time": time.time()}) | |
| update_strategy_review(args.run_dir) | |
| run_continuation(args, plans, chosen["training"], chosen["eval_workers"]) | |
| return | |
| snapshots = [(Path(identity["source_run"]).parent / "third_party_train_20261004_183932", 153530), | |
| (Path(identity["source_run"]), 611980), (Path(identity["source_run"]), 770084)] | |
| # Short real timing sweep on the same missing sessions, independent outputs. | |
| eval_ids = evaluation_ids(selection)[12:16] | |
| inference_results = [] | |
| for workers in (1, 2, 4): | |
| write_json(args.run_dir / "workflow_state.json", {"phase": "benchmark_inference", "workers": workers, "time": time.time()}) | |
| result = evaluate_checkpoint(snapshots[-1][0], 770084, args.manifest, | |
| args.run_dir / "inference_benchmark" / str(workers), workers=workers, | |
| subset=eval_ids, reuse=False) | |
| inference_results.append({"workers": workers, "seconds": result["elapsed_seconds"]}) | |
| write_json(args.run_dir / "inference_benchmarks.json", inference_results) | |
| fastest = min(r["seconds"] for r in inference_results) | |
| eval_workers = min(r["workers"] for r in inference_results if r["seconds"] <= fastest * 1.05) | |
| training_results = [] | |
| for batch in (2, 4, 8, 16): | |
| write_json(args.run_dir / "workflow_state.json", {"phase": "benchmark_training", "batch_size": batch, "time": time.time()}) | |
| directory = args.run_dir / "benchmarks" / f"batch_{batch}" | |
| directory.mkdir(parents=True) | |
| with (directory / "train.log").open("w") as log: | |
| result = subprocess.run([sys.executable, str(Path(__file__).resolve()), "--manifest", str(args.manifest), | |
| "--run-dir", str(directory), "--resume-checkpoint", str(args.resume_checkpoint), | |
| "--benchmark", "--batch-size", str(batch)], stdout=log, stderr=subprocess.STDOUT, timeout=1800) | |
| if result.returncode: | |
| training_results.append({"batch_size": batch, "status": "failed", "returncode": result.returncode}) | |
| else: | |
| training_results.append(json.loads((directory / "benchmark.json").read_text())) | |
| write_json(args.run_dir / "training_benchmarks.json", training_results) | |
| selected = choose_batch(training_results) | |
| write_json(args.run_dir / "chosen_strategy.json", {"training": selected, "eval_workers": eval_workers, | |
| "plan": plans[str(selected["batch_size"])]}) | |
| for source, step in snapshots: | |
| write_json(args.run_dir / "workflow_state.json", {"phase": "historical_metrics", "checkpoint_step": step, | |
| "count": 60, "workers": eval_workers, "time": time.time()}) | |
| evaluate_checkpoint(source, step, args.manifest, args.run_dir / "history" / f"step_{step}", | |
| workers=eval_workers, reuse=True) | |
| update_strategy_review(args.run_dir) | |
| # Fresh full-state restoration discards every benchmark parameter update. | |
| run_continuation(args, plans, selected, eval_workers) | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--manifest", type=Path, required=True) | |
| parser.add_argument("--run-dir", type=Path, required=True) | |
| parser.add_argument("--resume-checkpoint", type=Path, required=True) | |
| mode = parser.add_mutually_exclusive_group(required=True) | |
| for name in ("check-only", "launch", "worker", "benchmark", "continue-training"): | |
| mode.add_argument(f"--{name}", action="store_true") | |
| parser.add_argument("--batch-size", type=int, choices=(2, 4, 8, 16), default=2) | |
| parser.add_argument("--eval-workers", type=int, choices=(1, 2, 4), default=2) | |
| parser.add_argument("--strategy-from", type=Path, | |
| help="Reuse verified completed benchmarks and historical metrics after a safe stop") | |
| args = parser.parse_args() | |
| args.manifest, args.run_dir, args.resume_checkpoint = ( | |
| args.manifest.resolve(), args.run_dir.resolve(), args.resume_checkpoint.resolve()) | |
| setup_environment() | |
| _, identity, selection, plans = prepare(args) | |
| if args.strategy_from: | |
| args.strategy_from = args.strategy_from.resolve() | |
| verified_prior_strategy(args, identity, selection, plans) | |
| if args.check_only: | |
| print(json.dumps({"passed": True, "parent_step": identity["step"], | |
| "remaining_sources": plans['2']['source_count'], "planned_eval_count": len(evaluation_ids(selection)), | |
| "candidate_remaining_updates": {k:v['remaining_updates'] for k,v in plans.items()}})) | |
| return | |
| if args.launch: | |
| shared = [str(Path(__file__).resolve()), "--manifest", str(args.manifest), "--run-dir", str(args.run_dir), | |
| "--resume-checkpoint", str(args.resume_checkpoint)] | |
| if args.strategy_from: | |
| shared.extend(["--strategy-from", str(args.strategy_from)]) | |
| spec = {"schema": 1, "project_root": str(PROJECT), "run_dir": str(args.run_dir), "gpu": 3, | |
| "manifest_path": str(args.manifest), "readiness_file": str(args.manifest.parent / "acceptance.json"), | |
| "argv": [sys.executable, *shared, "--worker"], | |
| "preflight_command": [sys.executable, *shared, "--check-only"]} | |
| spec_path = args.manifest.parent / f"{args.run_dir.name}.launch.json" | |
| if spec_path.exists(): | |
| raise FileExistsError(spec_path) | |
| write_json(spec_path, spec) | |
| subprocess.run([sys.executable, str(PROJECT / "scripts/train/third_party_run.py"), "start", "--spec", str(spec_path)], check=True) | |
| return | |
| if os.environ.get("CUDA_VISIBLE_DEVICES") != "3" or os.environ.get("HFD_NPROC_PER_NODE") != "1": | |
| raise RuntimeError("Only use --launch for GPU work") | |
| args.run_dir.mkdir(parents=True, exist_ok=True) | |
| if args.benchmark: | |
| benchmark(args) | |
| elif args.continue_training: | |
| continue_training(args) | |
| else: | |
| paths = ("scripts/train_third_party_strategy.py", "scripts/third_party_strategy_plan.py", "scripts/infer/third_party_metrics.py") | |
| write_json(args.run_dir / "strategy_code_identity.json", {p: hashlib.sha256((PROJECT/p).read_bytes()).hexdigest() for p in paths}) | |
| workflow(args) | |
| if __name__ == "__main__": | |
| main() | |