code-HFD_lzk / scripts /train_third_party_strategy.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
55e0599 verified
Raw History Blame Contribute Delete
25.4 kB
#!/usr/bin/env python3
"""Measured batch migration, historical train-only metrics, then continuation.
The original model/loss/optimizer implementation is reused unchanged. The new
driver changes source grouping and explicitly maps the remaining LR schedule.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import math
import os
from pathlib import Path
import subprocess
import sys
import time
PROJECT = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(PROJECT), str(PROJECT / "src")]
from scripts.third_party_strategy_plan import (
evaluation_ids, remaining_plan, mapped_scheduler_step, choose_batch, write_json,
)
def setup_environment():
os.environ.update(HFD_SKIP_DOTENV="1", HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1",
PSI_SKIP_VLM_MODEL="0", PSI_FLOW_SIGMA_MIN="0.05", PSI_FLOW_SIGMA_MAX="1.0",
PSI_WORKER_TORCH_THREADS="1", PSI_DATALOADER_PIN_MEMORY="1",
PSI_FOREACH_GRAD_NORM="1", OMP_NUM_THREADS="4", OPENBLAS_NUM_THREADS="1")
os.environ.pop("_ENV_INFO_JSON", None)
for variable, name in (("TORCHINDUCTOR_CACHE_DIR", "inductor"),
("TRITON_CACHE_DIR", "triton"), ("CUDA_CACHE_PATH", "cuda")):
directory = PROJECT / "cache" / "third_party_training" / name
directory.mkdir(parents=True, exist_ok=True)
os.environ[variable] = str(directory)
from scripts.third_party_temp import configure_project_temp
configure_project_temp(PROJECT)
def prepare(args, output=None):
from scripts.train_third_party import build_config, contained_output, PRODUCTION
from scripts.third_party_resume import prepare_resume, frozen_resume_cursor
from scripts.data.prepare_third_party_training import validate_acceptance
from psi.trainers.posttrain_full_duplex import SourceLengthGroupedSampler
manifest = args.manifest.resolve()
run = contained_output(output or args.run_dir)
validate_acceptance(manifest, PRODUCTION, deep_payload=False)
cfg, identity = prepare_resume(build_config(manifest, run, "train"), args.resume_checkpoint,
manifest=manifest, run_dir=run)
selection = json.loads(manifest.read_text())
rows = selection["sessions"]
class Lengths:
def __len__(self):
return len(rows)
def estimated_stream_steps(self, i):
return rows[i]["dense_steps"]
sampler = SourceLengthGroupedSampler(Lengths(), world_size=1, batch_size=2, seed=cfg.seed)
cursor = identity["source_cursor"]
if cfg.train.train_batch_size != 2 or cursor["epoch"] != 2:
raise ValueError("This switch requires an original batch-2 third-epoch checkpoint")
plans = {str(b): remaining_plan(selection, cursor, sampler._ordered_indices(2), b)
for b in (2, 4, 8, 16)}
return cfg, identity, selection, plans
def restore_trainer(args, output):
import torch
from accelerate.utils import set_seed
from scripts.train import _initialize_accelerator
from scripts.third_party_resume import code_identity
from psi.trainers.posttrain_third_party import PosttrainThirdPartyTrainer, append_record
from psi.trainers.posttrain_full_duplex import PosttrainFullDuplexTrainer
cfg, identity, selection, plans = prepare(args, output)
os.environ["HFD_CONTINUOUS_DATA_ROOT"] = cfg.data.continuous_audio_root
os.environ["HFD_BACKGROUND_ASSIGNMENT_PATH"] = cfg.data.background_assignment_path
os.environ["HFD_CONTINUOUS_DATA_CONTRACT_PATH"] = str(
Path(cfg.data.continuous_audio_root) / "reports/production_complete.json")
output.mkdir(parents=True, exist_ok=True)
implementation = ("scripts/train_third_party_strategy.py", "scripts/third_party_strategy_plan.py",
"scripts/infer/third_party_metrics.py")
write_json(output / "strategy_runtime_code_identity.json", {
"time": time.time(), "files": {p: hashlib.sha256((PROJECT / p).read_bytes()).hexdigest() for p in implementation}})
write_json(output / "resume_identity.json", identity)
write_json(output / "code_identity.json", code_identity())
write_json(output / "input_identity.json", {"manifest_sha256": identity["manifest_sha256"],
"train_count": 300, "visual_inputs": 0, "other_task_inputs": 0, "heldout_inputs": 0})
class StrategyTrainer(PosttrainThirdPartyTrainer):
def _checkpoint_probe(self, global_step):
# Metrics are explicitly invoked by this driver after a complete
# checkpoint; never run the inherited 12-video pause.
return None
set_seed(int(cfg.seed))
torch.cuda.set_device(0)
trainer = StrategyTrainer(cfg, 0)
trainer.init_models()
accelerator = _initialize_accelerator(trainer)
trainer.create_datasets()
trainer.create_dataloaders(trainer.train_dataset, None)
trainer.create_optimizer_and_scheduler(trainer.max_training_steps)
trainer.prepare(accelerator)
trainer.resume_from_checkpoint()
assert trainer.global_step == identity["step"]
return trainer, identity, selection, plans
def source_batches(trainer, groups, workers=8):
import torch
from torch.utils.data import DataLoader
from psi.trainers.posttrain_full_duplex import _source_collator
from psi.trainers.trainer import worker_init_fn
from psi.utils import batch_str_to_tensor
flattened = [i for group in groups for i in group]
loader = DataLoader(trainer.train_dataset, batch_size=1, sampler=flattened,
num_workers=min(workers, len(flattened)), prefetch_factor=2, persistent_workers=False,
pin_memory=True, collate_fn=_source_collator, worker_init_fn=worker_init_fn,
generator=torch.Generator().manual_seed(26100601))
iterator = iter(loader)
for group in groups:
sources = [next(iterator)["sources"][0] for _ in group]
yield batch_str_to_tensor({"sources": sources})
def configure_training_batch(trainer, size):
trainer.train_cfg.train_batch_size = size
trainer.train_cfg.target_effective_sources_per_gpu = size
# Full-state restore must use the old contract; subsequent saves must use
# the new batch. The base trainer caches this metadata at initialization.
trainer._hfdv2_contract_cache = None
trainer._hfdv2_checkpoint_contract()
def benchmark(args):
import torch
trainer, identity, selection, plans = restore_trainer(args, args.run_dir)
size = args.batch_size
# Same long-source pool for all candidates. Benchmark weights are discarded.
pending = [i for g in plans["2"]["source_indices"] for i in g]
pending.sort(key=lambda i: selection["sessions"][i]["dense_steps"], reverse=True)
group = pending[:size]
configure_training_batch(trainer, size)
trainer.set_train()
batch = next(source_batches(trainer, [group]))
torch.cuda.reset_peak_memory_stats()
torch.cuda.synchronize()
measured_start = None
windows = 0
losses = []
started = time.monotonic()
for i, (_, values, _) in enumerate(trainer.iter_optimizer_steps(
batch, global_step_start=identity["step"], local_step=0, max_optimizer_events=240)):
loss = float(values["loss"])
if not math.isfinite(loss):
raise FloatingPointError("Non-finite benchmark loss")
losses.append(loss)
if i == 39:
torch.cuda.synchronize()
measured_start = time.monotonic()
elif i >= 40:
windows += size
torch.cuda.synchronize()
duration = time.monotonic() - measured_start
result = {"status": "passed", "batch_size": size, "updates": len(losses),
"measured_updates": len(losses) - 40, "active_windows_per_second": windows / duration,
"measured_seconds": duration, "total_compute_seconds": time.monotonic() - started,
"peak_allocated_gib": torch.cuda.max_memory_allocated() / 2**30,
"peak_reserved_gib": torch.cuda.max_memory_reserved() / 2**30,
"last_loss": losses[-1], "saved_weights": False}
write_json(args.run_dir / "benchmark.json", result)
print(json.dumps(result), flush=True)
def continue_training(args):
import torch
from scripts.infer.third_party_metrics import evaluate_checkpoint
from psi.trainers.posttrain_third_party import append_record
from psi.utils import flatten
from scripts.third_party_dashboard import update_dashboard
trainer, identity, selection, plans = restore_trainer(args, args.run_dir)
plan = plans[str(args.batch_size)]
write_json(args.run_dir / "strategy_plan.json", plan)
base = identity["step"]
cfg = trainer.train_cfg
configure_training_batch(trainer, args.batch_size)
cfg.max_training_steps = plan["final_optimizer_step"]
cfg.max_checkpoints_to_keep = 8
scheduler = getattr(trainer.lr_scheduler, "scheduler", trainer.lr_scheduler)
old_lambdas = list(scheduler.lr_lambdas)
scheduler.lr_lambdas = [
(lambda step, fn=fn: fn(mapped_scheduler_step(
step, base, plan["remaining_updates"], plan["original_final_step"])))
for fn in old_lambdas]
for group, fn in zip(trainer.optimizer.param_groups, scheduler.lr_lambdas):
expected = group["initial_lr"] * fn(base)
if not math.isclose(group["lr"], expected, rel_tol=1e-8, abs_tol=1e-12):
raise RuntimeError("LR mapping would jump at the migration boundary")
(args.run_dir / "run_config.json").write_text(trainer.cfg.model_dump_json(indent=2))
append_record(args.run_dir / "events.jsonl", {"event": "batch_migration_started", "time": time.time(),
"step": base, "batch_size": args.batch_size, "remaining_sources": plan["source_count"],
"remaining_updates": plan["remaining_updates"], "final_optimizer_step": plan["final_optimizer_step"],
"original_final_step": plan["original_final_step"], "learning_rates": scheduler.get_last_lr()})
trainer.set_train()
step = base
completed_sources = []
halfway_saved = False
for group_index, (indices, batch) in enumerate(zip(plan["source_indices"], source_batches(trainer, plan["source_indices"]))):
# An incomplete final source group is real, smaller training—not a
# dropped tail or padded duplicate. Checkpoints describe the max batch.
cfg.train_batch_size = len(indices)
cfg.target_effective_sources_per_gpu = len(indices)
group_start = step
boundary = False
for _, values, boundary in trainer.iter_optimizer_steps(
batch, global_step_start=step, local_step=group_index,
max_optimizer_events=plan["group_steps"][group_index]):
step += 1
trainer.global_step = step
metrics = flatten({**values, "epoch": 2}, parent_key="train")
metrics.update({"strategy/completed_sources": len(completed_sources),
"strategy/target_sources": plan["source_count"],
"strategy/source_batch_size": len(indices)})
trainer.log(metrics)
if step % 100 == 0:
print(f"[strategy_train] step={step}/{cfg.max_training_steps} loss={float(values['loss']):.5g} batch={len(indices)}", flush=True)
if not boundary or step - group_start != plan["group_steps"][group_index]:
raise RuntimeError("Source group ended outside the planned natural boundary")
completed_sources.extend(indices)
cfg.train_batch_size = args.batch_size
cfg.target_effective_sources_per_gpu = args.batch_size
state = {"step": step, "completed_group_count": group_index + 1,
"completed_source_indices": completed_sources,
"remaining_source_indices": [i for g in plan["source_indices"][group_index + 1:] for i in g],
"parent_checkpoint": identity["checkpoint"], "plan": plan}
write_json(args.run_dir / "strategy_state.json", state)
final = group_index == len(plan["source_indices"]) - 1
halfway = not halfway_saved and len(completed_sources) >= plan["source_count"] / 2
if halfway or final:
checkpoint = Path(trainer.save_checkpoint(step))
write_json(checkpoint / "strategy_state.json", state)
append_record(args.run_dir / "events.jsonl", {"event": "metrics_evaluation_start", "step": step,
"count": 300 if final else 60, "time": time.time()})
torch.cuda.empty_cache()
result = evaluate_checkpoint(args.run_dir, step, args.manifest,
args.run_dir / "metrics_evaluations" / f"step_{step}",
workers=args.eval_workers, count=300 if final else 60, reuse=False)
append_record(args.run_dir / "events.jsonl", {"event": "metrics_evaluation_complete", "step": step,
"count": result["count"], "aggregate": result["aggregate"], "time": time.time()})
halfway_saved = True
update_strategy_review(args.run_dir)
if step != plan["final_optimizer_step"] or len(set(completed_sources)) != plan["source_count"]:
raise RuntimeError("Remaining coverage incomplete")
append_record(args.run_dir / "events.jsonl", {"event": "strategy_training_complete", "step": step,
"remaining_sources_completed": len(completed_sources), "time": time.time()})
trainer.finalize()
update_strategy_review(args.run_dir)
def update_strategy_review(run):
lines = ["# 并行策略训练与训练集复查", "", "仅使用固定 300 条训练数据;中途无视频渲染。", "",
"| Checkpoint | 样例数 | 控制准确率 ↑ | 背景误触发率 ↓(逐条/逐帧) | 关节误差 ↓(弧度) | 指标 |",
"|---|---:|---:|---:|---:|---|"]
reports = list(run.glob("**/report.json"))
origin = run / "strategy_origin.json"
if origin.is_file():
reports.extend(Path(json.loads(origin.read_text())["previous_run"]).glob("history/*/report.json"))
for path in sorted(reports):
if "samples" in path.parts:
continue
result = json.loads(path.read_text())
if result.get("evaluation_split") != "train_only" or result.get("status") != "complete":
continue
if "inference_benchmark" in str(path):
continue
views = [(str(result["count"]), result["aggregate"])]
if result["count"] == 300 and "fixed_sixty_comparison" in result:
views.append(("其中固定 60", result["fixed_sixty_comparison"]["aggregate"]))
for label, a in views:
def percent(value):
return "无有效时段" if value is None else f"{value:.1%}"
bg = percent(a["background_only_false_motion_rate"])
weighted = percent(a["background_frame_weighted_false_motion_rate"])
lines.append(f"| {result['step']} | {label} | {a['control_accuracy']:.1%} | "
f"{bg}/{weighted} | {a['joint_mae_radians']:.3f} | [报告]({os.path.relpath(path, run)}) |")
lines.extend(["", "305990、459330 的权重已在本次工作前被轮换删除,不能补做;原 12 条报告保留。",
"背景误触发仅统计有背景且 GT 不要求动作的时段;分母见报告。控制准确率提高不等于背景误触发已降低。",
"最终 300 条同时单列相同随机种子的固定 60 条;不同样本数的整体均值不直接比较。",
"数值与视频分离;全量指标不代表测试集泛化。改变 batch 后更新步数不与旧步数等价,覆盖以 strategy_plan.json 为准。"])
(run / "strategy_review.md").write_text("\n".join(lines) + "\n")
def verified_prior_strategy(args, identity, selection, plans):
prior = args.strategy_from.resolve()
if not prior.is_relative_to(PROJECT / "runs") or prior == args.run_dir:
raise ValueError("Use a distinct existing local strategy run")
status = json.loads((prior / "run_status.json").read_text())
if status["phase"] not in ("failed", "completed") or status.get("keeper_restore_verified") is not True:
raise ValueError("Previous strategy manager has not safely exited")
chosen = json.loads((prior / "chosen_strategy.json").read_text())
measured = choose_batch(json.loads((prior / "training_benchmarks.json").read_text()))
if (chosen["training"] != measured or chosen["eval_workers"] not in (1, 2, 4)
or chosen["plan"] != plans[str(measured["batch_size"])]):
raise ValueError("Measured strategy differs from this continuation plan")
for step in (153530, 611980, identity["step"]):
directory = prior / "history" / f"step_{step}"
report = json.loads((directory / "report.json").read_text())
previous_identity = json.loads((directory / "identity.json").read_text())
if (report["status"] != "complete" or report["count"] != 60
or report["evaluation_split"] != "train_only"
or [row["session_id"] for row in report["samples"]] != evaluation_ids(selection)
or previous_identity["manifest_sha256"] != identity["manifest_sha256"]):
raise ValueError("Historical fixed-sixty evaluation incomplete or incompatible")
return chosen
def run_continuation(args, plans, selected, eval_workers):
command = [sys.executable, str(Path(__file__).resolve()), "--manifest", str(args.manifest),
"--run-dir", str(args.run_dir), "--resume-checkpoint", str(args.resume_checkpoint),
"--continue-training", "--batch-size", str(selected["batch_size"]),
"--eval-workers", str(eval_workers)]
write_json(args.run_dir / "workflow_state.json", {"phase": "continuing_training", "batch_size": selected["batch_size"],
"remaining_sources": plans[str(selected["batch_size"])]["source_count"], "time": time.time()})
subprocess.run(command, check=True)
write_json(args.run_dir / "workflow_state.json", {"phase": "completed", "time": time.time()})
def workflow(args):
from scripts.infer.third_party_metrics import evaluate_checkpoint
cfg, identity, selection, plans = prepare(args)
write_json(args.run_dir / "evaluation_selection.json", {"ids": evaluation_ids(selection),
"training_manifest": str(args.manifest), "seed": 26100601, "split": "train_only"})
if args.strategy_from:
chosen = verified_prior_strategy(args, identity, selection, plans)
for name in ("chosen_strategy.json", "training_benchmarks.json", "inference_benchmarks.json"):
write_json(args.run_dir / name, json.loads((args.strategy_from / name).read_text()))
write_json(args.run_dir / "strategy_origin.json", {"previous_run": str(args.strategy_from),
"reuse": "completed benchmarks and three fixed-sixty evaluations; no benchmark weights", "time": time.time()})
update_strategy_review(args.run_dir)
run_continuation(args, plans, chosen["training"], chosen["eval_workers"])
return
snapshots = [(Path(identity["source_run"]).parent / "third_party_train_20261004_183932", 153530),
(Path(identity["source_run"]), 611980), (Path(identity["source_run"]), 770084)]
# Short real timing sweep on the same missing sessions, independent outputs.
eval_ids = evaluation_ids(selection)[12:16]
inference_results = []
for workers in (1, 2, 4):
write_json(args.run_dir / "workflow_state.json", {"phase": "benchmark_inference", "workers": workers, "time": time.time()})
result = evaluate_checkpoint(snapshots[-1][0], 770084, args.manifest,
args.run_dir / "inference_benchmark" / str(workers), workers=workers,
subset=eval_ids, reuse=False)
inference_results.append({"workers": workers, "seconds": result["elapsed_seconds"]})
write_json(args.run_dir / "inference_benchmarks.json", inference_results)
fastest = min(r["seconds"] for r in inference_results)
eval_workers = min(r["workers"] for r in inference_results if r["seconds"] <= fastest * 1.05)
training_results = []
for batch in (2, 4, 8, 16):
write_json(args.run_dir / "workflow_state.json", {"phase": "benchmark_training", "batch_size": batch, "time": time.time()})
directory = args.run_dir / "benchmarks" / f"batch_{batch}"
directory.mkdir(parents=True)
with (directory / "train.log").open("w") as log:
result = subprocess.run([sys.executable, str(Path(__file__).resolve()), "--manifest", str(args.manifest),
"--run-dir", str(directory), "--resume-checkpoint", str(args.resume_checkpoint),
"--benchmark", "--batch-size", str(batch)], stdout=log, stderr=subprocess.STDOUT, timeout=1800)
if result.returncode:
training_results.append({"batch_size": batch, "status": "failed", "returncode": result.returncode})
else:
training_results.append(json.loads((directory / "benchmark.json").read_text()))
write_json(args.run_dir / "training_benchmarks.json", training_results)
selected = choose_batch(training_results)
write_json(args.run_dir / "chosen_strategy.json", {"training": selected, "eval_workers": eval_workers,
"plan": plans[str(selected["batch_size"])]})
for source, step in snapshots:
write_json(args.run_dir / "workflow_state.json", {"phase": "historical_metrics", "checkpoint_step": step,
"count": 60, "workers": eval_workers, "time": time.time()})
evaluate_checkpoint(source, step, args.manifest, args.run_dir / "history" / f"step_{step}",
workers=eval_workers, reuse=True)
update_strategy_review(args.run_dir)
# Fresh full-state restoration discards every benchmark parameter update.
run_continuation(args, plans, selected, eval_workers)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--manifest", type=Path, required=True)
parser.add_argument("--run-dir", type=Path, required=True)
parser.add_argument("--resume-checkpoint", type=Path, required=True)
mode = parser.add_mutually_exclusive_group(required=True)
for name in ("check-only", "launch", "worker", "benchmark", "continue-training"):
mode.add_argument(f"--{name}", action="store_true")
parser.add_argument("--batch-size", type=int, choices=(2, 4, 8, 16), default=2)
parser.add_argument("--eval-workers", type=int, choices=(1, 2, 4), default=2)
parser.add_argument("--strategy-from", type=Path,
help="Reuse verified completed benchmarks and historical metrics after a safe stop")
args = parser.parse_args()
args.manifest, args.run_dir, args.resume_checkpoint = (
args.manifest.resolve(), args.run_dir.resolve(), args.resume_checkpoint.resolve())
setup_environment()
_, identity, selection, plans = prepare(args)
if args.strategy_from:
args.strategy_from = args.strategy_from.resolve()
verified_prior_strategy(args, identity, selection, plans)
if args.check_only:
print(json.dumps({"passed": True, "parent_step": identity["step"],
"remaining_sources": plans['2']['source_count'], "planned_eval_count": len(evaluation_ids(selection)),
"candidate_remaining_updates": {k:v['remaining_updates'] for k,v in plans.items()}}))
return
if args.launch:
shared = [str(Path(__file__).resolve()), "--manifest", str(args.manifest), "--run-dir", str(args.run_dir),
"--resume-checkpoint", str(args.resume_checkpoint)]
if args.strategy_from:
shared.extend(["--strategy-from", str(args.strategy_from)])
spec = {"schema": 1, "project_root": str(PROJECT), "run_dir": str(args.run_dir), "gpu": 3,
"manifest_path": str(args.manifest), "readiness_file": str(args.manifest.parent / "acceptance.json"),
"argv": [sys.executable, *shared, "--worker"],
"preflight_command": [sys.executable, *shared, "--check-only"]}
spec_path = args.manifest.parent / f"{args.run_dir.name}.launch.json"
if spec_path.exists():
raise FileExistsError(spec_path)
write_json(spec_path, spec)
subprocess.run([sys.executable, str(PROJECT / "scripts/train/third_party_run.py"), "start", "--spec", str(spec_path)], check=True)
return
if os.environ.get("CUDA_VISIBLE_DEVICES") != "3" or os.environ.get("HFD_NPROC_PER_NODE") != "1":
raise RuntimeError("Only use --launch for GPU work")
args.run_dir.mkdir(parents=True, exist_ok=True)
if args.benchmark:
benchmark(args)
elif args.continue_training:
continue_training(args)
else:
paths = ("scripts/train_third_party_strategy.py", "scripts/third_party_strategy_plan.py", "scripts/infer/third_party_metrics.py")
write_json(args.run_dir / "strategy_code_identity.json", {p: hashlib.sha256((PROJECT/p).read_bytes()).hexdigest() for p in paths})
workflow(args)
if __name__ == "__main__":
main()