Spaces:
Running on Zero
Running on Zero
Download scripts/train.py from cyun9286/PURE-Studio: direct link, hf CLI and curl.
- Browser
- Download file 3.53 kB
-
https://huggingface.co/spaces/cyun9286/PURE-Studio/resolve/main/scripts/train.py
- Command line
-
hf download hf://spaces/cyun9286/PURE-Studio/scripts/train.py
-
curl -L -o train.py https://huggingface.co/spaces/cyun9286/PURE-Studio/resolve/main/scripts/train.py
3.53 kB
| """Train PURE Pi3 / DA3 / VGGT with one GPU or torchrun DDP.""" | |
| import argparse | |
| import json | |
| import os | |
| from pathlib import Path | |
| import sys | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| import torch | |
| import torch.distributed as dist | |
| from torch.nn.parallel import DistributedDataParallel | |
| from pure.models import create_model | |
| from pure.training.cache import ClipDataset | |
| from pure.training.checkpoint import normalize_model_config | |
| from pure.training.loading import load_weights | |
| from pure.training.engine import fit | |
| from pure.training.policy import configure_model | |
| def main(): | |
| p = argparse.ArgumentParser(description=__doc__) | |
| p.add_argument("--config", required=True) | |
| p.add_argument("--data", required=True) | |
| p.add_argument("--output", required=True) | |
| p.add_argument("--device", default="cuda") | |
| weights = p.add_mutually_exclusive_group(required=True) | |
| weights.add_argument("--checkpoint", help="Full refined checkpoint") | |
| weights.add_argument( | |
| "--base-checkpoint", | |
| help="Official base checkpoint; extra unused heads explicitly checked", | |
| ) | |
| weights.add_argument("--resume", help="Resume training including optimizer and RNG") | |
| args = p.parse_args() | |
| config = normalize_model_config(json.loads(Path(args.config).read_text())) | |
| name = config.get("model", "pi3") | |
| if name == "pi3" and config["stage"] == "posttrain" and args.base_checkpoint: | |
| p.error( | |
| "Pi3 posttrain needs an initialized convolutional head; run head_init first" | |
| ) | |
| # Validate the first cache before allocating a billion-parameter model. | |
| dataset = ClipDataset(args.data) | |
| dataset[0] | |
| world = int(os.environ.get("WORLD_SIZE", "1")) | |
| if world > 1: | |
| if args.device.startswith("cuda"): | |
| device = f"cuda:{int(os.environ['LOCAL_RANK'])}" | |
| torch.cuda.set_device(device) | |
| dist.init_process_group("nccl") | |
| else: | |
| device = args.device | |
| dist.init_process_group("gloo") | |
| else: | |
| device = args.device | |
| try: | |
| torch.manual_seed(config["seed"]) | |
| model = create_model(name) | |
| if not args.resume: | |
| report = load_weights( | |
| model, | |
| args.base_checkpoint or args.checkpoint, | |
| name, | |
| base=bool(args.base_checkpoint), | |
| ) | |
| if not dist.is_initialized() or dist.get_rank() == 0: | |
| print(json.dumps(report), flush=True) | |
| names = configure_model(model, config) | |
| if not dist.is_initialized() or dist.get_rank() == 0: | |
| print( | |
| json.dumps( | |
| { | |
| "model": name, | |
| "trainable_parameters": sum( | |
| p.numel() for p in model.parameters() if p.requires_grad | |
| ), | |
| "trainable_tensor_count": len(names), | |
| } | |
| ), | |
| flush=True, | |
| ) | |
| model.to(device) | |
| if world > 1: | |
| model = DistributedDataParallel( | |
| model, | |
| device_ids=( | |
| [torch.device(device).index] if device.startswith("cuda") else None | |
| ), | |
| find_unused_parameters=True, | |
| ) | |
| fit(model, dataset, config, args.output, device=device, resume=args.resume) | |
| finally: | |
| if dist.is_initialized(): | |
| dist.destroy_process_group() | |
| if __name__ == "__main__": | |
| main() | |