File size: 3,530 Bytes
b8ced22
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
"""Train PURE Pi3 / DA3 / VGGT with one GPU or torchrun DDP."""

import argparse
import json
import os
from pathlib import Path
import sys

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

import torch
import torch.distributed as dist
from torch.nn.parallel import DistributedDataParallel
from pure.models import create_model
from pure.training.cache import ClipDataset
from pure.training.checkpoint import normalize_model_config
from pure.training.loading import load_weights
from pure.training.engine import fit
from pure.training.policy import configure_model


def main():
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument("--config", required=True)
    p.add_argument("--data", required=True)
    p.add_argument("--output", required=True)
    p.add_argument("--device", default="cuda")
    weights = p.add_mutually_exclusive_group(required=True)
    weights.add_argument("--checkpoint", help="Full refined checkpoint")
    weights.add_argument(
        "--base-checkpoint",
        help="Official base checkpoint; extra unused heads explicitly checked",
    )
    weights.add_argument("--resume", help="Resume training including optimizer and RNG")
    args = p.parse_args()
    config = normalize_model_config(json.loads(Path(args.config).read_text()))
    name = config.get("model", "pi3")
    if name == "pi3" and config["stage"] == "posttrain" and args.base_checkpoint:
        p.error(
            "Pi3 posttrain needs an initialized convolutional head; run head_init first"
        )
    # Validate the first cache before allocating a billion-parameter model.
    dataset = ClipDataset(args.data)
    dataset[0]
    world = int(os.environ.get("WORLD_SIZE", "1"))
    if world > 1:
        if args.device.startswith("cuda"):
            device = f"cuda:{int(os.environ['LOCAL_RANK'])}"
            torch.cuda.set_device(device)
            dist.init_process_group("nccl")
        else:
            device = args.device
            dist.init_process_group("gloo")
    else:
        device = args.device
    try:
        torch.manual_seed(config["seed"])
        model = create_model(name)
        if not args.resume:
            report = load_weights(
                model,
                args.base_checkpoint or args.checkpoint,
                name,
                base=bool(args.base_checkpoint),
            )
            if not dist.is_initialized() or dist.get_rank() == 0:
                print(json.dumps(report), flush=True)
        names = configure_model(model, config)
        if not dist.is_initialized() or dist.get_rank() == 0:
            print(
                json.dumps(
                    {
                        "model": name,
                        "trainable_parameters": sum(
                            p.numel() for p in model.parameters() if p.requires_grad
                        ),
                        "trainable_tensor_count": len(names),
                    }
                ),
                flush=True,
            )
        model.to(device)
        if world > 1:
            model = DistributedDataParallel(
                model,
                device_ids=(
                    [torch.device(device).index] if device.startswith("cuda") else None
                ),
                find_unused_parameters=True,
            )
        fit(model, dataset, config, args.output, device=device, resume=args.resume)
    finally:
        if dist.is_initialized():
            dist.destroy_process_group()


if __name__ == "__main__":
    main()