Hugging Face's logo

aklein4
/
oloop-instruct_ablate-scales

Model card Files
xet
 Community
oloop-instruct_ablate-scales
File size: 4,359 Bytes
01cb208
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
{
    "git_commit": "b1fea531a1bb067c5975282a447929a416793831",
    "wandb_run_id": "ef3f1rcw",
    "config": {
        "seed": 42,
        "notes": null,
        "trainer": {
            "run_dir": "local_data/runs/ablate-scales",
            "checkpoint": {
                "load_state": false,
                "save_optimizer": false,
                "interval": 50
            },
            "wandb": {
                "project": "oloop-instruct",
                "enabled": true
            },
            "huggingface": {
                "private": false,
                "enabled": true
            },
            "global_batch_size": 128,
            "max_steps": 1000000,
            "num_logit_iterations": 4,
            "aux_loss_weight": 0.1,
            "max_grad_norm": 1.0,
            "debug": false,
            "groups": {
                "slow": {
                    "_target_": "optax.adamw",
                    "lr": 3e-05,
                    "b1": 0.9,
                    "b2": 0.95,
                    "eps": 1e-06,
                    "weight_decay": 0.0,
                    "warmup_steps": 100,
                    "stable_steps": 899900,
                    "decay_steps": 100000,
                    "min_lr_ratio": 0.0
                },
                "fast": {
                    "_target_": "optax.contrib.muon",
                    "lr": 0.0005,
                    "beta": 0.9,
                    "adam_b1": 0.9,
                    "adam_b2": 0.95,
                    "eps": 1e-06,
                    "weight_decay": 0.01,
                    "ns_steps": 5,
                    "consistent_rms": 0.2,
                    "nesterov": true,
                    "warmup_steps": 100,
                    "stable_steps": 899900,
                    "decay_steps": 100000,
                    "min_lr_ratio": 0.0
                }
            },
            "_target_": "trainers.forte_trainer.ForteTrainer"
        },
        "data": {
            "skip_batches": 0,
            "dataset": {
                "path": "aklein4/300k-horizons",
                "split": "train",
                "streaming": true
            },
            "collator": {
                "_target_": "collators.horizon.HorizonCollator",
                "tokenizer_url": "meta-llama/Llama-3.2-1B-Instruct",
                "max_length": 1024,
                "cluster_length": 64
            }
        },
        "distributed": {
            "initialize": "auto"
        },
        "mesh": {
            "fsdp_axis_size": -1
        },
        "name": "ablate-scales",
        "initialization": "aklein4/forte-init-instruct",
        "initialization_step": 0,
        "model": {
            "sharding": {
                "default": [],
                "rules": [
                    {
                        "pattern": "^embed_tokens/embedding$",
                        "spec": [
                            "fsdp",
                            null
                        ]
                    },
                    {
                        "pattern": "layers/mlp/(gate_proj|up_proj|down_proj)",
                        "spec": [
                            null,
                            null,
                            "fsdp"
                        ]
                    }
                ]
            },
            "_target_": "models.forte.ForteModel",
            "vocab_size": 128256,
            "pad_token_id": -100,
            "hidden_size": 2048,
            "intermediate_size": 8192,
            "num_hidden_layers": 16,
            "num_attention_heads": 32,
            "num_key_value_heads": 8,
            "head_dim": 64,
            "max_position_embeddings": 131072,
            "rope": {
                "theta": 500000.0,
                "factor": 32.0,
                "high_freq_factor": 4.0,
                "low_freq_factor": 1.0,
                "original_context_len": 8192
            },
            "rms_norm_eps": 1e-05,
            "compute_dtype": "bfloat16",
            "gradient_checkpointing": true,
            "attention_backend": "auto",
            "num_fast_vectors": 512,
            "num_fast_heads": 16,
            "base_lr": 0.0001,
            "pool_size": 512,
            "grad_rms_eps": 1e-12
        }
    },
    "step": 400,
    "checkpoint_items": [
        "params"
    ]
}