Hugging Face's logo

aklein4
/
oloop-instruct_baseline-fixed

Model card Files
xet
 Community
oloop-instruct_baseline-fixed
File size: 3,618 Bytes
0ed6663
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
{
    "git_commit": "a34fdf8bdceb4b644a9106819b9d6f7d6d0020e0",
    "wandb_run_id": "fnn7p0ul",
    "config": {
        "seed": 42,
        "notes": null,
        "trainer": {
            "run_dir": "local_data/runs/baseline-fixed",
            "checkpoint": {
                "load_state": false,
                "save_optimizer": false,
                "interval": 50
            },
            "wandb": {
                "project": "oloop-instruct",
                "enabled": true
            },
            "huggingface": {
                "private": false,
                "enabled": true
            },
            "global_batch_size": 128,
            "max_steps": 1000000,
            "num_logit_iterations": 4,
            "aux_loss_weight": 0.1,
            "max_grad_norm": 1.0,
            "debug": false,
            "groups": {
                "slow": {
                    "_target_": "optax.adamw",
                    "lr": 3e-05,
                    "b1": 0.9,
                    "b2": 0.95,
                    "eps": 1e-06,
                    "weight_decay": 0.0,
                    "warmup_steps": 100,
                    "stable_steps": 899900,
                    "decay_steps": 100000,
                    "min_lr_ratio": 0.0
                }
            },
            "_target_": "trainers.horizon_lm_trainer.HorizonLMTrainer"
        },
        "data": {
            "skip_batches": 0,
            "dataset": {
                "path": "aklein4/300k-horizons",
                "split": "train",
                "streaming": true
            },
            "collator": {
                "_target_": "collators.horizon.HorizonCollator",
                "tokenizer_url": "meta-llama/Llama-3.2-1B-Instruct",
                "max_length": 1024,
                "cluster_length": 64
            }
        },
        "distributed": {
            "initialize": "auto"
        },
        "mesh": {
            "fsdp_axis_size": -1
        },
        "name": "baseline-fixed",
        "initialization": "meta-llama/Llama-3.2-1B-Instruct",
        "initialization_step": null,
        "model": {
            "sharding": {
                "default": [],
                "rules": [
                    {
                        "pattern": "^embed_tokens/embedding$",
                        "spec": [
                            "fsdp",
                            null
                        ]
                    },
                    {
                        "pattern": "layers/mlp/(gate_proj|up_proj|down_proj)",
                        "spec": [
                            null,
                            null,
                            "fsdp"
                        ]
                    }
                ]
            },
            "_target_": "models.llama.LlamaForCausalLM",
            "vocab_size": 128256,
            "pad_token_id": -100,
            "hidden_size": 2048,
            "intermediate_size": 8192,
            "num_hidden_layers": 16,
            "num_attention_heads": 32,
            "num_key_value_heads": 8,
            "head_dim": 64,
            "max_position_embeddings": 131072,
            "rope": {
                "theta": 500000.0,
                "factor": 32.0,
                "high_freq_factor": 4.0,
                "low_freq_factor": 1.0,
                "original_context_len": 8192
            },
            "rms_norm_eps": 1e-05,
            "compute_dtype": "bfloat16",
            "gradient_checkpointing": true,
            "attention_backend": "auto"
        }
    },
    "step": 950,
    "checkpoint_items": [
        "params"
    ]
}