Hugging Face's logo

tiny-research
/
athena

Model card Files
xet
 Community
vxnuaj commited on
Commit
fd35af4
·
verified ·
1 Parent(s): de3ff0d

Delete ATHENA_V1_TINY_39.7M

Browse files
Files changed (4) hide show
  1. ATHENA_V1_TINY_39.7M/checkpoints/RUN_001/RUN_001_DATETIME_2025-07-02_00-13-33_CONFIG.json +0 -112
  2. ATHENA_V1_TINY_39.7M/checkpoints/RUN_001/RUN_001_DATETIME_2025-07-02_00-13-33_checkpoint_0_step_10_global_steps_10.pt +0 -3
  3. ATHENA_V1_TINY_39.7M/checkpoints/RUN_001/RUN_001_DATETIME_2025-07-03_21-22-29_CONFIG.json +0 -112
  4. ATHENA_V1_TINY_39.7M/checkpoints/RUN_001/RUN_001_DATETIME_2025-07-03_21-22-29_EPOCH_0_STEP_10_GLOBAL_STEPS_10.pt +0 -3
ATHENA_V1_TINY_39.7M/checkpoints/RUN_001/RUN_001_DATETIME_2025-07-02_00-13-33_CONFIG.json DELETED
@@ -1,112 +0,0 @@
1
- {
2
- "train_config": {
3
- "vocab_size": 10000,
4
- "context_length": 512,
5
- "epochs": 5,
6
- "checkpoint_steps": 10,
7
- "save_checkpoint_path": "main/checkpoints",
8
- "save_hf": true,
9
- "val_steps": 100000,
10
- "mixed_precision": false,
11
- "max_grad_norm": 1.0,
12
- "track_grad_norm": true,
13
- "parallel_type": "fsdp",
14
- "val_mixed_precision": false,
15
- "val_mixed_precision_dtype": "f16",
16
- "fsdp_wrap_policy": "transformer",
17
- "wandb": false,
18
- "log_level": "INFO",
19
- "_compile": false,
20
- "_compile_warmup_steps": null,
21
- "hf_repo_config": {
22
- "hf_repo_exists": true,
23
- "hf_repo_id": "tiny-research/athena",
24
- "hf_repo_type": "model",
25
- "hf_root_path": "ATHENA_V1_TINY_39.7M/checkpoints"
26
- },
27
- "mixed_precision_dtype": "f16",
28
- "disable": 1,
29
- "disable_exclude": null,
30
- "extra_args": {},
31
- "log_root_path": "main/logs"
32
- },
33
- "optimizer_config": {
34
- "lr": 0.0005,
35
- "betas": [
36
- 0.9,
37
- 0.999
38
- ],
39
- "eps": 1e-08,
40
- "weight_decay": 0.01,
41
- "fused": true,
42
- "extra_args": {}
43
- },
44
- "scheduler_config": {
45
- "warmup_steps": 500,
46
- "constant_steps": 0,
47
- "decay_steps": 409870,
48
- "max_lr": 0.0001,
49
- "min_lr": 1e-05,
50
- "extra_args": {}
51
- },
52
- "dataloader_config": {
53
- "train_dataloader_config": {
54
- "train_batch_size": 64,
55
- "train_num_workers": 12,
56
- "train_shuffle": true,
57
- "train_pin_memory": true,
58
- "train_data_root_path": "data/tensors/train"
59
- },
60
- "val_dataloader_config": {
61
- "val_batch_size": 64,
62
- "val_num_workers": 12,
63
- "val_shuffle": false,
64
- "val_pin_memory": true,
65
- "val_data_root_path": "data/tensors/val"
66
- },
67
- "extra_args": {}
68
- },
69
- "model_config": {
70
- "context_len": 512,
71
- "d_model": 512,
72
- "n_heads": 8,
73
- "n_blocks": 12,
74
- "vocab_size": 10000,
75
- "pos_emb_dropout_p": 0.1,
76
- "pos_emb_type": "rope",
77
- "learned": false,
78
- "ntk_rope_scaling": false,
79
- "dyn_scaling": false,
80
- "attn_type": "mhsa",
81
- "n_groups": null,
82
- "top_k_sparsev": null,
83
- "p_threshold": null,
84
- "p_threshold_steps_fraction": null,
85
- "flash_attn": true,
86
- "flash_attn_dtype": "float16",
87
- "supress_warnings": true,
88
- "verbose": null,
89
- "model_name": "ATHENA_V1_TINY_39.7M",
90
- "model_series_name": "ATHENA",
91
- "extra_args": {}
92
- },
93
- "criterion_config": {
94
- "reduction": "none",
95
- "ignore_index": 0,
96
- "extra_args": {}
97
- },
98
- "wandb_config": {
99
- "project": "ATHENA",
100
- "name": "ATHENA_V1_TINY_39.7M",
101
- "entity": "vxnuaj",
102
- "tags": [
103
- "ATHENA",
104
- "39.7M",
105
- "PRETRAIN",
106
- "TINY",
107
- "V1"
108
- ],
109
- "notes": "",
110
- "extra_args": {}
111
- }
112
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ATHENA_V1_TINY_39.7M/checkpoints/RUN_001/RUN_001_DATETIME_2025-07-02_00-13-33_checkpoint_0_step_10_global_steps_10.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:af1c16fe2aa6eb93766b41a9eef48644bc988ecb894ac0405293229fd8834595
3
- size 499814091
 
 
 
 
ATHENA_V1_TINY_39.7M/checkpoints/RUN_001/RUN_001_DATETIME_2025-07-03_21-22-29_CONFIG.json DELETED
@@ -1,112 +0,0 @@
1
- {
2
- "train_config": {
3
- "vocab_size": 10000,
4
- "context_length": 512,
5
- "epochs": 5,
6
- "checkpoint_steps": 10,
7
- "save_checkpoint_path": "main/checkpoints",
8
- "save_hf": true,
9
- "val_steps": 10,
10
- "mixed_precision": false,
11
- "max_grad_norm": 1.0,
12
- "track_grad_norm": true,
13
- "parallel_type": "fsdp",
14
- "val_mixed_precision": false,
15
- "val_mixed_precision_dtype": "f16",
16
- "fsdp_wrap_policy": "transformer",
17
- "wandb": true,
18
- "log_level": "INFO",
19
- "_compile": false,
20
- "_compile_warmup_steps": null,
21
- "hf_repo_config": {
22
- "hf_repo_exists": true,
23
- "hf_repo_id": "tiny-research/athena",
24
- "hf_repo_type": "model",
25
- "hf_root_path": "ATHENA_V1_TINY_39.7M/checkpoints"
26
- },
27
- "mixed_precision_dtype": "f16",
28
- "log_root_path": "main/logs",
29
- "load_checkpoint": false,
30
- "load_checkpoint_path": null,
31
- "extra_args": {}
32
- },
33
- "optimizer_config": {
34
- "lr": 0.0005,
35
- "betas": [
36
- 0.9,
37
- 0.999
38
- ],
39
- "eps": 1e-08,
40
- "weight_decay": 0.01,
41
- "fused": true,
42
- "extra_args": {}
43
- },
44
- "scheduler_config": {
45
- "warmup_steps": 1000,
46
- "constant_steps": 1000,
47
- "decay_steps": 8852,
48
- "max_lr": 0.0001,
49
- "min_lr": 1e-05,
50
- "extra_args": {}
51
- },
52
- "dataloader_config": {
53
- "train_dataloader_config": {
54
- "train_batch_size": 192,
55
- "train_num_workers": 12,
56
- "train_shuffle": true,
57
- "train_pin_memory": true,
58
- "train_data_root_path": "data/tensors/train"
59
- },
60
- "val_dataloader_config": {
61
- "val_batch_size": 64,
62
- "val_num_workers": 12,
63
- "val_shuffle": false,
64
- "val_pin_memory": true,
65
- "val_data_root_path": "data/tensors/val"
66
- },
67
- "extra_args": {}
68
- },
69
- "model_config": {
70
- "context_len": 512,
71
- "d_model": 512,
72
- "n_heads": 8,
73
- "n_blocks": 12,
74
- "vocab_size": 10000,
75
- "pos_emb_dropout_p": 0.1,
76
- "pos_emb_type": "rope",
77
- "learned": false,
78
- "ntk_rope_scaling": false,
79
- "dyn_scaling": false,
80
- "attn_type": "mhsa",
81
- "n_groups": null,
82
- "top_k_sparsev": null,
83
- "p_threshold": null,
84
- "p_threshold_steps_fraction": null,
85
- "flash_attn": true,
86
- "flash_attn_dtype": "float16",
87
- "supress_warnings": true,
88
- "verbose": null,
89
- "model_name": "ATHENA_V1_TINY_39.7M",
90
- "model_series_name": "ATHENA \n 39.7M",
91
- "extra_args": {}
92
- },
93
- "criterion_config": {
94
- "reduction": "none",
95
- "ignore_index": 0,
96
- "extra_args": {}
97
- },
98
- "wandb_config": {
99
- "project": "ATHENA",
100
- "name": "ATHENA_V1_TINY_39.7M",
101
- "entity": "vxnuaj",
102
- "tags": [
103
- "ATHENA",
104
- "39.7M",
105
- "PRETRAIN",
106
- "TINY",
107
- "V1"
108
- ],
109
- "notes": "",
110
- "extra_args": {}
111
- }
112
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ATHENA_V1_TINY_39.7M/checkpoints/RUN_001/RUN_001_DATETIME_2025-07-03_21-22-29_EPOCH_0_STEP_10_GLOBAL_STEPS_10.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:e38947a52bde3933e0a89931dc73752e992bc9293450142c214147c582a66be0
3
- size 499781160