belal212 commited on
Commit
7cd09ae
·
verified ·
1 Parent(s): 341d619

cleanup checkpoints/

Browse files
Files changed (1) hide show
  1. checkpoints/step_0004000/config.json +0 -135
checkpoints/step_0004000/config.json DELETED
@@ -1,135 +0,0 @@
1
- {
2
- "model": {
3
- "text": {
4
- "vocab_size": 2751,
5
- "max_length": 96,
6
- "d_model": 320,
7
- "n_layers": 5,
8
- "n_heads": 8,
9
- "ffn_mult": 4.0,
10
- "dropout": 0.1,
11
- "tie_embeddings": false
12
- },
13
- "generator": {
14
- "palette_size": 4097,
15
- "image_size": 16,
16
- "d_model": 384,
17
- "n_layers": 12,
18
- "n_heads": 8,
19
- "ffn_mult": 4.0,
20
- "dropout": 0.1,
21
- "cross_attention": true,
22
- "tie_embeddings": true,
23
- "null_tokens": 4,
24
- "elastic": false,
25
- "ffn_start_neurons": 128,
26
- "ffn_chunk": 64,
27
- "ffn_max_neurons": 1024,
28
- "ffn_min_neurons": 96,
29
- "ffn_widths": [
30
- 128,
31
- 128,
32
- 128,
33
- 128,
34
- 128,
35
- 128,
36
- 128,
37
- 128,
38
- 128,
39
- 128,
40
- 128,
41
- 128
42
- ]
43
- },
44
- "name": "adaptive",
45
- "contrastive_dim": 128
46
- },
47
- "train": {
48
- "preset": "adaptive",
49
- "max_steps": 8000,
50
- "batch_size": 64,
51
- "grad_accum": 1,
52
- "lr": 0.0001,
53
- "min_lr_ratio": 0.05,
54
- "warmup_steps": 200,
55
- "weight_decay": 0.05,
56
- "beta1": 0.9,
57
- "beta2": 0.95,
58
- "grad_clip": 1.0,
59
- "eval_interval": 500,
60
- "save_interval": 1000,
61
- "log_interval": 25,
62
- "val_examples": 2000,
63
- "max_train_examples": null,
64
- "output_dir": "/kaggle/working/pixelgen_runs/adaptive_continue",
65
- "resume": null,
66
- "seed": 42,
67
- "device": "auto",
68
- "precision": "auto",
69
- "num_prefetch": 8,
70
- "keep_last_n": 2,
71
- "arabic_mode": "curated",
72
- "arabic_cache": null,
73
- "p_arabic": 0.2,
74
- "p_egyptian": 0.2,
75
- "p_mixed": 0.05,
76
- "p_subject": 0.4,
77
- "p_drop_text": 0.1,
78
- "transparent_weight": 0.5,
79
- "label_smoothing": 0.0,
80
- "contrastive_weight": 0.05,
81
- "contrastive_temp": 0.07,
82
- "silma_distill_weight": 0.0,
83
- "silma_model": "silma-ai/silma-embedding-matryoshka-v0.1",
84
- "use_growth": false,
85
- "growth": {
86
- "enabled": false,
87
- "first_grow": 500,
88
- "grow_check_every": 500,
89
- "trial_steps": 300,
90
- "grow_neurons": 64,
91
- "eps_per_m": 0.004,
92
- "cooldown": 200,
93
- "prune_every": 1000,
94
- "prune_util_frac": 0.2,
95
- "prune_max_frac": 0.02,
96
- "prune_juvenile": 400,
97
- "min_neurons": 96,
98
- "max_params": 40000000,
99
- "val_batches": 4,
100
- "max_trials": 200
101
- },
102
- "compile": false,
103
- "init_from": "/kaggle/working/hf_checkpoint/checkpoints/step_0017000",
104
- "hub_repo": "nomeda-lab/pixel-art-tiny-adaptive",
105
- "hub_private": false,
106
- "hub_every": 1000,
107
- "hub_keep_best": true
108
- },
109
- "step": 4000,
110
- "active_params": 24496832,
111
- "allocated_params": 24496832,
112
- "ffn_widths": [
113
- 128,
114
- 128,
115
- 128,
116
- 128,
117
- 128,
118
- 128,
119
- 128,
120
- 128,
121
- 128,
122
- 128,
123
- 128,
124
- 128
125
- ],
126
- "metrics": {
127
- "last_train": {
128
- "token_accuracy": 0.71417236328125,
129
- "opaque_accuracy": 0.5608060359954834,
130
- "transparent_accuracy": 0.9498374462127686,
131
- "exact_match": 0.0,
132
- "non_empty_rate": 1.0
133
- }
134
- }
135
- }