Add matched T0-cap seed 2
Browse files
t0cap_matched_seed2/config.yaml
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
run_name: xsum_t0cap_matched_s2
|
| 2 |
+
seed: 2
|
| 3 |
+
data:
|
| 4 |
+
dataset: xsum
|
| 5 |
+
root: data
|
| 6 |
+
aux_path: ''
|
| 7 |
+
aux_batch: 128
|
| 8 |
+
model:
|
| 9 |
+
arch: ncm
|
| 10 |
+
d: 384
|
| 11 |
+
in_vocab_size: 50304
|
| 12 |
+
text_tokenizer: gpt2
|
| 13 |
+
context_length: 48
|
| 14 |
+
pad_id: 50257
|
| 15 |
+
eos_id: 50256
|
| 16 |
+
encoder_blocks: 5
|
| 17 |
+
decoder_blocks: 3
|
| 18 |
+
n_heads: 6
|
| 19 |
+
dropout: 0.1
|
| 20 |
+
levels:
|
| 21 |
+
- 4
|
| 22 |
+
- 6
|
| 23 |
+
- 10
|
| 24 |
+
- 14
|
| 25 |
+
- 20
|
| 26 |
+
- 26
|
| 27 |
+
- 32
|
| 28 |
+
- 40
|
| 29 |
+
- 48
|
| 30 |
+
codebook_size: 4096
|
| 31 |
+
pool_self_attn: false
|
| 32 |
+
attn_prior: true
|
| 33 |
+
prior_sharpness: 4.0
|
| 34 |
+
qk_norm: true
|
| 35 |
+
quant_norm: true
|
| 36 |
+
query_init_std: 0.02
|
| 37 |
+
final_level_identity: true
|
| 38 |
+
ste: per_level
|
| 39 |
+
m: 3
|
| 40 |
+
vq:
|
| 41 |
+
decay: 0.99
|
| 42 |
+
epsilon: 1.0e-05
|
| 43 |
+
commitment: 0.25
|
| 44 |
+
restart_threshold: 0.0
|
| 45 |
+
nested_dropout:
|
| 46 |
+
prob: 0.5
|
| 47 |
+
min_keep: 1
|
| 48 |
+
pre_quant_dropout: 0.25
|
| 49 |
+
cascade_norm: true
|
| 50 |
+
heads:
|
| 51 |
+
paraphrase:
|
| 52 |
+
enabled: false
|
| 53 |
+
levels:
|
| 54 |
+
- 1
|
| 55 |
+
- 2
|
| 56 |
+
- 3
|
| 57 |
+
weight: 0.2
|
| 58 |
+
temperature: 0.1
|
| 59 |
+
proj_dim: 256
|
| 60 |
+
summary:
|
| 61 |
+
enabled: false
|
| 62 |
+
levels:
|
| 63 |
+
- 1
|
| 64 |
+
- 2
|
| 65 |
+
- 3
|
| 66 |
+
weight: 0.2
|
| 67 |
+
blocks: 2
|
| 68 |
+
n_heads: 6
|
| 69 |
+
max_len: 32
|
| 70 |
+
train:
|
| 71 |
+
steps: 30000
|
| 72 |
+
micro_batch: 256
|
| 73 |
+
grad_accum: 1
|
| 74 |
+
lr: 0.0005
|
| 75 |
+
lr_floor: 5.0e-05
|
| 76 |
+
warmup: 1000
|
| 77 |
+
betas:
|
| 78 |
+
- 0.9
|
| 79 |
+
- 0.95
|
| 80 |
+
weight_decay: 0.01
|
| 81 |
+
grad_clip: 1.0
|
| 82 |
+
max_nonfinite_steps: 200
|
| 83 |
+
runtime:
|
| 84 |
+
precision: bf16
|
| 85 |
+
compile: false
|
| 86 |
+
ckpt_every: 1000
|
| 87 |
+
keep_ckpts: 2
|
| 88 |
+
log_every: 50
|
| 89 |
+
eval_every: 5000
|
| 90 |
+
eval_examples: 2000
|
| 91 |
+
eval_batch_size: 64
|
| 92 |
+
wandb: false
|
| 93 |
+
wandb_project: cruxtok
|
| 94 |
+
output_root: /workspace/runs/t0match_s2
|
t0cap_matched_seed2/eval.jsonl
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"step": 5000, "accuracy": [52.546875, 56.32500076293945, 60.665626525878906, 65.09062194824219, 71.72187805175781, 78.42708587646484, 86.4281234741211, 90.57707977294922, 96.51249694824219], "accuracy_text": [16.18625259399414, 22.847375869750977, 30.32520294189453, 38.08943176269531, 49.84478759765625, 61.733184814453125, 75.92941284179688, 83.28529357910156, 93.8137435913086], "ce": [3.6121532917022705, 2.895202159881592, 2.3713901042938232, 1.959071397781372, 1.4786038398742676, 1.0925371646881104, 0.6985892057418823, 0.5180878639221191, 0.27049994468688965], "utilization": [31.3232421875, 47.802734375, 48.7548828125, 18.4326171875, 19.4091796875, 15.8935546875, 13.6962890625, 5.6396484375, 27.5634765625], "perplexity": [1016.4212705395477, 1197.276645201566, 1216.2493135384555, 232.571736806032, 132.1224284533805, 112.66052152530413, 96.73605609028141, 36.880517116282824, 140.21463590290423]}
|
| 2 |
+
{"step": 10000, "accuracy": [54.42916488647461, 59.13228988647461, 64.23542022705078, 68.9781265258789, 75.51875305175781, 81.96770477294922, 89.51457977294922, 93.25833129882812, 98.02812194824219], "accuracy_text": [19.587953567504883, 27.78085708618164, 36.64264678955078, 44.9852180480957, 56.57612609863281, 68.013671875, 81.40058898925781, 88.04138946533203, 96.5022201538086], "ce": [3.4197893142700195, 2.679182529449463, 2.132370948791504, 1.7026352882385254, 1.2396855354309082, 0.8711570501327515, 0.5214420557022095, 0.3606824576854706, 0.16408094763755798], "utilization": [33.7646484375, 57.91015625, 51.2451171875, 23.14453125, 26.6845703125, 22.4853515625, 22.265625, 9.6923828125, 52.294921875], "perplexity": [1014.4936396788137, 1759.3146609690473, 1410.4037077131263, 287.2940479207108, 218.1548799152468, 210.90846353185634, 182.50395619819912, 70.61086505527453, 238.63872332685193]}
|
| 3 |
+
{"step": 15000, "accuracy": [54.365623474121094, 60.0625, 66.28333282470703, 71.74583435058594, 79.10520935058594, 85.50521087646484, 92.33125305175781, 95.35416412353516, 98.52395629882812], "accuracy_text": [19.3514404296875, 29.425350189208984, 40.2420539855957, 49.88913345336914, 62.939762115478516, 74.28861999511719, 86.39689636230469, 91.7590560913086, 97.38174438476562], "ce": [3.4412662982940674, 2.6051223278045654, 1.9880985021591187, 1.5232709646224976, 1.0509891510009766, 0.6975536942481995, 0.3914955258369446, 0.26544201374053955, 0.13083402812480927], "utilization": [26.5380859375, 59.08203125, 53.5400390625, 26.123046875, 32.0556640625, 27.6123046875, 28.857421875, 12.158203125, 65.283203125], "perplexity": [861.5374197486369, 1825.0639874088695, 1467.931820212721, 328.1983667343168, 267.5534151286317, 267.64605381447143, 240.525070068357, 118.98993457102834, 296.44540204996525]}
|
| 4 |
+
{"step": 20000, "accuracy": [54.66145706176758, 61.45520782470703, 67.75521087646484, 73.73124694824219, 80.90520477294922, 87.29166412353516, 93.51667022705078, 96.08333587646484, 98.70207977294922], "accuracy_text": [19.733924865722656, 31.729490280151367, 42.83259582519531, 53.412784576416016, 66.13082122802734, 77.4593505859375, 88.4996337890625, 93.05247497558594, 97.69770812988281], "ce": [3.3990602493286133, 2.511897563934326, 1.8850430250167847, 1.411331057548523, 0.9507760405540466, 0.613806962966919, 0.3395535945892334, 0.2309127002954483, 0.11868786811828613], "utilization": [24.169921875, 58.3740234375, 55.37109375, 29.150390625, 36.9873046875, 31.4208984375, 33.0322265625, 13.5498046875, 72.2900390625], "perplexity": [795.4419384850265, 1812.3634515481463, 1513.0759198741043, 417.5274936608772, 455.4391295976151, 384.8333130715301, 381.98165407894135, 190.58807828527077, 393.9319462992754]}
|
| 5 |
+
{"step": 25000, "accuracy": [54.30729293823242, 61.259376525878906, 68.79582977294922, 75.36666870117188, 83.10208129882812, 89.23332977294922, 94.75208282470703, 96.83333587646484, 98.75521087646484], "accuracy_text": [19.17036247253418, 31.48558807373047, 44.726531982421875, 56.31744384765625, 70.02587127685547, 80.90354919433594, 90.69105529785156, 94.38285064697266, 97.79194641113281], "ce": [3.4056575298309326, 2.478764772415161, 1.7969239950180054, 1.3026455640792847, 0.838316798210144, 0.5269913077354431, 0.28873831033706665, 0.2023913413286209, 0.11513102054595947], "utilization": [23.14453125, 58.3251953125, 56.4453125, 32.470703125, 40.8203125, 33.3740234375, 36.376953125, 14.1845703125, 75.0244140625], "perplexity": [778.7425562547581, 1841.5677681162717, 1475.6502501153702, 478.82022162225167, 542.791761448882, 479.41353406348196, 478.3592387367867, 221.6142004022529, 540.821043693431]}
|
| 6 |
+
{"step": 30000, "accuracy": [54.62291717529297, 61.74687576293945, 69.3343734741211, 76.10208129882812, 83.984375, 90.08958435058594, 95.29895782470703, 97.19271087646484, 98.8218765258789], "accuracy_text": [19.637842178344727, 32.2985954284668, 45.64301681518555, 57.618255615234375, 71.59275817871094, 82.42054748535156, 91.66112518310547, 95.02032470703125, 97.91020202636719], "ce": [3.3780367374420166, 2.4394912719726562, 1.7586508989334106, 1.2620974779129028, 0.7964036464691162, 0.490298867225647, 0.26893094182014465, 0.18828432261943817, 0.11162012815475464], "utilization": [22.607421875, 57.8857421875, 57.763671875, 34.716796875, 43.6767578125, 34.6923828125, 37.6953125, 14.9169921875, 77.2216796875], "perplexity": [759.7373012558278, 1837.8535250705347, 1500.6096925465667, 543.3451487036333, 597.1381506348529, 538.3679620236708, 527.0756032636492, 264.77421634815744, 557.677065850416]}
|
t0cap_matched_seed2/tokenizer.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:767cf7bc64acb6c48cd35c4bc5fe13862624704fdc659a820b52871d93013ab4
|
| 3 |
+
size 349367307
|