Invictus22 commited on
Commit
1f8f95e
·
verified ·
1 Parent(s): f02694e

Upload part1/results/results.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. part1/results/results.json +46 -46
part1/results/results.json CHANGED
@@ -2,8 +2,8 @@
2
  "Transformer": {
3
  "checkpoint": "models/part1/Transformer_seed0.pt",
4
  "output": "output/part1/test_decrypt_Transformer.jsonl",
5
- "peak_inference_memory_gib": 0.08918523788452148,
6
- "parameters": 7527680,
7
  "flops_per_sequence": 7659585536,
8
  "flops_per_token": 14960128,
9
  "flops_seq_len": 512,
@@ -17,21 +17,21 @@
17
  "Transformer",
18
  "Transformer"
19
  ],
20
- "char_accuracy": 0.1698593828136067,
21
- "word_accuracy": 0.00013066639863302845,
22
- "precision": 0.18118996347768976,
23
- "recall": 0.03803020853334762,
24
- "f1": 0.012851883099290851,
25
- "micro_f1": 0.1698593828136067,
26
- "levenshtein": 737.8839150227618,
27
- "levenshtein_normalised": 0.8032517207278034,
28
- "exact_match": 0.017450682852807285
29
  },
30
  "Mamba": {
31
  "checkpoint": "models/part1/Mamba_seed0.pt",
32
  "output": "output/part1/test_decrypt_Mamba.jsonl",
33
- "peak_inference_memory_gib": 0.11726140975952148,
34
- "parameters": 8109312,
35
  "flops_per_sequence": 7691042816,
36
  "flops_per_token": 15021568,
37
  "flops_seq_len": 512,
@@ -53,21 +53,21 @@
53
  "Mamba",
54
  "Mamba"
55
  ],
56
- "char_accuracy": 0.169717724524122,
57
- "word_accuracy": 0.00023620464368278218,
58
- "precision": 0.138869310222489,
59
- "recall": 0.03795232455429236,
60
- "f1": 0.012666634132577913,
61
- "micro_f1": 0.169717724524122,
62
- "levenshtein": 737.9742033383915,
63
- "levenshtein_normalised": 0.8013879246329688,
64
- "exact_match": 0.01896813353566009
65
  },
66
  "DeltaNet": {
67
  "checkpoint": "models/part1/DeltaNet_seed0.pt",
68
  "output": "output/part1/test_decrypt_DeltaNet.jsonl",
69
- "peak_inference_memory_gib": 0.10111761093139648,
70
- "parameters": 7567104,
71
  "flops_per_sequence": 7579194624,
72
  "flops_per_token": 14803114,
73
  "flops_seq_len": 512,
@@ -81,23 +81,23 @@
81
  "DeltaNet",
82
  "DeltaNet"
83
  ],
84
- "char_accuracy": 0.1697723397682607,
85
- "word_accuracy": 0.0002613327972660569,
86
- "precision": 0.13768460842627803,
87
- "recall": 0.037968041644930124,
88
- "f1": 0.01270526169443308,
89
- "micro_f1": 0.1697723397682607,
90
- "levenshtein": 737.9362670713202,
91
- "levenshtein_normalised": 0.8017135174885176,
92
- "exact_match": 0.018209408194233688
93
  },
94
  "MambaDeltaNetHybrid": {
95
  "checkpoint": "models/part1/MambaDeltaNetHybrid_seed0.pt",
96
  "output": "output/part1/test_decrypt.jsonl",
97
- "peak_inference_memory_gib": 0.11436843872070312,
98
- "parameters": 7332416,
99
- "flops_per_sequence": 7140889888,
100
- "flops_per_token": 13947050,
101
  "flops_seq_len": 512,
102
  "layers": [
103
  "Mamba",
@@ -111,14 +111,14 @@
111
  "Mamba",
112
  "DeltaNet"
113
  ],
114
- "char_accuracy": 0.16975100568851903,
115
- "word_accuracy": 0.00026635842798271184,
116
- "precision": 0.14123473405377648,
117
- "recall": 0.03795450431851667,
118
- "f1": 0.012664626101314877,
119
- "micro_f1": 0.16975100568851903,
120
- "levenshtein": 737.9817905918057,
121
- "levenshtein_normalised": 0.8032012952368074,
122
- "exact_match": 0.017450682852807285
123
  }
124
  }
 
2
  "Transformer": {
3
  "checkpoint": "models/part1/Transformer_seed0.pt",
4
  "output": "output/part1/test_decrypt_Transformer.jsonl",
5
+ "peak_inference_memory_gib": 0.4342060089111328,
6
+ "parameters": 10689280,
7
  "flops_per_sequence": 7659585536,
8
  "flops_per_token": 14960128,
9
  "flops_seq_len": 512,
 
17
  "Transformer",
18
  "Transformer"
19
  ],
20
+ "char_accuracy": 0.17969354021132686,
21
+ "word_accuracy": 0.009141622273595336,
22
+ "precision": 0.6888934470389246,
23
+ "recall": 0.04929379338330081,
24
+ "f1": 0.03531646835514501,
25
+ "micro_f1": 0.17969354021132686,
26
+ "levenshtein": 728.7215477996965,
27
+ "levenshtein_normalised": 0.4915511741989003,
28
+ "exact_match": 0.26707132018209406
29
  },
30
  "Mamba": {
31
  "checkpoint": "models/part1/Mamba_seed0.pt",
32
  "output": "output/part1/test_decrypt_Mamba.jsonl",
33
+ "peak_inference_memory_gib": 0.5600619316101074,
34
+ "parameters": 11270912,
35
  "flops_per_sequence": 7691042816,
36
  "flops_per_token": 15021568,
37
  "flops_seq_len": 512,
 
53
  "Mamba",
54
  "Mamba"
55
  ],
56
+ "char_accuracy": 0.18583690181373813,
57
+ "word_accuracy": 0.014096894160217107,
58
+ "precision": 0.7343933451703206,
59
+ "recall": 0.056084312826216,
60
+ "f1": 0.0483374060263977,
61
+ "micro_f1": 0.18583690181373813,
62
+ "levenshtein": 723.4628224582701,
63
+ "levenshtein_normalised": 0.41525410838729937,
64
+ "exact_match": 0.3543247344461305
65
  },
66
  "DeltaNet": {
67
  "checkpoint": "models/part1/DeltaNet_seed0.pt",
68
  "output": "output/part1/test_decrypt_DeltaNet.jsonl",
69
+ "peak_inference_memory_gib": 0.5755305290222168,
70
+ "parameters": 10728704,
71
  "flops_per_sequence": 7579194624,
72
  "flops_per_token": 14803114,
73
  "flops_seq_len": 512,
 
81
  "DeltaNet",
82
  "DeltaNet"
83
  ],
84
+ "char_accuracy": 0.17969183348494752,
85
+ "word_accuracy": 0.009905518142526886,
86
+ "precision": 0.6773651224459333,
87
+ "recall": 0.049651820813145724,
88
+ "f1": 0.036024966746743124,
89
+ "micro_f1": 0.17969183348494752,
90
+ "levenshtein": 728.695751138088,
91
+ "levenshtein_normalised": 0.48123425796128627,
92
+ "exact_match": 0.30045523520485584
93
  },
94
  "MambaDeltaNetHybrid": {
95
  "checkpoint": "models/part1/MambaDeltaNetHybrid_seed0.pt",
96
  "output": "output/part1/test_decrypt.jsonl",
97
+ "peak_inference_memory_gib": 0.621424674987793,
98
+ "parameters": 10494016,
99
+ "flops_per_sequence": 7224775968,
100
+ "flops_per_token": 14110890,
101
  "flops_seq_len": 512,
102
  "layers": [
103
  "Mamba",
 
111
  "Mamba",
112
  "DeltaNet"
113
  ],
114
+ "char_accuracy": 0.1854366744777844,
115
+ "word_accuracy": 0.012825409588903408,
116
+ "precision": 0.6949363139374243,
117
+ "recall": 0.055777903053427606,
118
+ "f1": 0.047823507802699707,
119
+ "micro_f1": 0.1854366744777844,
120
+ "levenshtein": 723.6077389984825,
121
+ "levenshtein_normalised": 0.4222130271023305,
122
+ "exact_match": 0.33611532625189683
123
  }
124
  }