Upload part1/results/results.json with huggingface_hub
Browse files- part1/results/results.json +46 -46
part1/results/results.json
CHANGED
|
@@ -2,8 +2,8 @@
|
|
| 2 |
"Transformer": {
|
| 3 |
"checkpoint": "models/part1/Transformer_seed0.pt",
|
| 4 |
"output": "output/part1/test_decrypt_Transformer.jsonl",
|
| 5 |
-
"peak_inference_memory_gib": 0.
|
| 6 |
-
"parameters":
|
| 7 |
"flops_per_sequence": 7659585536,
|
| 8 |
"flops_per_token": 14960128,
|
| 9 |
"flops_seq_len": 512,
|
|
@@ -17,21 +17,21 @@
|
|
| 17 |
"Transformer",
|
| 18 |
"Transformer"
|
| 19 |
],
|
| 20 |
-
"char_accuracy": 0.
|
| 21 |
-
"word_accuracy": 0.
|
| 22 |
-
"precision": 0.
|
| 23 |
-
"recall": 0.
|
| 24 |
-
"f1": 0.
|
| 25 |
-
"micro_f1": 0.
|
| 26 |
-
"levenshtein":
|
| 27 |
-
"levenshtein_normalised": 0.
|
| 28 |
-
"exact_match": 0.
|
| 29 |
},
|
| 30 |
"Mamba": {
|
| 31 |
"checkpoint": "models/part1/Mamba_seed0.pt",
|
| 32 |
"output": "output/part1/test_decrypt_Mamba.jsonl",
|
| 33 |
-
"peak_inference_memory_gib": 0.
|
| 34 |
-
"parameters":
|
| 35 |
"flops_per_sequence": 7691042816,
|
| 36 |
"flops_per_token": 15021568,
|
| 37 |
"flops_seq_len": 512,
|
|
@@ -53,21 +53,21 @@
|
|
| 53 |
"Mamba",
|
| 54 |
"Mamba"
|
| 55 |
],
|
| 56 |
-
"char_accuracy": 0.
|
| 57 |
-
"word_accuracy": 0.
|
| 58 |
-
"precision": 0.
|
| 59 |
-
"recall": 0.
|
| 60 |
-
"f1": 0.
|
| 61 |
-
"micro_f1": 0.
|
| 62 |
-
"levenshtein":
|
| 63 |
-
"levenshtein_normalised": 0.
|
| 64 |
-
"exact_match": 0.
|
| 65 |
},
|
| 66 |
"DeltaNet": {
|
| 67 |
"checkpoint": "models/part1/DeltaNet_seed0.pt",
|
| 68 |
"output": "output/part1/test_decrypt_DeltaNet.jsonl",
|
| 69 |
-
"peak_inference_memory_gib": 0.
|
| 70 |
-
"parameters":
|
| 71 |
"flops_per_sequence": 7579194624,
|
| 72 |
"flops_per_token": 14803114,
|
| 73 |
"flops_seq_len": 512,
|
|
@@ -81,23 +81,23 @@
|
|
| 81 |
"DeltaNet",
|
| 82 |
"DeltaNet"
|
| 83 |
],
|
| 84 |
-
"char_accuracy": 0.
|
| 85 |
-
"word_accuracy": 0.
|
| 86 |
-
"precision": 0.
|
| 87 |
-
"recall": 0.
|
| 88 |
-
"f1": 0.
|
| 89 |
-
"micro_f1": 0.
|
| 90 |
-
"levenshtein":
|
| 91 |
-
"levenshtein_normalised": 0.
|
| 92 |
-
"exact_match": 0.
|
| 93 |
},
|
| 94 |
"MambaDeltaNetHybrid": {
|
| 95 |
"checkpoint": "models/part1/MambaDeltaNetHybrid_seed0.pt",
|
| 96 |
"output": "output/part1/test_decrypt.jsonl",
|
| 97 |
-
"peak_inference_memory_gib": 0.
|
| 98 |
-
"parameters":
|
| 99 |
-
"flops_per_sequence":
|
| 100 |
-
"flops_per_token":
|
| 101 |
"flops_seq_len": 512,
|
| 102 |
"layers": [
|
| 103 |
"Mamba",
|
|
@@ -111,14 +111,14 @@
|
|
| 111 |
"Mamba",
|
| 112 |
"DeltaNet"
|
| 113 |
],
|
| 114 |
-
"char_accuracy": 0.
|
| 115 |
-
"word_accuracy": 0.
|
| 116 |
-
"precision": 0.
|
| 117 |
-
"recall": 0.
|
| 118 |
-
"f1": 0.
|
| 119 |
-
"micro_f1": 0.
|
| 120 |
-
"levenshtein":
|
| 121 |
-
"levenshtein_normalised": 0.
|
| 122 |
-
"exact_match": 0.
|
| 123 |
}
|
| 124 |
}
|
|
|
|
| 2 |
"Transformer": {
|
| 3 |
"checkpoint": "models/part1/Transformer_seed0.pt",
|
| 4 |
"output": "output/part1/test_decrypt_Transformer.jsonl",
|
| 5 |
+
"peak_inference_memory_gib": 0.4342060089111328,
|
| 6 |
+
"parameters": 10689280,
|
| 7 |
"flops_per_sequence": 7659585536,
|
| 8 |
"flops_per_token": 14960128,
|
| 9 |
"flops_seq_len": 512,
|
|
|
|
| 17 |
"Transformer",
|
| 18 |
"Transformer"
|
| 19 |
],
|
| 20 |
+
"char_accuracy": 0.17969354021132686,
|
| 21 |
+
"word_accuracy": 0.009141622273595336,
|
| 22 |
+
"precision": 0.6888934470389246,
|
| 23 |
+
"recall": 0.04929379338330081,
|
| 24 |
+
"f1": 0.03531646835514501,
|
| 25 |
+
"micro_f1": 0.17969354021132686,
|
| 26 |
+
"levenshtein": 728.7215477996965,
|
| 27 |
+
"levenshtein_normalised": 0.4915511741989003,
|
| 28 |
+
"exact_match": 0.26707132018209406
|
| 29 |
},
|
| 30 |
"Mamba": {
|
| 31 |
"checkpoint": "models/part1/Mamba_seed0.pt",
|
| 32 |
"output": "output/part1/test_decrypt_Mamba.jsonl",
|
| 33 |
+
"peak_inference_memory_gib": 0.5600619316101074,
|
| 34 |
+
"parameters": 11270912,
|
| 35 |
"flops_per_sequence": 7691042816,
|
| 36 |
"flops_per_token": 15021568,
|
| 37 |
"flops_seq_len": 512,
|
|
|
|
| 53 |
"Mamba",
|
| 54 |
"Mamba"
|
| 55 |
],
|
| 56 |
+
"char_accuracy": 0.18583690181373813,
|
| 57 |
+
"word_accuracy": 0.014096894160217107,
|
| 58 |
+
"precision": 0.7343933451703206,
|
| 59 |
+
"recall": 0.056084312826216,
|
| 60 |
+
"f1": 0.0483374060263977,
|
| 61 |
+
"micro_f1": 0.18583690181373813,
|
| 62 |
+
"levenshtein": 723.4628224582701,
|
| 63 |
+
"levenshtein_normalised": 0.41525410838729937,
|
| 64 |
+
"exact_match": 0.3543247344461305
|
| 65 |
},
|
| 66 |
"DeltaNet": {
|
| 67 |
"checkpoint": "models/part1/DeltaNet_seed0.pt",
|
| 68 |
"output": "output/part1/test_decrypt_DeltaNet.jsonl",
|
| 69 |
+
"peak_inference_memory_gib": 0.5755305290222168,
|
| 70 |
+
"parameters": 10728704,
|
| 71 |
"flops_per_sequence": 7579194624,
|
| 72 |
"flops_per_token": 14803114,
|
| 73 |
"flops_seq_len": 512,
|
|
|
|
| 81 |
"DeltaNet",
|
| 82 |
"DeltaNet"
|
| 83 |
],
|
| 84 |
+
"char_accuracy": 0.17969183348494752,
|
| 85 |
+
"word_accuracy": 0.009905518142526886,
|
| 86 |
+
"precision": 0.6773651224459333,
|
| 87 |
+
"recall": 0.049651820813145724,
|
| 88 |
+
"f1": 0.036024966746743124,
|
| 89 |
+
"micro_f1": 0.17969183348494752,
|
| 90 |
+
"levenshtein": 728.695751138088,
|
| 91 |
+
"levenshtein_normalised": 0.48123425796128627,
|
| 92 |
+
"exact_match": 0.30045523520485584
|
| 93 |
},
|
| 94 |
"MambaDeltaNetHybrid": {
|
| 95 |
"checkpoint": "models/part1/MambaDeltaNetHybrid_seed0.pt",
|
| 96 |
"output": "output/part1/test_decrypt.jsonl",
|
| 97 |
+
"peak_inference_memory_gib": 0.621424674987793,
|
| 98 |
+
"parameters": 10494016,
|
| 99 |
+
"flops_per_sequence": 7224775968,
|
| 100 |
+
"flops_per_token": 14110890,
|
| 101 |
"flops_seq_len": 512,
|
| 102 |
"layers": [
|
| 103 |
"Mamba",
|
|
|
|
| 111 |
"Mamba",
|
| 112 |
"DeltaNet"
|
| 113 |
],
|
| 114 |
+
"char_accuracy": 0.1854366744777844,
|
| 115 |
+
"word_accuracy": 0.012825409588903408,
|
| 116 |
+
"precision": 0.6949363139374243,
|
| 117 |
+
"recall": 0.055777903053427606,
|
| 118 |
+
"f1": 0.047823507802699707,
|
| 119 |
+
"micro_f1": 0.1854366744777844,
|
| 120 |
+
"levenshtein": 723.6077389984825,
|
| 121 |
+
"levenshtein_normalised": 0.4222130271023305,
|
| 122 |
+
"exact_match": 0.33611532625189683
|
| 123 |
}
|
| 124 |
}
|