File size: 9,055 Bytes
2c87024 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 | {
"format": "vimeml_public_evaluation_v1",
"bundle_manifest_sha256": "4284cdc5491513cc817633081677b363b14f59c0b4ffb78cdf7fcfb221f71b3c",
"coreml_manifest_sha256": "a9d516233fa57aaadd09477e2e909760972ff49b5577919f8a441c489f5c2245",
"training": {
"prediction_targets": 573295237,
"data_passes": 1.0,
"full_validation": {
"loss": 4.541570895102848,
"prediction_pairs": 5890326,
"perplexity": 93.83809420004854
},
"test_used_for_selection": false
},
"ranking": {
"dev": {
"original_order": {
"cases": 137,
"top1_correct": 111,
"top1_accuracy": 0.8102189781021898,
"top5_correct": 130,
"top5_accuracy": 0.948905109489051,
"candidate_pool_covered": 136,
"candidate_pool_recall": 0.9927007299270073,
"covered_top1_accuracy": 0.8161764705882353,
"mean_min_cer": 0.04746998743349109,
"mrr": 0.8745925911071898,
"corrected_vs_azookey": 0,
"regressed_vs_azookey": 0,
"fallback_cases": 0
},
"fp32_lm": {
"cases": 137,
"top1_correct": 122,
"top1_accuracy": 0.8905109489051095,
"top5_correct": 134,
"top5_accuracy": 0.9781021897810219,
"candidate_pool_covered": 136,
"candidate_pool_recall": 0.9927007299270073,
"covered_top1_accuracy": 0.8970588235294118,
"mean_min_cer": 0.03045798483754688,
"mrr": 0.935349322210636,
"corrected_vs_azookey": 15,
"regressed_vs_azookey": 4,
"fallback_cases": 0
},
"int8_lm": {
"cases": 137,
"top1_correct": 122,
"top1_accuracy": 0.8905109489051095,
"top5_correct": 134,
"top5_accuracy": 0.9781021897810219,
"candidate_pool_covered": 136,
"candidate_pool_recall": 0.9927007299270073,
"covered_top1_accuracy": 0.8970588235294118,
"mean_min_cer": 0.03045798483754688,
"mrr": 0.935349322210636,
"corrected_vs_azookey": 15,
"regressed_vs_azookey": 4,
"fallback_cases": 0
},
"windows_fp32_comparison": {
"order_changed_cases": 119,
"top1_changed_cases": 0,
"max_score_sum_abs": 1.9853296279907227,
"mean_score_sum_abs": 0.20199488408841776
},
"original_report_comparison_baseline": "conservative Core ML",
"original_report_comparison": {
"order_changed_cases": 118,
"top1_changed_cases": 0,
"max_score_sum_abs": 1.994759202003479,
"mean_score_sum_abs": 0.2013680215637894
},
"benchmark_source_sha256": "a945f2117570530ef73b1dd4ef53b9dc69b07fa69ac98a6d67c9268b540eafa8",
"candidate_export_sha256": "ab192509dce1cb4f4b20273e623f7bfd4237b1c3691950438e6921b8a510ae21"
},
"ajimee": {
"original_order": {
"cases": 200,
"top1_correct": 87,
"top1_accuracy": 0.435,
"top5_correct": 143,
"top5_accuracy": 0.715,
"candidate_pool_covered": 162,
"candidate_pool_recall": 0.81,
"covered_top1_accuracy": 0.5370370370370371,
"mean_min_cer": 0.0821630611981915,
"mrr": 0.554232017982018,
"corrected_vs_azookey": 0,
"regressed_vs_azookey": 0,
"fallback_cases": 0
},
"fp32_lm": {
"cases": 200,
"top1_correct": 124,
"top1_accuracy": 0.62,
"top5_correct": 151,
"top5_accuracy": 0.755,
"candidate_pool_covered": 162,
"candidate_pool_recall": 0.81,
"covered_top1_accuracy": 0.7654320987654321,
"mean_min_cer": 0.05657040945427151,
"mrr": 0.684061974789916,
"corrected_vs_azookey": 48,
"regressed_vs_azookey": 11,
"fallback_cases": 0
},
"int8_lm": {
"cases": 200,
"top1_correct": 124,
"top1_accuracy": 0.62,
"top5_correct": 151,
"top5_accuracy": 0.755,
"candidate_pool_covered": 162,
"candidate_pool_recall": 0.81,
"covered_top1_accuracy": 0.7654320987654321,
"mean_min_cer": 0.05657040945427151,
"mrr": 0.6841512605042016,
"corrected_vs_azookey": 48,
"regressed_vs_azookey": 11,
"fallback_cases": 0
},
"windows_fp32_comparison": {
"order_changed_cases": 163,
"top1_changed_cases": 2,
"max_score_sum_abs": 2.6928324103355408,
"mean_score_sum_abs": 0.33380034679328247
},
"original_report_comparison_baseline": "Windows FP32",
"original_report_comparison": {
"order_changed_cases": 163,
"top1_changed_cases": 2,
"max_score_sum_abs": 2.6928324103355408,
"mean_score_sum_abs": 0.33380034679328247
},
"benchmark_source_sha256": "e9eb668fd6aa14b1e26436f429b5550108af0a1dfd443b8cea0bcb3ab3028fca",
"candidate_export_sha256": "6b532af2e7b1536ded98e9d75cbba4fd7338b4204987fc4dafb16df7edfdf223"
}
},
"alignment": {
"strict_passed": false,
"max_logit_abs": 3.6601905822753906,
"invariants": {
"right_pad": {
"max_abs": 2.8133392333984375e-05,
"mean_abs": 2.5544544911072975e-06,
"rmse": 3.420821628891477e-06,
"outside_tolerance": 0,
"values": 114688
},
"causal": {
"max_abs": 0.0,
"mean_abs": 0.0,
"rmse": 0.0,
"outside_tolerance": 0,
"values": 262144
}
}
},
"distribution": {
"original_report": {
"model": "artifacts/deployment/tiny-ja-v1-conservative-int8-b32-v1",
"compute_units": "CPU_ONLY",
"benchmark_text": {
"kl_coreml": {
"mean": 0.005891128288349137,
"max": 0.025890197604894638
},
"kl_simulated": {
"mean": 0.005977873008232564,
"max": 0.028363749384880066
},
"nll_fp32": {
"mean": 5.566977231502533,
"max": 8.531479835510254
},
"nll_coreml": {
"mean": 5.572893209457398,
"max": 8.633211135864258
},
"top1_agreement": {
"mean": 0.943656535744667,
"max": 1.0
},
"max_logprob_diff": {
"mean": 0.9077084493637085,
"max": 3.5512266159057617
}
},
"random_tokens": {
"kl_coreml": {
"mean": 0.015501508116722107,
"max": 0.01807662844657898
},
"kl_simulated": {
"mean": 0.015804098173975944,
"max": 0.01901794597506523
},
"nll_fp32": {
"mean": 11.800455474853516,
"max": 12.100545883178711
},
"nll_coreml": {
"mean": 11.795885848999024,
"max": 12.123295783996582
},
"top1_agreement": {
"mean": 0.9140625,
"max": 0.9296875
},
"max_logprob_diff": {
"mean": 2.206966495513916,
"max": 2.564694404602051
}
}
},
"aggregation": "Mean of per-example position means; not corpus token-weighted.",
"content_token_limit": 127,
"truncation_count_recorded": false,
"original_report_has_full_identity_metadata": false
},
"source_reports_sha256": {
"outputs/deployment/conservative-int8-b32-dev-v1/metrics.json": "839e220c3024ec9798c921fb39d8862c5f874889c9dd9c2066cb339894658329",
"outputs/deployment/conservative-int8-b32-dev-v1/scores.jsonl": "f00a6b56b96e6be1db3ba7c58abe53bc91e7614c2f7f7959c27b0a8cce8db936",
"outputs/ime-eval/tiny-ja-v1-dev-v2-scores/manifest.json": "16bbd3b5b921e54896cfdca880d77c78fedd76b2aa8956fbaaa4c45bd4a4030e",
"outputs/ime-eval/tiny-ja-v1-dev-v2-scores/scores.jsonl": "2710b88428b5f73c6aeac5d45d442fa0d9bca1e038db2928c6059be993f3cfcb",
"outputs/deployment/conservative-dev-v3/metrics.json": "387d80e1b52a2c8dbcbb2196471ae548ae869b1e3f259dc1ee972c592e6b4863",
"outputs/deployment/conservative-dev-v3/scores.jsonl": "bb4fa9f52a074144a3cb432777ec6c94ad07637cdecd5b2126ad5d64a4e6a1d2",
"outputs/deployment/conservative-int8-b32-ajimee-v1/metrics.json": "3b59e396173328c78e8164f809da24764474685a2bd35765457fac5df644aac5",
"outputs/deployment/conservative-int8-b32-ajimee-v1/scores.jsonl": "9daf557818ab0aef12fb74e68b8cd4ae502806650d3d7e0cd23eeb2e1499e89e",
"outputs/ime-eval/tiny-ja-v1-ajimee/metrics.json": "219e15ec8ee54b09b455bd14ce60ab99bff239510ab9df344b69d2afc4b21196",
"outputs/ime-eval/tiny-ja-v1-ajimee/scores.jsonl": "247b29b59285551acaf73fabf9bee79c9617360623329e6bb60deb2d33ee7ddf",
"outputs/deployment/conservative-int8-b32-alignment-v1/alignment.json": "6773fafbd84e27dc1c349234fe601898e9f2af56ec5e2dcc6a32b12fbe14d018",
"outputs/deployment/conservative-int8-b32-distribution-v1/report.json": "6970eafdbb0d8e8b075d5b38b241cfa8fa46777961106ac32714755950722852",
"artifacts/models/tiny-ja-v1/summary.json": "4d9417b3bb8702b025b1ce4962d38ea36ad6d52928d19267dccdcaed3bf6c022"
},
"scope": "Offline fixed pools and saved Mac observations; not app-wide accuracy or a Windows Core ML runtime rerun."
}
|