| { |
| "variant": "fp16-length-and-cache-mask", |
| "tested_on": "2026-10-12", |
| "affected_A16_iOS_27_2_validation": "pending", |
| "unit_tests_passed": 17, |
| "encoder_cpu_parity": { |
| "calls": 201, |
| "outputs_per_call": 6, |
| "comparison": "bit-for-bit equal", |
| "coverage": "20s real speech; silence; partial bootstrap; all mel lengths 0..32 at cache lengths 0,1,55,56; two additional language-prompt slots" |
| }, |
| "published_encoder_palettes_preserved": 296, |
| "public_feature_interface": "unchanged", |
| "unchanged_runtime_files": [ |
| "config.json", |
| "decoder.mlmodelc/analytics/coremldata.bin", |
| "decoder.mlmodelc/coremldata.bin", |
| "decoder.mlmodelc/model.mil", |
| "decoder.mlmodelc/weights/weight.bin", |
| "joint.mlmodelc/analytics/coremldata.bin", |
| "joint.mlmodelc/coremldata.bin", |
| "joint.mlmodelc/model.mil", |
| "joint.mlmodelc/weights/weight.bin", |
| "languages.json", |
| "tokenizer.model", |
| "vocab.json" |
| ], |
| "mac": { |
| "hardware": "Apple M5 Pro", |
| "os": "macOS 26.6.2 (25G83)", |
| "transcription": { |
| "transcript-published-cpu": { |
| "bundle": "published", |
| "compute_units": "cpu", |
| "load_seconds": 0.16790483298245817, |
| "audio_seconds": 20.0, |
| "transcription_seconds": 2.187830250011757, |
| "real_time_factor": 0.10939151250058785, |
| "transcript": "Can you guarantee that the replacement part will be shipped tomorrow", |
| "tokens": [ |
| 3013, |
| 113, |
| 2810, |
| 2, |
| 836, |
| 9833, |
| 1313, |
| 26, |
| 2811, |
| 150, |
| 2, |
| 311, |
| 97, |
| 284, |
| 73, |
| 1037, |
| 2981, |
| 2834, |
| 427, |
| 2, |
| 40, |
| 2833, |
| 2343, |
| 2816, |
| 12945, |
| 411, |
| 409, |
| 59, |
| 2952 |
| ], |
| "reference_tokens_match": null, |
| "scope": "Python Core ML streaming encoder and greedy RNN-T decoder; one English fixture" |
| }, |
| "transcript-candidate-ane": { |
| "bundle": "candidate", |
| "compute_units": "ane", |
| "load_seconds": 6.747368875076063, |
| "audio_seconds": 20.0, |
| "transcription_seconds": 0.8391279169591144, |
| "real_time_factor": 0.04195639584795572, |
| "transcript": "Can you guarantee that the replacement part will be shipped tomorrow", |
| "tokens": [ |
| 3013, |
| 113, |
| 2810, |
| 2, |
| 836, |
| 9833, |
| 1313, |
| 26, |
| 2811, |
| 150, |
| 2, |
| 311, |
| 97, |
| 284, |
| 73, |
| 1037, |
| 2981, |
| 2834, |
| 427, |
| 2, |
| 40, |
| 2833, |
| 2343, |
| 2816, |
| 12945, |
| 411, |
| 409, |
| 59, |
| 2952 |
| ], |
| "reference_tokens_match": true, |
| "scope": "Python Core ML streaming encoder and greedy RNN-T decoder; one English fixture" |
| }, |
| "transcript-published-ane": { |
| "bundle": "published", |
| "compute_units": "ane", |
| "load_seconds": 6.377891334006563, |
| "audio_seconds": 20.0, |
| "transcription_seconds": 0.6900540000060573, |
| "real_time_factor": 0.034502700000302866, |
| "transcript": "Can you guarantee that the replacement part will be shipped tomorrow", |
| "tokens": [ |
| 3013, |
| 113, |
| 2810, |
| 2, |
| 836, |
| 9833, |
| 1313, |
| 26, |
| 2811, |
| 150, |
| 2, |
| 311, |
| 97, |
| 284, |
| 73, |
| 1037, |
| 2981, |
| 2834, |
| 427, |
| 2, |
| 40, |
| 2833, |
| 2343, |
| 2816, |
| 12945, |
| 411, |
| 409, |
| 59, |
| 2952 |
| ], |
| "reference_tokens_match": true, |
| "scope": "Python Core ML streaming encoder and greedy RNN-T decoder; one English fixture" |
| }, |
| "transcript-candidate-cpu": { |
| "bundle": "candidate", |
| "compute_units": "cpu", |
| "load_seconds": 0.16314362501725554, |
| "audio_seconds": 20.0, |
| "transcription_seconds": 2.3523729579756036, |
| "real_time_factor": 0.11761864789878018, |
| "transcript": "Can you guarantee that the replacement part will be shipped tomorrow", |
| "tokens": [ |
| 3013, |
| 113, |
| 2810, |
| 2, |
| 836, |
| 9833, |
| 1313, |
| 26, |
| 2811, |
| 150, |
| 2, |
| 311, |
| 97, |
| 284, |
| 73, |
| 1037, |
| 2981, |
| 2834, |
| 427, |
| 2, |
| 40, |
| 2833, |
| 2343, |
| 2816, |
| 12945, |
| 411, |
| 409, |
| 59, |
| 2952 |
| ], |
| "reference_tokens_match": true, |
| "scope": "Python Core ML streaming encoder and greedy RNN-T decoder; one English fixture" |
| } |
| }, |
| "encoder_load_probes": { |
| "mac-candidate-ane": { |
| "bundle": "candidate", |
| "computeUnits": "cpuAndNeuralEngine", |
| "machine": "arm64", |
| "models": [ |
| { |
| "component": "encoder", |
| "firstObserved": { |
| "firstPredictionMs": 14.432708267122507, |
| "loadMs": 7202.025166596286, |
| "medianRepeatedPredictionMs": 9.578666649758816 |
| }, |
| "inputs": [ |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 131104)", |
| "name": "audio_length", |
| "shape": [ |
| 1 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65568)", |
| "name": "audio_signal", |
| "shape": [ |
| 1, |
| 128, |
| 32 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65568)", |
| "name": "cache_last_channel", |
| "shape": [ |
| 24, |
| 1, |
| 56, |
| 1024 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 131104)", |
| "name": "cache_last_channel_len", |
| "shape": [ |
| 1 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65568)", |
| "name": "cache_last_time", |
| "shape": [ |
| 24, |
| 1, |
| 1024, |
| 8 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65568)", |
| "name": "language_mask", |
| "shape": [ |
| 1, |
| 128 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65568)", |
| "name": "pre_cache", |
| "shape": [ |
| 1, |
| 128, |
| 9 |
| ] |
| } |
| ], |
| "outputs": [ |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 131104)", |
| "name": "encoded_length", |
| "shape": [ |
| 1 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65552)", |
| "name": "encoded_output", |
| "shape": [ |
| 1, |
| 4, |
| 1024 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65552)", |
| "name": "new_cache_last_channel", |
| "shape": [ |
| 24, |
| 1, |
| 56, |
| 1024 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 131104)", |
| "name": "new_cache_last_channel_len", |
| "shape": [ |
| 1 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65552)", |
| "name": "new_cache_last_time", |
| "shape": [ |
| 24, |
| 1, |
| 1024, |
| 8 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65552)", |
| "name": "new_pre_cache", |
| "shape": [ |
| 1, |
| 128, |
| 9 |
| ] |
| } |
| ], |
| "plannedOperationsByPreferredDevice": { |
| "cpu": 19, |
| "neuralEngine": 1653 |
| }, |
| "sameProcessReload": { |
| "firstPredictionMs": 17.375333234667778, |
| "loadMs": 59.644958353601396, |
| "medianRepeatedPredictionMs": 8.94320837687701 |
| } |
| }, |
| { |
| "component": "decoder", |
| "firstObserved": { |
| "firstPredictionMs": 0.9465833427384496, |
| "loadMs": 86.13462501671165, |
| "medianRepeatedPredictionMs": 0.08145836181938648 |
| }, |
| "inputs": [ |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65552)", |
| "name": "c", |
| "shape": [ |
| 2, |
| 1, |
| 640 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65552)", |
| "name": "h", |
| "shape": [ |
| 2, |
| 1, |
| 640 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 131104)", |
| "name": "token", |
| "shape": [ |
| 1, |
| 1 |
| ] |
| } |
| ], |
| "outputs": [ |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65552)", |
| "name": "c_out", |
| "shape": [ |
| 2, |
| 1, |
| 640 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65552)", |
| "name": "decoder_output", |
| "shape": [ |
| 1, |
| 1, |
| 640 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65552)", |
| "name": "h_out", |
| "shape": [ |
| 2, |
| 1, |
| 640 |
| ] |
| } |
| ], |
| "plannedOperationsByPreferredDevice": { |
| "cpu": 14 |
| }, |
| "sameProcessReload": { |
| "firstPredictionMs": 0.8322499925270677, |
| "loadMs": 5.513999960385263, |
| "medianRepeatedPredictionMs": 0.08637504652142525 |
| } |
| }, |
| { |
| "component": "joint", |
| "firstObserved": { |
| "firstPredictionMs": 0.6359167164191604, |
| "loadMs": 135.69379155524075, |
| "medianRepeatedPredictionMs": 0.2609583316370845 |
| }, |
| "inputs": [ |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65552)", |
| "name": "decoder_output", |
| "shape": [ |
| 1, |
| 1, |
| 640 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65552)", |
| "name": "encoder_output", |
| "shape": [ |
| 1, |
| 1, |
| 1024 |
| ] |
| } |
| ], |
| "outputs": [ |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65552)", |
| "name": "logits", |
| "shape": [ |
| 1, |
| 1, |
| 13088 |
| ] |
| } |
| ], |
| "plannedOperationsByPreferredDevice": { |
| "neuralEngine": 10 |
| }, |
| "sameProcessReload": { |
| "firstPredictionMs": 0.5140416324138641, |
| "loadMs": 6.4516665879637, |
| "medianRepeatedPredictionMs": 0.2882083645090461 |
| } |
| } |
| ], |
| "os": "Version 26.6.2 (Build 25G83)" |
| }, |
| "mac-published-ane": { |
| "bundle": "published", |
| "computeUnits": "cpuAndNeuralEngine", |
| "machine": "arm64", |
| "models": [ |
| { |
| "component": "encoder", |
| "firstObserved": { |
| "firstPredictionMs": 22.793249925598502, |
| "loadMs": 7534.660249948502, |
| "medianRepeatedPredictionMs": 8.524958393536508 |
| }, |
| "inputs": [ |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 131104)", |
| "name": "audio_length", |
| "shape": [ |
| 1 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65568)", |
| "name": "audio_signal", |
| "shape": [ |
| 1, |
| 128, |
| 32 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65568)", |
| "name": "cache_last_channel", |
| "shape": [ |
| 24, |
| 1, |
| 56, |
| 1024 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 131104)", |
| "name": "cache_last_channel_len", |
| "shape": [ |
| 1 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65568)", |
| "name": "cache_last_time", |
| "shape": [ |
| 24, |
| 1, |
| 1024, |
| 8 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65568)", |
| "name": "language_mask", |
| "shape": [ |
| 1, |
| 128 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65568)", |
| "name": "pre_cache", |
| "shape": [ |
| 1, |
| 128, |
| 9 |
| ] |
| } |
| ], |
| "outputs": [ |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 131104)", |
| "name": "encoded_length", |
| "shape": [ |
| 1 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65552)", |
| "name": "encoded_output", |
| "shape": [ |
| 1, |
| 4, |
| 1024 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65552)", |
| "name": "new_cache_last_channel", |
| "shape": [ |
| 24, |
| 1, |
| 56, |
| 1024 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 131104)", |
| "name": "new_cache_last_channel_len", |
| "shape": [ |
| 1 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65552)", |
| "name": "new_cache_last_time", |
| "shape": [ |
| 24, |
| 1, |
| 1024, |
| 8 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65552)", |
| "name": "new_pre_cache", |
| "shape": [ |
| 1, |
| 128, |
| 9 |
| ] |
| } |
| ], |
| "plannedOperationsByPreferredDevice": { |
| "cpu": 86, |
| "neuralEngine": 1592 |
| }, |
| "sameProcessReload": { |
| "firstPredictionMs": 17.927875043824315, |
| "loadMs": 62.79983336571604, |
| "medianRepeatedPredictionMs": 9.84187494032085 |
| } |
| }, |
| { |
| "component": "decoder", |
| "firstObserved": { |
| "firstPredictionMs": 0.9524166816845536, |
| "loadMs": 91.23366663698107, |
| "medianRepeatedPredictionMs": 0.07620826363563538 |
| }, |
| "inputs": [ |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65552)", |
| "name": "c", |
| "shape": [ |
| 2, |
| 1, |
| 640 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65552)", |
| "name": "h", |
| "shape": [ |
| 2, |
| 1, |
| 640 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 131104)", |
| "name": "token", |
| "shape": [ |
| 1, |
| 1 |
| ] |
| } |
| ], |
| "outputs": [ |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65552)", |
| "name": "c_out", |
| "shape": [ |
| 2, |
| 1, |
| 640 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65552)", |
| "name": "decoder_output", |
| "shape": [ |
| 1, |
| 1, |
| 640 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65552)", |
| "name": "h_out", |
| "shape": [ |
| 2, |
| 1, |
| 640 |
| ] |
| } |
| ], |
| "plannedOperationsByPreferredDevice": { |
| "cpu": 14 |
| }, |
| "sameProcessReload": { |
| "firstPredictionMs": 0.7374166743829846, |
| "loadMs": 5.401666625402868, |
| "medianRepeatedPredictionMs": 0.09624997619539499 |
| } |
| }, |
| { |
| "component": "joint", |
| "firstObserved": { |
| "firstPredictionMs": 0.6203333614394069, |
| "loadMs": 135.20483335014433, |
| "medianRepeatedPredictionMs": 0.2691249828785658 |
| }, |
| "inputs": [ |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65552)", |
| "name": "decoder_output", |
| "shape": [ |
| 1, |
| 1, |
| 640 |
| ] |
| }, |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65552)", |
| "name": "encoder_output", |
| "shape": [ |
| 1, |
| 1, |
| 1024 |
| ] |
| } |
| ], |
| "outputs": [ |
| { |
| "dtype": "MLMultiArrayDataType(rawValue: 65552)", |
| "name": "logits", |
| "shape": [ |
| 1, |
| 1, |
| 13088 |
| ] |
| } |
| ], |
| "plannedOperationsByPreferredDevice": { |
| "neuralEngine": 10 |
| }, |
| "sameProcessReload": { |
| "firstPredictionMs": 0.5433750338852406, |
| "loadMs": 6.290458375588059, |
| "medianRepeatedPredictionMs": 0.30408333986997604 |
| } |
| } |
| ], |
| "os": "Version 26.6.2 (Build 25G83)" |
| } |
| } |
| }, |
| "iphone": { |
| "model": "iPhone 16 Pro", |
| "chip": "A18 Pro", |
| "os": "iOS 26.6 (23G71)", |
| "speech_checks": { |
| "phone-published-cpu-speech": { |
| "bundle": "published", |
| "computeUnits": "cpuOnly", |
| "loadSeconds": 32.7600520416745, |
| "predictionSeconds": 4.448477875033859, |
| "realTimeFactor": 0.22242389375169297, |
| "transcript": "Can you guarantee that the replacement part will be shipped tomorrow", |
| "reference_tokens_match": true, |
| "timing_valid_for_comparison": false |
| }, |
| "phone-published-ane-speech": { |
| "bundle": "published", |
| "computeUnits": "cpuAndNeuralEngine", |
| "loadSeconds": 16.41530641663121, |
| "predictionSeconds": 1.184877083345782, |
| "realTimeFactor": 0.0592438541672891, |
| "transcript": "Can you guarantee that the replacement part will be shipped tomorrow", |
| "reference_tokens_match": true |
| }, |
| "phone-candidate-cpu-speech": { |
| "bundle": "candidate", |
| "computeUnits": "cpuOnly", |
| "loadSeconds": 26.050377291685436, |
| "predictionSeconds": 2.0800217500072904, |
| "realTimeFactor": 0.10400108750036452, |
| "transcript": "Can you guarantee that the replacement part will be shipped tomorrow", |
| "reference_tokens_match": true |
| }, |
| "phone-candidate-ane-speech": { |
| "bundle": "candidate", |
| "computeUnits": "cpuAndNeuralEngine", |
| "loadSeconds": 20.91144558333326, |
| "predictionSeconds": 1.5292800416355021, |
| "realTimeFactor": 0.0764640020817751, |
| "transcript": "Can you guarantee that the replacement part will be shipped tomorrow", |
| "reference_tokens_match": true |
| } |
| }, |
| "validation": { |
| "configurations": 4, |
| "audio_seconds_per_configuration": 20, |
| "mel_chunks_per_configuration": 63, |
| "reference_token_count": 29, |
| "all_tokens_exactly_equal": true, |
| "scope": "Dedicated native CoreML harness with frozen mel features; no Swift SDK suite rebuild", |
| "candidate_ane_prediction_seconds": 1.5292800416355021, |
| "published_ane_prediction_seconds": 1.184877083345782, |
| "timing_interpretation": "Single passes do not establish a speedup; candidate was slower in this CPU+ANE pass", |
| "cache_timing_status": "Additional restarted-process load comparison not completed due storage and device connection constraints" |
| } |
| }, |
| "limitations": [ |
| "No validation on the affected A16/iOS27.2 build", |
| "No Swift SDK suite was rebuilt; Mac and iPhone transcription checks use standalone CoreML greedy streaming decoders", |
| "One English real-speech fixture; native iPhone tests use frozen mel features, and extra language-prompt checks use synthetic input", |
| "Compute-plan preferences describe planned placement, not measured ANE utilization", |
| "Published CPU native timing is excluded because BNNS compilation reported No space left on device", |
| "Additional restarted-process cache timing comparison was not completed; no speedup claim" |
| ] |
| } |
|
|
