{ "variant": "fp16-length-and-cache-mask", "tested_on": "2026-10-12", "affected_A16_iOS_27_2_validation": "pending", "unit_tests_passed": 17, "encoder_cpu_parity": { "calls": 201, "outputs_per_call": 6, "comparison": "bit-for-bit equal", "coverage": "20s real speech; silence; partial bootstrap; all mel lengths 0..32 at cache lengths 0,1,55,56; two additional language-prompt slots" }, "published_encoder_palettes_preserved": 296, "public_feature_interface": "unchanged", "unchanged_runtime_files": [ "config.json", "decoder.mlmodelc/analytics/coremldata.bin", "decoder.mlmodelc/coremldata.bin", "decoder.mlmodelc/model.mil", "decoder.mlmodelc/weights/weight.bin", "joint.mlmodelc/analytics/coremldata.bin", "joint.mlmodelc/coremldata.bin", "joint.mlmodelc/model.mil", "joint.mlmodelc/weights/weight.bin", "languages.json", "tokenizer.model", "vocab.json" ], "mac": { "hardware": "Apple M5 Pro", "os": "macOS 26.6.2 (25G83)", "transcription": { "transcript-published-cpu": { "bundle": "published", "compute_units": "cpu", "load_seconds": 0.16790483298245817, "audio_seconds": 20.0, "transcription_seconds": 2.187830250011757, "real_time_factor": 0.10939151250058785, "transcript": "Can you guarantee that the replacement part will be shipped tomorrow", "tokens": [ 3013, 113, 2810, 2, 836, 9833, 1313, 26, 2811, 150, 2, 311, 97, 284, 73, 1037, 2981, 2834, 427, 2, 40, 2833, 2343, 2816, 12945, 411, 409, 59, 2952 ], "reference_tokens_match": null, "scope": "Python Core ML streaming encoder and greedy RNN-T decoder; one English fixture" }, "transcript-candidate-ane": { "bundle": "candidate", "compute_units": "ane", "load_seconds": 6.747368875076063, "audio_seconds": 20.0, "transcription_seconds": 0.8391279169591144, "real_time_factor": 0.04195639584795572, "transcript": "Can you guarantee that the replacement part will be shipped tomorrow", "tokens": [ 3013, 113, 2810, 2, 836, 9833, 1313, 26, 2811, 150, 2, 311, 97, 284, 73, 1037, 2981, 2834, 427, 2, 40, 2833, 2343, 2816, 12945, 411, 409, 59, 2952 ], "reference_tokens_match": true, "scope": "Python Core ML streaming encoder and greedy RNN-T decoder; one English fixture" }, "transcript-published-ane": { "bundle": "published", "compute_units": "ane", "load_seconds": 6.377891334006563, "audio_seconds": 20.0, "transcription_seconds": 0.6900540000060573, "real_time_factor": 0.034502700000302866, "transcript": "Can you guarantee that the replacement part will be shipped tomorrow", "tokens": [ 3013, 113, 2810, 2, 836, 9833, 1313, 26, 2811, 150, 2, 311, 97, 284, 73, 1037, 2981, 2834, 427, 2, 40, 2833, 2343, 2816, 12945, 411, 409, 59, 2952 ], "reference_tokens_match": true, "scope": "Python Core ML streaming encoder and greedy RNN-T decoder; one English fixture" }, "transcript-candidate-cpu": { "bundle": "candidate", "compute_units": "cpu", "load_seconds": 0.16314362501725554, "audio_seconds": 20.0, "transcription_seconds": 2.3523729579756036, "real_time_factor": 0.11761864789878018, "transcript": "Can you guarantee that the replacement part will be shipped tomorrow", "tokens": [ 3013, 113, 2810, 2, 836, 9833, 1313, 26, 2811, 150, 2, 311, 97, 284, 73, 1037, 2981, 2834, 427, 2, 40, 2833, 2343, 2816, 12945, 411, 409, 59, 2952 ], "reference_tokens_match": true, "scope": "Python Core ML streaming encoder and greedy RNN-T decoder; one English fixture" } }, "encoder_load_probes": { "mac-candidate-ane": { "bundle": "candidate", "computeUnits": "cpuAndNeuralEngine", "machine": "arm64", "models": [ { "component": "encoder", "firstObserved": { "firstPredictionMs": 14.432708267122507, "loadMs": 7202.025166596286, "medianRepeatedPredictionMs": 9.578666649758816 }, "inputs": [ { "dtype": "MLMultiArrayDataType(rawValue: 131104)", "name": "audio_length", "shape": [ 1 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65568)", "name": "audio_signal", "shape": [ 1, 128, 32 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65568)", "name": "cache_last_channel", "shape": [ 24, 1, 56, 1024 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 131104)", "name": "cache_last_channel_len", "shape": [ 1 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65568)", "name": "cache_last_time", "shape": [ 24, 1, 1024, 8 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65568)", "name": "language_mask", "shape": [ 1, 128 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65568)", "name": "pre_cache", "shape": [ 1, 128, 9 ] } ], "outputs": [ { "dtype": "MLMultiArrayDataType(rawValue: 131104)", "name": "encoded_length", "shape": [ 1 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65552)", "name": "encoded_output", "shape": [ 1, 4, 1024 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65552)", "name": "new_cache_last_channel", "shape": [ 24, 1, 56, 1024 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 131104)", "name": "new_cache_last_channel_len", "shape": [ 1 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65552)", "name": "new_cache_last_time", "shape": [ 24, 1, 1024, 8 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65552)", "name": "new_pre_cache", "shape": [ 1, 128, 9 ] } ], "plannedOperationsByPreferredDevice": { "cpu": 19, "neuralEngine": 1653 }, "sameProcessReload": { "firstPredictionMs": 17.375333234667778, "loadMs": 59.644958353601396, "medianRepeatedPredictionMs": 8.94320837687701 } }, { "component": "decoder", "firstObserved": { "firstPredictionMs": 0.9465833427384496, "loadMs": 86.13462501671165, "medianRepeatedPredictionMs": 0.08145836181938648 }, "inputs": [ { "dtype": "MLMultiArrayDataType(rawValue: 65552)", "name": "c", "shape": [ 2, 1, 640 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65552)", "name": "h", "shape": [ 2, 1, 640 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 131104)", "name": "token", "shape": [ 1, 1 ] } ], "outputs": [ { "dtype": "MLMultiArrayDataType(rawValue: 65552)", "name": "c_out", "shape": [ 2, 1, 640 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65552)", "name": "decoder_output", "shape": [ 1, 1, 640 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65552)", "name": "h_out", "shape": [ 2, 1, 640 ] } ], "plannedOperationsByPreferredDevice": { "cpu": 14 }, "sameProcessReload": { "firstPredictionMs": 0.8322499925270677, "loadMs": 5.513999960385263, "medianRepeatedPredictionMs": 0.08637504652142525 } }, { "component": "joint", "firstObserved": { "firstPredictionMs": 0.6359167164191604, "loadMs": 135.69379155524075, "medianRepeatedPredictionMs": 0.2609583316370845 }, "inputs": [ { "dtype": "MLMultiArrayDataType(rawValue: 65552)", "name": "decoder_output", "shape": [ 1, 1, 640 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65552)", "name": "encoder_output", "shape": [ 1, 1, 1024 ] } ], "outputs": [ { "dtype": "MLMultiArrayDataType(rawValue: 65552)", "name": "logits", "shape": [ 1, 1, 13088 ] } ], "plannedOperationsByPreferredDevice": { "neuralEngine": 10 }, "sameProcessReload": { "firstPredictionMs": 0.5140416324138641, "loadMs": 6.4516665879637, "medianRepeatedPredictionMs": 0.2882083645090461 } } ], "os": "Version 26.6.2 (Build 25G83)" }, "mac-published-ane": { "bundle": "published", "computeUnits": "cpuAndNeuralEngine", "machine": "arm64", "models": [ { "component": "encoder", "firstObserved": { "firstPredictionMs": 22.793249925598502, "loadMs": 7534.660249948502, "medianRepeatedPredictionMs": 8.524958393536508 }, "inputs": [ { "dtype": "MLMultiArrayDataType(rawValue: 131104)", "name": "audio_length", "shape": [ 1 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65568)", "name": "audio_signal", "shape": [ 1, 128, 32 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65568)", "name": "cache_last_channel", "shape": [ 24, 1, 56, 1024 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 131104)", "name": "cache_last_channel_len", "shape": [ 1 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65568)", "name": "cache_last_time", "shape": [ 24, 1, 1024, 8 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65568)", "name": "language_mask", "shape": [ 1, 128 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65568)", "name": "pre_cache", "shape": [ 1, 128, 9 ] } ], "outputs": [ { "dtype": "MLMultiArrayDataType(rawValue: 131104)", "name": "encoded_length", "shape": [ 1 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65552)", "name": "encoded_output", "shape": [ 1, 4, 1024 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65552)", "name": "new_cache_last_channel", "shape": [ 24, 1, 56, 1024 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 131104)", "name": "new_cache_last_channel_len", "shape": [ 1 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65552)", "name": "new_cache_last_time", "shape": [ 24, 1, 1024, 8 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65552)", "name": "new_pre_cache", "shape": [ 1, 128, 9 ] } ], "plannedOperationsByPreferredDevice": { "cpu": 86, "neuralEngine": 1592 }, "sameProcessReload": { "firstPredictionMs": 17.927875043824315, "loadMs": 62.79983336571604, "medianRepeatedPredictionMs": 9.84187494032085 } }, { "component": "decoder", "firstObserved": { "firstPredictionMs": 0.9524166816845536, "loadMs": 91.23366663698107, "medianRepeatedPredictionMs": 0.07620826363563538 }, "inputs": [ { "dtype": "MLMultiArrayDataType(rawValue: 65552)", "name": "c", "shape": [ 2, 1, 640 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65552)", "name": "h", "shape": [ 2, 1, 640 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 131104)", "name": "token", "shape": [ 1, 1 ] } ], "outputs": [ { "dtype": "MLMultiArrayDataType(rawValue: 65552)", "name": "c_out", "shape": [ 2, 1, 640 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65552)", "name": "decoder_output", "shape": [ 1, 1, 640 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65552)", "name": "h_out", "shape": [ 2, 1, 640 ] } ], "plannedOperationsByPreferredDevice": { "cpu": 14 }, "sameProcessReload": { "firstPredictionMs": 0.7374166743829846, "loadMs": 5.401666625402868, "medianRepeatedPredictionMs": 0.09624997619539499 } }, { "component": "joint", "firstObserved": { "firstPredictionMs": 0.6203333614394069, "loadMs": 135.20483335014433, "medianRepeatedPredictionMs": 0.2691249828785658 }, "inputs": [ { "dtype": "MLMultiArrayDataType(rawValue: 65552)", "name": "decoder_output", "shape": [ 1, 1, 640 ] }, { "dtype": "MLMultiArrayDataType(rawValue: 65552)", "name": "encoder_output", "shape": [ 1, 1, 1024 ] } ], "outputs": [ { "dtype": "MLMultiArrayDataType(rawValue: 65552)", "name": "logits", "shape": [ 1, 1, 13088 ] } ], "plannedOperationsByPreferredDevice": { "neuralEngine": 10 }, "sameProcessReload": { "firstPredictionMs": 0.5433750338852406, "loadMs": 6.290458375588059, "medianRepeatedPredictionMs": 0.30408333986997604 } } ], "os": "Version 26.6.2 (Build 25G83)" } } }, "iphone": { "model": "iPhone 16 Pro", "chip": "A18 Pro", "os": "iOS 26.6 (23G71)", "speech_checks": { "phone-published-cpu-speech": { "bundle": "published", "computeUnits": "cpuOnly", "loadSeconds": 32.7600520416745, "predictionSeconds": 4.448477875033859, "realTimeFactor": 0.22242389375169297, "transcript": "Can you guarantee that the replacement part will be shipped tomorrow", "reference_tokens_match": true, "timing_valid_for_comparison": false }, "phone-published-ane-speech": { "bundle": "published", "computeUnits": "cpuAndNeuralEngine", "loadSeconds": 16.41530641663121, "predictionSeconds": 1.184877083345782, "realTimeFactor": 0.0592438541672891, "transcript": "Can you guarantee that the replacement part will be shipped tomorrow", "reference_tokens_match": true }, "phone-candidate-cpu-speech": { "bundle": "candidate", "computeUnits": "cpuOnly", "loadSeconds": 26.050377291685436, "predictionSeconds": 2.0800217500072904, "realTimeFactor": 0.10400108750036452, "transcript": "Can you guarantee that the replacement part will be shipped tomorrow", "reference_tokens_match": true }, "phone-candidate-ane-speech": { "bundle": "candidate", "computeUnits": "cpuAndNeuralEngine", "loadSeconds": 20.91144558333326, "predictionSeconds": 1.5292800416355021, "realTimeFactor": 0.0764640020817751, "transcript": "Can you guarantee that the replacement part will be shipped tomorrow", "reference_tokens_match": true } }, "validation": { "configurations": 4, "audio_seconds_per_configuration": 20, "mel_chunks_per_configuration": 63, "reference_token_count": 29, "all_tokens_exactly_equal": true, "scope": "Dedicated native CoreML harness with frozen mel features; no Swift SDK suite rebuild", "candidate_ane_prediction_seconds": 1.5292800416355021, "published_ane_prediction_seconds": 1.184877083345782, "timing_interpretation": "Single passes do not establish a speedup; candidate was slower in this CPU+ANE pass", "cache_timing_status": "Additional restarted-process load comparison not completed due storage and device connection constraints" } }, "limitations": [ "No validation on the affected A16/iOS27.2 build", "No Swift SDK suite was rebuilt; Mac and iPhone transcription checks use standalone CoreML greedy streaming decoders", "One English real-speech fixture; native iPhone tests use frozen mel features, and extra language-prompt checks use synthetic input", "Compute-plan preferences describe planned placement, not measured ANE utilization", "Published CPU native timing is excluded because BNNS compilation reported No space left on device", "Additional restarted-process cache timing comparison was not completed; no speedup claim" ] }