Spaces:
Running
Running
File size: 10,762 Bytes
3d610ae d3c4073 3d610ae d3c4073 3d610ae | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 | {
"schema": "ekko-browser-validation-v1",
"date": "2026-09-21",
"status": "current-artifact-export-smoke-passed",
"platform": "macOS arm64",
"runtime": {
"sherpa_onnx": "1.13.6",
"onnxruntime_web": "1.29.0",
"sherpa_commit": "1cb484af5e69d3c7803c1eb0b3b5ab8041e0e911",
"emscripten": "4.0.23",
"wasm_sha256": "9b1d4b5ff9ff9f849fa725baa58e90a1c246cac7ff2f0303fa67aa3ad6ccd993",
"data_sha256": "f4452611fa61d1d51d6b764705812829c39088b1720c70ff01930b299d43796d",
"bundle_bytes_approximate": 13600000
},
"model": {
"revision": "e37e698805c538e447c25062ab13fa862462ee9e",
"download_bytes": 135307048,
"local_verified_cache_server": true,
"reason": "The private v5 artifact revision loaded through the checksum-verified local cache server."
},
"current_artifact_smoke": {
"onnx_structure": "passed",
"onnxruntime_cpu_load": "passed",
"sherpa_onnx_decode": "passed",
"native_q5_load_and_decode": "passed",
"browser_wasm": "passed-local-private-preview",
"browser_model_download_mib": 129.7
},
"pnc_v1_browser_validation": {
"revision": "b1ff0133c7a2bf63ae6288640c267462229a719f",
"download_bytes": 24434481,
"loads_after_asr_ready": true,
"local_verified_cache_server": true
},
"pnc_v2_browser_validation": {
"revision": "2eed18a93ff921de00e4ec53046ae9b1f33de172",
"manifest_verified_against_box_1_release": true,
"browser_runtime_status": "local-wasm-smoke-passed",
"platform": "macOS in-app Chromium browser",
"checks": {
"onnxruntime_web_load": "passed",
"browser_python_parity_on_three_fixed_inputs": "passed",
"interior_sentence_stop_preserved": true,
"cross_segment_sentence_start_capitalized": true,
"word_timestamps_preserved": true,
"one_word_output": "Ja."
},
"remaining_checks": [
"Live microphone speech with v2 PnC on physical desktop and mobile devices",
"Compare false sentence stops on one-word and short speech against v1",
"Memory and latency on desktop and mobile browsers"
]
},
"live_chunking_20260925": {
"status": "local-wasm-smoke-passed",
"fixture": "CoRal-project__coral-v3_train_0000002.wav",
"method": "Injected three copies with one-second pauses into the actual browser ASR worker",
"provisional_results_before_stop": 3,
"final_vad_results_before_stop": 3,
"first_preview_after_audio_seconds": 3.3,
"final_word_timestamps_present": true,
"capture_drain_acknowledgement_unit_test": "passed",
"full_demo_ui_with_synthetic_microphone": {
"status": "passed",
"preview_visible_while_speech_detected": true,
"final_punctuated_transcript_visible_before_stop": true,
"stop_completes_with_transcript_retained": true
},
"limitations": [
"Browser speech used a synthetic MediaStream rather than a physical microphone",
"The VAD maximum speech duration is a soft limit, so uninterrupted speech is provisional until a pause or stop",
"Physical desktop and mobile microphone checks remain"
]
},
"long_form_replay_20260925": {
"status": "completed-with-remaining-recognition-gaps",
"method": "User-provided Danish telephone recording converted to 16 kHz mono and queued through the actual browser WASM VAD and ASR worker; final segments then processed by the browser PnC worker",
"audio_seconds": 774.27,
"browser_worker_processing_seconds": 228.1,
"final_segments": 113,
"provisional_updates": 175,
"worker_errors": 0,
"overlapping_final_segments": 0,
"raw_words": 1736,
"pnc_preserved_segment_timing_and_word_count": true,
"pnc_processing_seconds": 1.6,
"quality_reference": "ElevenLabs transcript in the supplied Markdown, treated as gold for this evaluation",
"reference_words": 2316,
"normalization": "Speaker labels and punctuation removed; Unicode word tokens lowercased",
"wer_before_gap_recovery": 0.3735,
"wer_after_gap_recovery": 0.3549,
"final_substitutions": 200,
"final_deletions": 601,
"final_insertions": 21,
"remaining_issue": "One 16-second interval remained undertranscribed; this replay does not prove complete speech capture or physical-microphone behavior."
},
"sparse_boundary_retry_20260925": {
"status": "local-wasm-replay-passed",
"method": "Retry browser ASR on VAD segments at least 5 seconds long with fewer than 2.5 words per second, shifting the raw-audio decode window by plus and minus 64 samples (4 ms); select a retry only when it yields at least 3 more words",
"audio_seconds": 774.27,
"browser_worker_processing_seconds": 280.4,
"baseline_processing_seconds": 228.1,
"final_segments": 113,
"provisional_updates": 175,
"worker_errors": 0,
"changed_segments": 8,
"raw_words": 1841,
"pnc_preserved_segment_timing_and_word_count": true,
"pnc_processing_seconds": 1.79,
"quality_reference": "ElevenLabs transcript in the supplied Markdown, treated as gold for this evaluation",
"wer_before_retry": 0.3549,
"wer_after_retry": 0.3174,
"final_substitutions": 212,
"final_deletions": 499,
"final_insertions": 24,
"cross_service_probe": {
"source": "transcription-api-service phase vote and comfort-noise findings",
"strict_majority_vote_method": "Offline vote over baseline and plus/minus 4 ms browser decodes for the 17 sparse segments",
"strict_majority_vote_wer": 0.3390,
"changed_segments_overlapping_digital_silence_runs": 0,
"digital_silence_definition": "Absolute amplitude at most 0.0001 for at least 0.2 seconds",
"adopted": false
},
"limitations": [
"A 16-second interval remains undertranscribed",
"One recording does not establish the retry rule's accuracy on other speakers or devices",
"Accelerated file replay does not measure live microphone latency or mobile behavior"
]
},
"checks": {
"wasm_initialization": "passed",
"model_sha256_verification": "passed",
"localhost_preview_server": "passed",
"microphone_live_speech": {
"status": "passed-with-real-speech-through-synthetic-media-stream",
"word_count": 12,
"vad_segment_emitted": true,
"pnc_output": "Ja, sådan er det jo kraftigt, og sådan et metal, ikke ansigt."
},
"microphone_multi_chunk_speech": {
"status": "passed-with-real-speech-through-synthetic-media-stream",
"audio_seconds": 20.53,
"segments_before_stop": 5,
"segments_after_stop": 5,
"explicit_inter_word_spacing": true,
"text_stays_below_orb_controls": true,
"toolbar_stays_below_transcript": true,
"stop_page_errors": 0
},
"microphone_onset_preroll": {
"status": "passed-with-real-speech-through-synthetic-media-stream",
"model_revision": "aff2c6e39817e66186a0aa0b3cc8d78fc1ea68ad",
"fixture": "CoRal-project__coral-v3_train_0000002.wav",
"runs": 2,
"opening_word": "nu",
"opening_word_preserved_runs": 2,
"first_word_start_seconds": 0.0,
"second_session_zero_based": true,
"page_errors": 0,
"pnc_enabled": false
},
"microphone_one_word": {
"status": "passed-with-real-speech-through-synthetic-media-stream",
"model_revision": "aff2c6e39817e66186a0aa0b3cc8d78fc1ea68ad",
"fixture_durations_seconds": [0.47, 0.62, 0.82],
"expected_word": "ja",
"recognized_before_stop": ["ja", "ja", "ja"],
"minimum_speech_seconds": 0.12
},
"pnc_export_regression": {
"status": "passed",
"source_words": 176,
"output_words": 176,
"lowercase_after_sentence_stop": false,
"malformed_mid_segment_stops_removed": true,
"cross_segment_sentence_starts_capitalized": true,
"word_timestamps_retained": true
},
"v5_browser_wasm_smoke": {
"status": "passed-local-private-preview",
"model_revision": "e37e698805c538e447c25062ab13fa862462ee9e",
"int8_bundle_bytes": 135307058,
"current_matrix_mean_wer": 12.945774,
"one_word_fixture_seconds": 0.47,
"one_word_output": "ja",
"sentence_output": "nu tager du det her billede for det er lidt mærkeligt",
"native_timestamps_present": true,
"resource_errors": 0
},
"conversation_history": {
"status": "passed",
"storage": "localStorage",
"maximum_sessions": 20,
"persists_across_reload": true,
"reopens_saved_transcript": true,
"individual_delete": true,
"clear_all": true,
"stores_audio": false,
"mobile_dialog_fits_viewport": true
},
"pnc_browser_parity_probe": {
"status": "passed",
"input": "hej mit navn er emil hvordan går det",
"browser_and_python_output": "Hej, mit navn er Emil, hvordan går det?",
"word_count": 8,
"timestamps_preserved": true
},
"rye_voice_interaction": {
"orb_shader_parity": "passed",
"mobile_stage_css_pixels": 218,
"mobile_orb_css_pixels": 192,
"shader_colors": ["#27AE60", "#8CBFFF", "#FFAAAD"],
"mobile_orb_lift_pixels": 186,
"mobile_idle_label_gap_pixels": 42,
"animated_words": 12,
"mobile_orb_transcript_overlap": false,
"desktop_orb_transcript_overlap": false,
"desktop_transcript_toolbar_overlap": false
},
"warmed_decoder_performance_harness": {
"device": "Apple M1 Pro",
"browser": "Chromium 148 / Electron 42",
"initial_model_download_and_load_excluded": true,
"note": "Measured through the same offline decoder before file upload was removed from the public UI.",
"short_clip": {
"audio_seconds": 3.12,
"runs": 5,
"median_processing_seconds": 0.526,
"median_rtf": 0.16859,
"times_realtime": 5.93
},
"long_clip": {
"audio_seconds": 17.86,
"runs": 3,
"median_processing_seconds": 2.099,
"median_rtf": 0.11753,
"times_realtime": 8.51
}
},
"mobile_390x844_horizontal_overflow": false,
"desktop_1440x1000_horizontal_overflow": false,
"mobile_dialogs_fit_viewport": true,
"clean_reload_resource_errors": 0
},
"checks_model_revision": "7cb1747204bddd558ff449399689d5bb641fadd8",
"remaining_release_checks": [
"Load the current Tiny and PnC models directly from their public Hugging Face revisions without interception.",
"Run the complete current-model WER matrix through WebAssembly; local initialization cannot establish browser parity.",
"Run microphone speech on physical desktop and mobile devices.",
"Verify Safari, Chromium, and Firefox memory behavior.",
"Publish the static Space only after model terms and visibility are final."
]
}
|