validation.json

{
  "schema": "ekko-browser-validation-v1",
  "date": "2026-09-21",
  "status": "current-artifact-export-smoke-passed",
  "platform": "macOS arm64",
  "runtime": {
    "sherpa_onnx": "1.13.6",
    "onnxruntime_web": "1.29.0",
    "sherpa_commit": "1cb484af5e69d3c7803c1eb0b3b5ab8041e0e911",
    "emscripten": "4.0.23",
    "wasm_sha256": "9b1d4b5ff9ff9f849fa725baa58e90a1c246cac7ff2f0303fa67aa3ad6ccd993",
    "data_sha256": "f4452611fa61d1d51d6b764705812829c39088b1720c70ff01930b299d43796d",
    "bundle_bytes_approximate": 13600000
  },
  "model": {
    "revision": "e37e698805c538e447c25062ab13fa862462ee9e",
    "download_bytes": 135307048,
    "local_verified_cache_server": true,
    "reason": "The private v5 artifact revision loaded through the checksum-verified local cache server."
  },
  "current_artifact_smoke": {
    "onnx_structure": "passed",
    "onnxruntime_cpu_load": "passed",
    "sherpa_onnx_decode": "passed",
    "native_q5_load_and_decode": "passed",
    "browser_wasm": "passed-local-private-preview",
    "browser_model_download_mib": 129.7
  },
  "pnc_v1_browser_validation": {
    "revision": "b1ff0133c7a2bf63ae6288640c267462229a719f",
    "download_bytes": 24434481,
    "loads_after_asr_ready": true,
    "local_verified_cache_server": true
  },
  "pnc_v2_browser_validation": {
    "revision": "2eed18a93ff921de00e4ec53046ae9b1f33de172",
    "manifest_verified_against_box_1_release": true,
    "browser_runtime_status": "local-wasm-smoke-passed",
    "platform": "macOS in-app Chromium browser",
    "checks": {
      "onnxruntime_web_load": "passed",
      "browser_python_parity_on_three_fixed_inputs": "passed",
      "interior_sentence_stop_preserved": true,
      "cross_segment_sentence_start_capitalized": true,
      "word_timestamps_preserved": true,
      "one_word_output": "Ja."
    },
    "remaining_checks": [
      "Live microphone speech with v2 PnC on physical desktop and mobile devices",
      "Compare false sentence stops on one-word and short speech against v1",
      "Memory and latency on desktop and mobile browsers"
    ]
  },
  "live_chunking_20260925": {
    "status": "local-wasm-smoke-passed",
    "fixture": "CoRal-project__coral-v3_train_0000002.wav",
    "method": "Injected three copies with one-second pauses into the actual browser ASR worker",
    "provisional_results_before_stop": 3,
    "final_vad_results_before_stop": 3,
    "first_preview_after_audio_seconds": 3.3,
    "final_word_timestamps_present": true,
    "capture_drain_acknowledgement_unit_test": "passed",
    "full_demo_ui_with_synthetic_microphone": {
      "status": "passed",
      "preview_visible_while_speech_detected": true,
      "final_punctuated_transcript_visible_before_stop": true,
      "stop_completes_with_transcript_retained": true
    },
    "limitations": [
      "Browser speech used a synthetic MediaStream rather than a physical microphone",
      "The VAD maximum speech duration is a soft limit, so uninterrupted speech is provisional until a pause or stop",
      "Physical desktop and mobile microphone checks remain"
    ]
  },
  "long_form_replay_20260925": {
    "status": "completed-with-remaining-recognition-gaps",
    "method": "User-provided Danish telephone recording converted to 16 kHz mono and queued through the actual browser WASM VAD and ASR worker; final segments then processed by the browser PnC worker",
    "audio_seconds": 774.27,
    "browser_worker_processing_seconds": 228.1,
    "final_segments": 113,
    "provisional_updates": 175,
    "worker_errors": 0,
    "overlapping_final_segments": 0,
    "raw_words": 1736,
    "pnc_preserved_segment_timing_and_word_count": true,
    "pnc_processing_seconds": 1.6,
    "quality_reference": "ElevenLabs transcript in the supplied Markdown, treated as gold for this evaluation",
    "reference_words": 2316,
    "normalization": "Speaker labels and punctuation removed; Unicode word tokens lowercased",
    "wer_before_gap_recovery": 0.3735,
    "wer_after_gap_recovery": 0.3549,
    "final_substitutions": 200,
    "final_deletions": 601,
    "final_insertions": 21,
    "remaining_issue": "One 16-second interval remained undertranscribed; this replay does not prove complete speech capture or physical-microphone behavior."
  },
  "sparse_boundary_retry_20260925": {
    "status": "local-wasm-replay-passed",
    "method": "Retry browser ASR on VAD segments at least 5 seconds long with fewer than 2.5 words per second, shifting the raw-audio decode window by plus and minus 64 samples (4 ms); select a retry only when it yields at least 3 more words",
    "audio_seconds": 774.27,
    "browser_worker_processing_seconds": 280.4,
    "baseline_processing_seconds": 228.1,
    "final_segments": 113,
    "provisional_updates": 175,
    "worker_errors": 0,
    "changed_segments": 8,
    "raw_words": 1841,
    "pnc_preserved_segment_timing_and_word_count": true,
    "pnc_processing_seconds": 1.79,
    "quality_reference": "ElevenLabs transcript in the supplied Markdown, treated as gold for this evaluation",
    "wer_before_retry": 0.3549,
    "wer_after_retry": 0.3174,
    "final_substitutions": 212,
    "final_deletions": 499,
    "final_insertions": 24,
    "cross_service_probe": {
      "source": "transcription-api-service phase vote and comfort-noise findings",
      "strict_majority_vote_method": "Offline vote over baseline and plus/minus 4 ms browser decodes for the 17 sparse segments",
      "strict_majority_vote_wer": 0.3390,
      "changed_segments_overlapping_digital_silence_runs": 0,
      "digital_silence_definition": "Absolute amplitude at most 0.0001 for at least 0.2 seconds",
      "adopted": false
    },
    "limitations": [
      "A 16-second interval remains undertranscribed",
      "One recording does not establish the retry rule's accuracy on other speakers or devices",
      "Accelerated file replay does not measure live microphone latency or mobile behavior"
    ]
  },
  "checks": {
    "wasm_initialization": "passed",
    "model_sha256_verification": "passed",
    "localhost_preview_server": "passed",
    "microphone_live_speech": {
      "status": "passed-with-real-speech-through-synthetic-media-stream",
      "word_count": 12,
      "vad_segment_emitted": true,
      "pnc_output": "Ja, sådan er det jo kraftigt, og sådan et metal, ikke ansigt."
    },
    "microphone_multi_chunk_speech": {
      "status": "passed-with-real-speech-through-synthetic-media-stream",
      "audio_seconds": 20.53,
      "segments_before_stop": 5,
      "segments_after_stop": 5,
      "explicit_inter_word_spacing": true,
      "text_stays_below_orb_controls": true,
      "toolbar_stays_below_transcript": true,
      "stop_page_errors": 0
    },
    "microphone_onset_preroll": {
      "status": "passed-with-real-speech-through-synthetic-media-stream",
      "model_revision": "aff2c6e39817e66186a0aa0b3cc8d78fc1ea68ad",
      "fixture": "CoRal-project__coral-v3_train_0000002.wav",
      "runs": 2,
      "opening_word": "nu",
      "opening_word_preserved_runs": 2,
      "first_word_start_seconds": 0.0,
      "second_session_zero_based": true,
      "page_errors": 0,
      "pnc_enabled": false
    },
    "microphone_one_word": {
      "status": "passed-with-real-speech-through-synthetic-media-stream",
      "model_revision": "aff2c6e39817e66186a0aa0b3cc8d78fc1ea68ad",
      "fixture_durations_seconds": [0.47, 0.62, 0.82],
      "expected_word": "ja",
      "recognized_before_stop": ["ja", "ja", "ja"],
      "minimum_speech_seconds": 0.12
    },
    "pnc_export_regression": {
      "status": "passed",
      "source_words": 176,
      "output_words": 176,
      "lowercase_after_sentence_stop": false,
      "malformed_mid_segment_stops_removed": true,
      "cross_segment_sentence_starts_capitalized": true,
      "word_timestamps_retained": true
    },
    "v5_browser_wasm_smoke": {
      "status": "passed-local-private-preview",
      "model_revision": "e37e698805c538e447c25062ab13fa862462ee9e",
      "int8_bundle_bytes": 135307058,
      "current_matrix_mean_wer": 12.945774,
      "one_word_fixture_seconds": 0.47,
      "one_word_output": "ja",
      "sentence_output": "nu tager du det her billede for det er lidt mærkeligt",
      "native_timestamps_present": true,
      "resource_errors": 0
    },
    "conversation_history": {
      "status": "passed",
      "storage": "localStorage",
      "maximum_sessions": 20,
      "persists_across_reload": true,
      "reopens_saved_transcript": true,
      "individual_delete": true,
      "clear_all": true,
      "stores_audio": false,
      "mobile_dialog_fits_viewport": true
    },
    "pnc_browser_parity_probe": {
      "status": "passed",
      "input": "hej mit navn er emil hvordan går det",
      "browser_and_python_output": "Hej, mit navn er Emil, hvordan går det?",
      "word_count": 8,
      "timestamps_preserved": true
    },
    "rye_voice_interaction": {
      "orb_shader_parity": "passed",
      "mobile_stage_css_pixels": 218,
      "mobile_orb_css_pixels": 192,
      "shader_colors": ["#27AE60", "#8CBFFF", "#FFAAAD"],
      "mobile_orb_lift_pixels": 186,
      "mobile_idle_label_gap_pixels": 42,
      "animated_words": 12,
      "mobile_orb_transcript_overlap": false,
      "desktop_orb_transcript_overlap": false,
      "desktop_transcript_toolbar_overlap": false
    },
    "warmed_decoder_performance_harness": {
      "device": "Apple M1 Pro",
      "browser": "Chromium 148 / Electron 42",
      "initial_model_download_and_load_excluded": true,
      "note": "Measured through the same offline decoder before file upload was removed from the public UI.",
      "short_clip": {
        "audio_seconds": 3.12,
        "runs": 5,
        "median_processing_seconds": 0.526,
        "median_rtf": 0.16859,
        "times_realtime": 5.93
      },
      "long_clip": {
        "audio_seconds": 17.86,
        "runs": 3,
        "median_processing_seconds": 2.099,
        "median_rtf": 0.11753,
        "times_realtime": 8.51
      }
    },
    "mobile_390x844_horizontal_overflow": false,
    "desktop_1440x1000_horizontal_overflow": false,
    "mobile_dialogs_fit_viewport": true,
    "clean_reload_resource_errors": 0
  },
  "checks_model_revision": "7cb1747204bddd558ff449399689d5bb641fadd8",
  "remaining_release_checks": [
    "Load the current Tiny and PnC models directly from their public Hugging Face revisions without interception.",
    "Run the complete current-model WER matrix through WebAssembly; local initialization cannot establish browser parity.",
    "Run microphone speech on physical desktop and mobile devices.",
    "Verify Safari, Chromium, and Firefox memory behavior.",
    "Publish the static Space only after model terms and visibility are final."
  ]
}