stt-gemma_embedding-2: Core ML package (fp16, multifunction), tokenizer, subword 6-gram LM

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

NOTICE

@@ -0,0 +1,44 @@
+V1 speech recognition on Core ML
+Copyright Harshit B.
+
+The repository as a whole is licensed under the Apache License, Version 2.0 (LICENSE).
+
+1. Core ML package (v1.mlpackage), derived from google/embeddinggemma-2
+
+   It contains layers 0-11 of the audio tower of EmbeddingGemma 2 by Google DeepMind,
+   https://huggingface.co/google/embeddinggemma-2 (revision 914f7f89142e33e77833254d9c9b90c3cef7303b),
+   licensed under the Apache License, Version 2.0
+   (https://ai.google.dev/gemma/docs/gemma_4_license).
+
+   Modifications made by Harshit B.:
+   - the audio tower truncated after layer 11 and converted to Core ML ML programs,
+     one function per fixed input window (5, 10, 20, 35 s) sharing one set of weights;
+   - blocked local attention rewritten as dense banded attention (identical in fp32);
+   - RMSNorm inputs scaled by powers of two so fp16 norms on the CPU / GPU do not
+     overflow, and constant padding fill (identical in fp32);
+   - log-softmax rewritten in a numerically stable form; static shapes;
+   - a per-frame int8 quantize/dequantize step, standardization and a CTC head appended;
+   - weights stored in float16.
+
+   The upstream model card states that deployments must adhere to the Gemma Prohibited
+   Use Policy: https://ai.google.dev/gemma/prohibited_use_policy
+
+   config.json embeds the upstream preprocessor_config.json unchanged.
+
+2. CTC head (inside v1.mlpackage) and tokenizer (spm_unigram256.model)
+
+   Trained by Harshit B. on the LibriSpeech ASR corpus, OpenSLR SLR12,
+   https://www.openslr.org/12/ (license stated there: CC BY 4.0).
+   V. Panayotov, G. Chen, D. Povey and S. Khudanpur, "LibriSpeech: an ASR corpus based
+   on public domain audio books", ICASSP 2015.
+
+3. Language model (sp256_6gram.bin)
+
+   Built with KenLM (https://github.com/kpu/kenlm; no KenLM code is included) from
+   librispeech-lm-norm.txt.gz of "LibriSpeech language models, vocabulary and G2P models",
+   OpenSLR SLR11, https://www.openslr.org/11/ (license stated there: Public domain).
+
+4. Decoder (not included; used by the usage example)
+
+   ctc-beam-decoder 0.2.0, https://pypi.org/project/ctc-beam-decoder/,
+   Apache License, Version 2.0.

README.md

@@ -0,0 +1,271 @@
+---
+language:
+- en
+license: apache-2.0
+library_name: coreml
+pipeline_tag: automatic-speech-recognition
+base_model: google/embeddinggemma-2
+datasets:
+- openslr/librispeech_asr
+tags:
+- speech-recognition
+- streaming
+- ctc
+- coreml
+- apple-neural-engine
+- kenlm
+- embeddinggemma
+model-index:
+- name: stt-gemma_embedding-2
+  results:
+  - task:
+      type: automatic-speech-recognition
+      name: Automatic Speech Recognition
+    dataset:
+      name: LibriSpeech (clean)
+      type: openslr/librispeech_asr
+      config: clean
+      split: test
+      args:
+        language: en
+    metrics:
+    - type: wer
+      value: 4.18
+      name: Test WER (Core ML fp16, beam search + subword 6-gram LM)
+  - task:
+      type: automatic-speech-recognition
+      name: Automatic Speech Recognition
+    dataset:
+      name: LibriSpeech (other)
+      type: openslr/librispeech_asr
+      config: other
+      split: test
+      args:
+        language: en
+    metrics:
+    - type: wer
+      value: 10.71
+      name: Test WER (Core ML fp16, beam search + subword 6-gram LM)
+---
+
+# stt-gemma_embedding-2
+
+**English speech recognition for the Apple Neural Engine, built by training a 13M-parameter CTC head on top of the frozen audio encoder of Google's [EmbeddingGemma 2](https://huggingface.co/google/embeddinggemma-2).**
+
+stt-gemma_embedding-2 keeps EmbeddingGemma 2's 305M-parameter audio encoder completely frozen and trains only a small causal head that turns the encoder's frames into text. The encoder never looks at future audio and the head looks ahead by just 160 ms, so the model can transcribe while someone is speaking: in streaming mode, words appear a median of **207 ms** after they are spoken. Exported to Core ML in fp16, it runs on the Neural Engine. Offline, the full pipeline (encoder, head, and beam search with a language model) transcribes at **233× real time** on a MacBook Pro with an M4 Pro; in streaming mode, each 160 ms step takes 5.6 ms of compute.
+
+| | test-clean | test-other | Streaming latency (median / p90) | Offline speed (M4 Pro) |
+|---|---|---|---|---|
+| **stt-gemma_embedding-2, beam search + subword 6-gram LM** | **4.18%** | **10.71%** | 207 / 287 ms | **233× real time** |
+
+Word error rate on LibriSpeech, corpus-level, of the released Core ML package. Latency is from the end of a spoken word to its final text on screen, measured on dev-clean. See [Evaluation protocol](#evaluation-protocol) for how every number was measured.
+
+## Why this model
+
+Many fast on-device speech recognizers either work on fixed windows of audio, which rules out true streaming, or train a large encoder from scratch. This model takes a different route: it reuses an audio encoder that Google has already trained at scale, freezes it, and trains only what is needed to read text out of it. The encoder is EmbeddingGemma 2's audio encoder (a Gemma 4 audio tower, `gemma4_audio`), which also makes it possible to produce a transcript and a search embedding from a single encoder pass.
+
+The trainable part is small (13.3M parameters), it was trained on a single laptop, and it runs on the Neural Engine, leaving the GPU free for other work.
+
+## Results
+
+### Accuracy
+
+WER in percent. Decoder settings were tuned on dev-clean only; the test sets were each evaluated once with the final settings.
+
+| Pipeline | dev-clean | dev-other | test-clean | test-other |
+|---|---|---|---|---|
+| Reference (PyTorch), greedy, no LM | 5.92 | 15.47 | 6.30 | 15.61 |
+| Reference (PyTorch), beam 32 + subword 6-gram LM | 3.73 | 10.47 | 4.14 | 10.70 |
+| **This release (Core ML fp16), beam 32 + subword 6-gram LM** | **3.74** | **10.50** | **4.18** | **10.71** |
+
+The released Core ML package reproduces the reference beam + LM results to within 0.04 points (dev-clean +0.011, dev-other +0.026, test-clean +0.038, test-other +0.009). It gives the same dev-clean WER under all three Core ML compute settings: 3.737% on the Neural Engine, 3.746% on the GPU, 3.731% on the CPU only.
+
+### Streaming
+
+The streaming build processes 160 ms of audio per step and keeps the encoder's and head's state between steps; the decoder re-decodes the open segment after every step. Its feature extraction is bitwise identical to the offline pipeline, and 100% of its operations run on the Neural Engine. The streaming build is not part of this repository yet; it lives in the code repository.
+
+| Split | Latency median / p90 | WER, streaming | WER, reference | Transcripts identical to reference | Revised words |
+|---|---|---|---|---|---|
+| dev-clean | 207 / 287 ms | 3.731% | 3.726% | 2,617 / 2,703 | 4.6% |
+| dev-other | 207 / 288 ms | 10.479% | 10.470% | 2,561 / 2,864 | 10.2% |
+
+Latency is measured from the end of a word in the audio to the moment its final text is displayed, and includes the head's 160 ms lookahead, the wait for each 160 ms chunk, and compute. Every decoded word is shown as soon as it appears; "revised words" counts displayed words that are later replaced by a different word as more audio arrives (a word growing while it is spelled out does not count). Each 160 ms step takes 5.62 ms of compute (median; p99 6.03 ms), about 3.5% of real time.
+
+### Speed
+
+Offline transcription of this package, measured on a MacBook Pro (M4 Pro, 24 GB) on dev-clean, single stream, after a 20-file warm-up, with model loading excluded.
+
+| Component | Hardware | Real-time factor |
+|---|---|---|
+| End to end (16 kHz audio in memory → text) | Neural Engine + CPU | 233.3× |
+| Encoder + head (Core ML, fp16) | Neural Engine | 253.9× |
+| Beam search + language model | CPU | 8,452× |
+
+| Compute units | End to end | Peak memory | Load, first / later |
+|---|---|---|---|
+| CPU_AND_NE (recommended) | 233.3× | 3.3 GB | 54.6 s / 3.3 s |
+| CPU_AND_GPU | 194.2× | 4.7 GB | 11.3 s / 3.5 s |
+| CPU_ONLY | 111.7× | 5.2 GB | 7.5 s / 2.7 s |
+
+The first load on the Neural Engine includes the device compile; later loads reuse the compiled copy.
+
+### Comparison with Phonon-2
+
+For context, we ran [FluidInference/phonon-2-coreml](https://huggingface.co/FluidInference/phonon-2-coreml), a strong Core ML speech recognizer for Apple silicon, on the same Mac, scored it with the same scorer, and timed it with the same method.
+
+| Model | test-clean | test-other | Offline speed (M4 Pro) |
+|---|---|---|---|
+| Phonon-2 (FluidAudio, Neural Engine) | **2.47%** | **4.66%** | 187× (test-clean) |
+| stt-gemma_embedding-2 (beam + LM, Neural Engine) | 4.18% | 10.71% | **233×** (dev-clean) |
+
+Phonon-2 is clearly more accurate, especially on harder audio. This table shows the trade-off rather than claiming better accuracy: stt-gemma_embedding-2 streams with about 0.2 s of latency and is somewhat faster offline, at a higher error rate. Phonon-2 was run with FluidAudio at commit 90c1b07e; because it outputs punctuation and casing, punctuation other than apostrophes was mapped to spaces before scoring.
+
+## How it works
+
+```
+16 kHz mono audio
+  → log-mel features (128 bins, 100 frames per second)
+  → EmbeddingGemma 2 audio encoder, frozen
+      12 layers, 305M parameters, causal (no right context)
+  → output of the 12th layer: 1024 numbers per frame, 25 frames per second
+  → causal CTC head, trained
+      4 blocks, 13.3M parameters, 160 ms lookahead
+  → scores for 256 SentencePiece tokens + a CTC blank, every 40 ms
+  → CTC beam search with a subword 6-gram language model
+  → text
+```
+
+The head first rounds each encoder frame to int8 (matching how the training features were cached), standardizes it with statistics from the training set, and projects it from 1024 to 512 dimensions. Each of its four blocks then applies masked self-attention (8 heads, up to 64 past frames, or 2.56 s), a causal depthwise convolution (kernel size 5) and a 2048-wide feed-forward layer, each with a residual connection. Only the last block's attention may see future frames, 4 of them (160 ms), so the head's total lookahead is exactly 160 ms; this was checked when the head was trained and again on the Core ML package. The encoder itself has no lookahead: changing the audio after any point never changes its output before that point.
+
+The output is CTC over a 256-piece SentencePiece unigram vocabulary. A beam search ([ctc-beam-decoder](https://pypi.org/project/ctc-beam-decoder/), a C++ CTC prefix beam search that reproduces pyctcdecode's behaviour) combines the head's scores with a KenLM 6-gram over the same SentencePiece pieces.
+
+The Core ML package is the offline build: one package with shared weights and four functions for input windows of 5, 10, 20 and 35 s (`win_5s` … `win_35s`); longer audio is split at pauses.
+
+## Usage
+
+Requirements: a Mac with Apple silicon and macOS 15 or later, Python 3.12, `coremltools==9.0`, `transformers==5.19.0`, `numpy==2.5.4`, `sentencepiece==0.2.2`, `soundfile==0.14.0`, `ctc-beam-decoder==0.2.0`, `huggingface_hub`.
+
+```python
+import json, os
+import numpy as np
+import soundfile as sf
+import coremltools as ct
+import sentencepiece as spm
+import transformers
+from ctc_beam_decoder import BeamSearchDecoder
+from huggingface_hub import snapshot_download
+
+root = snapshot_download("harshit2312/stt-gemma_embedding-2")
+cfg = json.load(open(f"{root}/config.json"))
+
+# 1. Mel features (CPU), exactly as in training.
+fe_cfg = cfg["feature_extractor"]
+fe = getattr(transformers, fe_cfg["class"]).from_dict(dict(fe_cfg["preprocessor_config"]))
+wav, sr = sf.read("speech.flac", dtype="float32")         # 16 kHz mono
+assert sr == 16000 and wav.ndim == 1
+f = fe([wav], sampling_rate=16000, padding="longest", truncation=False, return_tensors="np")
+n = int(f["input_features_mask"][0].sum())
+mel = np.asarray(f["input_features"][0, :n], dtype=np.float32)
+
+# 2. Compile the package once and keep the compiled model.
+compiled = os.path.expanduser("~/Library/Caches/stt-gemma_embedding-2/v1.mlmodelc")
+if not os.path.isdir(compiled):
+    os.makedirs(os.path.dirname(compiled), exist_ok=True)
+    ct.utils.compile_model(f"{root}/{cfg['package']['file']}", compiled)
+
+# 3. Encoder + CTC head in the smallest window that fits (<= 35 s here).
+win = min((w for w in cfg["windows"] if w["mel_frames"] >= n), key=lambda w: w["mel_frames"])
+model = ct.models.CompiledMLModel(compiled, compute_units=ct.ComputeUnit.CPU_AND_NE,   # or CPU_AND_GPU / CPU_ONLY
+                                  function_name=win["function"])
+x = np.zeros((1, win["mel_frames"], 128), np.float32)
+x[0, :n] = mel
+out = model.predict({"mel": x, "valid_len": np.array([n], np.int32)})
+log_probs = np.array(out["log_probs"], np.float32)[0, : -(-n // 4)]   # [frames, 257]
+
+# 4. Beam search + subword 6-gram: every piece is an LM "word" t<id>.
+d = cfg["decoder"]
+sp = spm.SentencePieceProcessor(model_file=f"{root}/{cfg['vocab']['tokenizer']}")
+pieces = [f"t{i}" for i in range(sp.get_piece_size())]
+dec = BeamSearchDecoder([f"\u2581{p}" for p in pieces], kenlm_model_path=f"{root}/{d['lm']}",
+                        unigrams=pieces, alpha=d["alpha"], beta=d["beta"],
+                        unk_score_offset=d["unk_score_offset"], score_boundary=d["lm_score_boundary"])
+best = dec.decode_beams(log_probs, beam_width=d["beam"], beam_prune_logp=d["beam_prune_logp"],
+                        token_min_logp=d["token_min_logp"], prune_history=d["prune_history"])[0][0]
+print(" ".join(sp.decode([int(w[1:]) for w in best.split()]).lower().split()))
+```
+
+**Compile once.** `ct.models.MLModel("v1.mlpackage")` compiles the package to a new temporary `.mlmodelc` on every load. Compile it once, as in step 2, and load the compiled model with `CompiledMLModel`; one `.mlmodelc` holds all four functions, picked with `function_name=`.
+
+**Long-running processes: keep the input arrays alive.** Core ML releases a prediction's input arrays later, on its own queue; if Python frees them first, a process that keeps predicting can crash. Allocate one `mel` / `valid_len` pair per window, keep it for the model's lifetime, overwrite it in place for each call (float32 / int32; fp16 inputs are cached by Core ML), serialize calls with a lock if you use threads, and copy the outputs before the next call.
+
+## Files
+
+| File | Contents |
+|---|---|
+| `v1.mlpackage` | Core ML package: frozen encoder + trained head, fp16, functions `win_5s`, `win_10s`, `win_20s`, `win_35s` with shared weights |
+| `spm_unigram256.model` | SentencePiece unigram tokenizer, 256 pieces (blank = id 256) |
+| `sp256_6gram.bin` | KenLM 6-gram over the tokenizer's pieces, 8-bit quantized trie (762 MB) |
+| `config.json` | Windows → functions, inputs / outputs, feature extractor settings, decoder settings (beam 32, α = 0.8, β = 1, pruning), minimum OS |
+| `LICENSE`, `NOTICE` | Apache License 2.0; upstream attribution, modifications and data sources |
+| `SHA256SUMS` | SHA-256 of every file (`shasum -a 256 -c SHA256SUMS`) |
+
+## Training
+
+**Data.** LibriSpeech 960 h (train-clean-100, train-clean-360 and train-other-500; 281,241 utterances), with lowercased transcripts.
+
+**Features.** The frozen encoder was run once over all training audio (in bf16 on the GPU), and the output of its 12th layer was cached on disk as per-frame int8 (about 91 GB for all splits); the 961 h of training audio took 4.1 hours, at 234× real time.
+
+**Head.** 15 epochs on the cached features with CTC loss, AdamW (learning rate 1e-3, weight decay 1e-4), 1,000 warmup steps followed by cosine decay, dropout 0.1, and batches of up to 12,000 frames. Training took about 11 hours on one MacBook Pro M4 Pro (24 GB). The released head is from epoch 15, the best on dev-clean (5.92% greedy WER).
+
+**Language model.** A KenLM 6-gram over SentencePiece pieces, trained only on the OpenSLR `librispeech-lm-norm` corpus (no dev or test transcripts; checked by searching the corpus for dev and test sentences) and quantized to 8 bits.
+
+**Decoder settings.** Beam width 32, LM weight α = 0.8 and word bonus β = 1, tuned on dev-clean only.
+
+## Evaluation protocol
+
+WER is corpus-level: total word edits divided by total reference words. Before scoring, text is lowercased and whitespace is collapsed (for Phonon-2, punctuation other than apostrophes was also mapped to spaces). Offline speed is the audio duration divided by the processing time, with 16 kHz samples already in memory, model loading excluded, a 20-file warm-up, and a single stream. Streaming latency runs each utterance through the streaming build 160 ms at a time and measures, for every word, the time from the word's end in the audio (from the CTC alignment of the final transcript) to the moment the displayed text up to that word stops changing.
+
+## Limitations
+
+This release is English only and was trained only on read audiobook speech. Expect noticeably higher error rates on conversational, accented, noisy or far-field audio; its test-other error rate is about 2.3 times Phonon-2's, so the gap to stronger models widens on harder audio. In streaming mode, harder audio also means more on-screen corrections: about 10% of displayed words are revised on dev-other, against 4.6% on dev-clean. Names, numbers and rare words are weak points, since both the tokenizer and the language model come from book text, and the output is lowercase without punctuation. The language model needs about 0.8 GB of memory, so the default setup targets desktop Macs rather than phones. The model should not be used for high-stakes transcription (legal, medical or similar) without human review.
+
+## License and attribution
+
+This model includes, and depends on, the audio encoder of [google/embeddinggemma-2](https://huggingface.co/google/embeddinggemma-2), redistributed under that model's Apache 2.0 license; deployments must also follow the [Gemma Prohibited Use Policy](https://ai.google.dev/gemma/prohibited_use_policy). The trained head, tokenizer, language model and code are released under the license declared above. Training data: [LibriSpeech](https://www.openslr.org/12) (CC BY 4.0). Language-model text: the LibriSpeech LM corpus ([OpenSLR 11](https://www.openslr.org/11), public domain). The decoder, ctc-beam-decoder (Apache-2.0), links KenLM (LGPL) dynamically.
+
+This project is independent and is not affiliated with or endorsed by Google or FluidInference.
+
+## Citation
+
+```bibtex
+@misc{stt_gemma_embedding_2_2026,
+  title  = {stt-gemma\_embedding-2: streaming CTC speech recognition on a frozen EmbeddingGemma 2 audio encoder},
+  author = {Harshit B.},
+  year   = {2026},
+  url    = {https://huggingface.co/harshit2312/stt-gemma_embedding-2}
+}
+
+@inproceedings{panayotov2015librispeech,
+  title     = {Librispeech: an {ASR} corpus based on public domain audio books},
+  author    = {Panayotov, Vassil and Chen, Guoguo and Povey, Daniel and Khudanpur, Sanjeev},
+  booktitle = {2015 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
+  pages     = {5206--5210},
+  year      = {2015}
+}
+
+@inproceedings{graves2006ctc,
+  title     = {Connectionist temporal classification: labelling unsegmented sequence data with recurrent neural networks},
+  author    = {Graves, Alex and Fern{\'a}ndez, Santiago and Gomez, Faustino and Schmidhuber, J{\"u}rgen},
+  booktitle = {Proceedings of the 23rd International Conference on Machine Learning},
+  pages     = {369--376},
+  year      = {2006}
+}
+
+@inproceedings{heafield2011kenlm,
+  title     = {{KenLM}: Faster and smaller language model queries},
+  author    = {Heafield, Kenneth},
+  booktitle = {Proceedings of the Sixth Workshop on Statistical Machine Translation},
+  pages     = {187--197},
+  year      = {2011}
+}
+```

SHA256SUMS

@@ -0,0 +1,9 @@
+339dff58281aa2956050b09204c5f74f3acfef8b24d70d7f487fc3593681f402  ./config.json
+cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30  ./LICENSE
+0f3fef6d25d4c3121b7c68d58052adda26c5128adb9a831230c6e91eef528bab  ./NOTICE
+dbab7fdfbd5f5cb579d44d490a455e3278f85b6aac0ef023a367eb0368826ffa  ./README.md
+381c408f36fb5ea3cbece7eba408deb2b2320f0fe4679fd45d6bd394395fb371  ./sp256_6gram.bin
+50002ddf56a79f19c6598240e684cf065a4a94b1fc88c5aff25b05962cccfe05  ./spm_unigram256.model
+de945eb076ccf0a4db01847e224ed561a49d8d4c7febb72e1d039455d5a107d1  ./v1.mlpackage/Data/com.apple.CoreML/model.mlmodel
+075ca55dc69f1688d2d6dab6cea3225e5164f32b9a8ab0221cd5a586df0d3909  ./v1.mlpackage/Data/com.apple.CoreML/weights/weight.bin
+8a1768f199c7aba04043bd8254bd54fd2fa12867856b9cc6a4b04d77b410491d  ./v1.mlpackage/Manifest.json

config.json

@@ -0,0 +1,172 @@
+{
+  "name": "V1: EmbeddingGemma 2 audio tower (layers 0-11) + causal CTC head, Core ML",
+  "version": "v1",
+  "license": "apache-2.0",
+  "base_model": {
+    "id": "google/embeddinggemma-2",
+    "revision": "914f7f89142e33e77833254d9c9b90c3cef7303b",
+    "license": "apache-2.0",
+    "component": "audio_tower",
+    "layers": [
+      0,
+      11
+    ]
+  },
+  "package": {
+    "file": "v1.mlpackage",
+    "format": "mlprogram (multifunction, shared weights)",
+    "precision": "fp16",
+    "source_build": "fp16safe/fp16",
+    "specification_version": 9,
+    "minimum_os": "macOS 15 / iOS 18",
+    "compute_units": {
+      "recommended": "CPU_AND_NE",
+      "supported": [
+        "CPU_AND_NE",
+        "CPU_AND_GPU",
+        "CPU_ONLY"
+      ]
+    },
+    "default_function": "win_10s"
+  },
+  "windows": [
+    {
+      "function": "win_5s",
+      "seconds": 5.0,
+      "mel_frames": 500,
+      "output_frames": 125
+    },
+    {
+      "function": "win_10s",
+      "seconds": 10.0,
+      "mel_frames": 1000,
+      "output_frames": 250
+    },
+    {
+      "function": "win_20s",
+      "seconds": 20.0,
+      "mel_frames": 2000,
+      "output_frames": 500
+    },
+    {
+      "function": "win_35s",
+      "seconds": 35.0,
+      "mel_frames": 3500,
+      "output_frames": 875
+    }
+  ],
+  "window_choice": "smallest window with mel_frames >= valid mel frames",
+  "inputs": [
+    {
+      "name": "mel",
+      "shape": [
+        "1",
+        "T_mel",
+        "128"
+      ],
+      "dtype": "float32",
+      "description": "log-mel frames (see feature_extractor), zero-padded to the function's window"
+    },
+    {
+      "name": "valid_len",
+      "shape": [
+        "1"
+      ],
+      "dtype": "int32",
+      "description": "number of valid mel frames"
+    }
+  ],
+  "outputs": [
+    {
+      "name": "log_probs",
+      "shape": [
+        "1",
+        "T_mel/4",
+        "257"
+      ],
+      "dtype": "float16",
+      "description": "CTC log-probabilities (natural log), 256 pieces + blank"
+    },
+    {
+      "name": "l11",
+      "shape": [
+        "1",
+        "T_mel/4",
+        "1024"
+      ],
+      "dtype": "float16",
+      "description": "layer-11 features after the int8 round trip"
+    }
+  ],
+  "valid_output_frames": "ceil(valid_len / 4)",
+  "feature_extractor": {
+    "class": "Gemma4AudioFeatureExtractor",
+    "preprocessor_config": {
+      "dither": 0.0,
+      "feature_extractor_type": "Gemma4AudioFeatureExtractor",
+      "feature_size": 128,
+      "fft_length": 512,
+      "fft_overdrive": false,
+      "frame_length": 320,
+      "hop_length": 160,
+      "input_scale_factor": 1.0,
+      "max_frequency": 8000.0,
+      "mel_floor": 0.001,
+      "min_frequency": 0.0,
+      "padding_side": "right",
+      "padding_value": 0.0,
+      "per_bin_mean": null,
+      "per_bin_stddev": null,
+      "preemphasis": 0.0,
+      "preemphasis_htk_flavor": true,
+      "return_attention_mask": true,
+      "sampling_rate": 16000
+    },
+    "source": "google/embeddinggemma-2@914f7f89142e33e77833254d9c9b90c3cef7303b/preprocessor_config.json (unchanged)",
+    "call": {
+      "sampling_rate": 16000,
+      "padding": "longest",
+      "truncation": false
+    },
+    "valid_mel_frames": "sum(input_features_mask)",
+    "mel_frame_rate_hz": 100
+  },
+  "frame_rate_hz": 25,
+  "subsampling": 4,
+  "lookahead_ms": 160,
+  "encoder_lookahead_ms": 0,
+  "ctc_head": {
+    "params": 13283585,
+    "future_frames": 4,
+    "causal_conv": true
+  },
+  "vocab": {
+    "tokenizer": "spm_unigram256.model",
+    "type": "sentencepiece-unigram",
+    "size": 256,
+    "blank_id": 256,
+    "num_classes": 257
+  },
+  "decoder": {
+    "library": "ctc-beam-decoder",
+    "library_version": "0.2.0",
+    "lm": "sp256_6gram.bin",
+    "lm_order": 6,
+    "piece_word_prefix": "t",
+    "alpha": 0.8,
+    "beta": 1.0,
+    "beam": 32,
+    "units": "piece",
+    "beam_prune_logp": -10.0,
+    "token_min_logp": -5.0,
+    "unk_score_offset": -10.0,
+    "lm_score_boundary": true,
+    "prune_history": true
+  },
+  "long_audio": {
+    "max_mel_frames": 3500,
+    "left_context_s": 2.0,
+    "cut": "quietest point in the last 10 s (code repo: coreml/transcribe.py)"
+  },
+  "text_normalization": "lowercase, collapse whitespace"
+}

sp256_6gram.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:381c408f36fb5ea3cbece7eba408deb2b2320f0fe4679fd45d6bd394395fb371
+size 761681354

spm_unigram256.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50002ddf56a79f19c6598240e684cf065a4a94b1fc88c5aff25b05962cccfe05
+size 243583

v1.mlpackage/Data/com.apple.CoreML/model.mlmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de945eb076ccf0a4db01847e224ed561a49d8d4c7febb72e1d039455d5a107d1
+size 5701462

v1.mlpackage/Data/com.apple.CoreML/weights/weight.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:075ca55dc69f1688d2d6dab6cea3225e5164f32b9a8ab0221cd5a586df0d3909
+size 646897778

v1.mlpackage/Manifest.json

@@ -0,0 +1,18 @@
+{
+    "fileFormatVersion": "1.0.0",
+    "itemInfoEntries": {
+        "40950C18-BA78-47BF-8E28-EA22171D289B": {
+            "author": "com.apple.CoreML",
+            "description": "CoreML Model Weights",
+            "name": "weights",
+            "path": "com.apple.CoreML/weights"
+        },
+        "A976F9F4-38B8-441C-AF08-A672DFE4B336": {
+            "author": "com.apple.CoreML",
+            "description": "CoreML Model Specification",
+            "name": "model.mlmodel",
+            "path": "com.apple.CoreML/model.mlmodel"
+        }
+    },
+    "rootModelIdentifier": "A976F9F4-38B8-441C-AF08-A672DFE4B336"
+}