alexshah commited on
Commit
e97985b
·
1 Parent(s): 92fb9a5

minor fixes

Browse files
Files changed (2) hide show
  1. app.js +1 -1
  2. index.html +22 -44
app.js CHANGED
@@ -4,7 +4,7 @@ const DATASET_LABELS = {
4
  fleurs: "FLEURS",
5
  poems: "Poems",
6
  movies: "Movies",
7
- infocom_may_26: "Infocom",
8
  };
9
 
10
  const METRICS = ["WER", "CER", "WER-norm", "CER-norm"];
 
4
  fleurs: "FLEURS",
5
  poems: "Poems",
6
  movies: "Movies",
7
+ infocom_may_26: "News",
8
  };
9
 
10
  const METRICS = ["WER", "CER", "WER-norm", "CER-norm"];
index.html CHANGED
@@ -40,11 +40,9 @@
40
  </button>
41
  </div>
42
  <div class="hero-copy">
43
- <h1><span class="heading-flag" role="img" aria-label="Armenian flag">🇦🇲</span> ArmBench-ASR: Armenian Speech Recognition in the Real World</h1>
44
  <p class="lede">
45
  <span class="version-badge">v0.1</span>
46
- A clear, comparable view of Armenian ASR performance across public and
47
- private evaluation sets.
48
  </p>
49
  </div>
50
  <div class="hero-art" aria-hidden="true">
@@ -113,39 +111,37 @@
113
  <p>10,113 clips and 20.7 hours of speech spanning read, literary, cinematic, and conversational Armenian. Every model is evaluated on the same fixed test material.</p>
114
  </div>
115
  <div class="dataset-grid">
116
- <article class="dataset-card accent">
117
- <div class="dataset-card-top"><span>01</span><span class="dataset-tag">Aggregate</span></div>
118
- <h3>Combined</h3>
119
- <p>A pooled view across all five sources, balancing large public test sets with smaller domain-focused evaluation data.</p>
120
- <dl class="dataset-meta"><div><dt>Samples</dt><dd>10,113</dd></div><div><dt>Audio</dt><dd>20.7 h</dd></div><div><dt>Sources</dt><dd>5 datasets</dd></div></dl>
121
- </article>
122
  <article class="dataset-card">
123
- <div class="dataset-card-top"><span>02</span><span class="dataset-tag public">Public</span></div>
124
  <h3><a class="dataset-source" href="https://mozilladatacollective.com/datasets/cmqinsp2y00xsnr07ra8gxfoi" target="_blank" rel="noreferrer">Common Voice 26 <span>↗</span></a></h3>
125
  <p>Mozilla Common Voice Scripted Speech 26.0 for Armenian, using the full <code>hy-AM</code> test split released in June 2026.</p>
126
  <dl class="dataset-meta"><div><dt>Samples</dt><dd>6,892</dd></div><div><dt>Audio</dt><dd>10.8 h</dd></div><div><dt>Style</dt><dd>Read speech</dd></div></dl>
127
  </article>
 
128
  <article class="dataset-card">
129
- <div class="dataset-card-top"><span>03</span><span class="dataset-tag public">Public</span></div>
130
  <h3><a class="dataset-source" href="https://huggingface.co/datasets/google/fleurs" target="_blank" rel="noreferrer">FLEURS <span>↗</span></a></h3>
131
  <p>Google FLEURS Armenian (<code>hy_am</code>) test split. Raw transcriptions retain punctuation and capitalization for strict scoring, with some <a href="https://huggingface.co/datasets/Metric-AI/fleurs-corrections" target="_blank" rel="noreferrer">manual corrections</a> applied.</p>
132
  <dl class="dataset-meta"><div><dt>Samples</dt><dd>932</dd></div><div><dt>Audio</dt><dd>3.0 h</dd></div><div><dt>Style</dt><dd>Read speech</dd></div></dl>
133
  </article>
 
134
  <article class="dataset-card">
135
- <div class="dataset-card-top"><span>04</span><span class="dataset-tag private">Private</span></div>
136
  <h3>Poems</h3>
137
  <p>Read Armenian poetry with background music, literary vocabulary, expressive phrasing, and text structures underrepresented in general ASR sets.</p>
138
  <dl class="dataset-meta"><div><dt>Samples</dt><dd>969</dd></div><div><dt>Audio</dt><dd>3.3 h</dd></div><div><dt>Style</dt><dd>Literary</dd></div></dl>
139
  </article>
 
140
  <article class="dataset-card">
141
- <div class="dataset-card-top"><span>05</span><span class="dataset-tag private">Private</span></div>
142
  <h3>Movies</h3>
143
  <p>Armenian movie dialogue with conversational delivery, produced-media acoustics, and occasional background noise.</p>
144
  <dl class="dataset-meta"><div><dt>Samples</dt><dd>820</dd></div><div><dt>Audio</dt><dd>1.7 h</dd></div><div><dt>Style</dt><dd>Dialogue</dd></div></dl>
145
  </article>
 
146
  <article class="dataset-card">
147
- <div class="dataset-card-top"><span>06</span><span class="dataset-tag private">Private</span></div>
148
- <h3>Infocom</h3>
149
  <p>A compact Armenian news dataset recited by a speaker, designed to evaluate clear narrated speech beyond standard public test sets.</p>
150
  <dl class="dataset-meta"><div><dt>Samples</dt><dd>500</dd></div><div><dt>Audio</dt><dd>1.9 h</dd></div><div><dt>Style</dt><dd>Narrated news</dd></div></dl>
151
  </article>
@@ -179,36 +175,18 @@
179
  <strong>API settings</strong>
180
  <p>Hosted systems use fixed model identifiers and transcription-only prompts. Evaluated Gemini runs use temperature 0, a 1,000-token output cap, and audio limited to 40 seconds. Thinking is disabled or set to the lowest applicable level: budget 0 for Gemini 2.5 Flash, budget 128 for Gemini 2.5 Pro, and <b>MINIMAL</b> for Gemini 3.1 Flash Lite, 3.5 Flash Lite, and 3.6 Flash.</p>
181
  </div>
 
 
 
 
182
 
183
- <section class="method-section">
184
- <div class="section-intro compact">
185
- <h2>Cleaning without hiding mistakes</h2>
186
- <p>Original references and predictions remain auditable. The benchmark derives strict and normalized text from both sides with the same deterministic pipeline, so normalization never changes only one side of a comparison.</p>
187
- </div>
188
- <div class="pipeline-grid">
189
- <article class="pipeline-card">
190
- <span>01</span><h3>Source fidelity</h3>
191
- <p>Predictions are kept verbatim. FLEURS uses raw transcriptions when available, preserving case and punctuation for strict evaluation.</p>
192
- </article>
193
- <article class="pipeline-card">
194
- <span>02</span><h3>Strict preprocessing</h3>
195
- <p>Unicode NFKC, Armenian punctuation, quotes, apostrophes, hyphens, grouped numbers, spacing, trailing segment punctuation, capitalization, and whitespace are made consistent.</p>
196
- </article>
197
- <article class="pipeline-card">
198
- <span>03</span><h3>Normalized scoring</h3>
199
- <p>Text is lowercased; standalone Armenian “Եվ” and hyphenated suffixes are canonicalized; dashes become spaces; Armenian, English, Russian letters and digits are retained. Other punctuation is removed, except separators in recognized <code>HH:MM</code> clock times, which are preserved in Armenian form (<code>12։00</code>).</p>
200
- </article>
201
- </div>
202
- <p class="method-footnote">Both views are published: <b>WER / CER</b> reflect the stricter cleaned transcription, while <b>WER-norm / CER-norm</b> reduce formatting and orthographic noise.</p>
203
- </section>
204
-
205
- <section class="lab-card">
206
- <div>
207
- <h2>Metric AI Lab</h2>
208
- <p>Armenian ASR Benchmark is a Metric AI Lab project. The applied AI research lab builds production systems across custom language and vision-language models, multilingual voice agents, Physical AI, retrieval, and analytics—while publishing frontier research and open benchmarks.</p>
209
- </div>
210
- <a href="https://metricailab.com" target="_blank" rel="noreferrer">Visit Metric AI Lab <span>↗</span></a>
211
- </section>
212
  </section>
213
 
214
  <footer>
 
40
  </button>
41
  </div>
42
  <div class="hero-copy">
43
+ <h1><span class="heading-flag" role="img" aria-label="Armenian flag">🇦🇲</span> ArmBench-ASR: Benchmarking Speech-to-text models on Armenian Speech Recognition Tasks</h1>
44
  <p class="lede">
45
  <span class="version-badge">v0.1</span>
 
 
46
  </p>
47
  </div>
48
  <div class="hero-art" aria-hidden="true">
 
111
  <p>10,113 clips and 20.7 hours of speech spanning read, literary, cinematic, and conversational Armenian. Every model is evaluated on the same fixed test material.</p>
112
  </div>
113
  <div class="dataset-grid">
 
 
 
 
 
 
114
  <article class="dataset-card">
115
+ <div class="dataset-card-top"><span>01</span><span class="dataset-tag public">Public</span></div>
116
  <h3><a class="dataset-source" href="https://mozilladatacollective.com/datasets/cmqinsp2y00xsnr07ra8gxfoi" target="_blank" rel="noreferrer">Common Voice 26 <span>↗</span></a></h3>
117
  <p>Mozilla Common Voice Scripted Speech 26.0 for Armenian, using the full <code>hy-AM</code> test split released in June 2026.</p>
118
  <dl class="dataset-meta"><div><dt>Samples</dt><dd>6,892</dd></div><div><dt>Audio</dt><dd>10.8 h</dd></div><div><dt>Style</dt><dd>Read speech</dd></div></dl>
119
  </article>
120
+
121
  <article class="dataset-card">
122
+ <div class="dataset-card-top"><span>02</span><span class="dataset-tag public">Public</span></div>
123
  <h3><a class="dataset-source" href="https://huggingface.co/datasets/google/fleurs" target="_blank" rel="noreferrer">FLEURS <span>↗</span></a></h3>
124
  <p>Google FLEURS Armenian (<code>hy_am</code>) test split. Raw transcriptions retain punctuation and capitalization for strict scoring, with some <a href="https://huggingface.co/datasets/Metric-AI/fleurs-corrections" target="_blank" rel="noreferrer">manual corrections</a> applied.</p>
125
  <dl class="dataset-meta"><div><dt>Samples</dt><dd>932</dd></div><div><dt>Audio</dt><dd>3.0 h</dd></div><div><dt>Style</dt><dd>Read speech</dd></div></dl>
126
  </article>
127
+
128
  <article class="dataset-card">
129
+ <div class="dataset-card-top"><span>03</span><span class="dataset-tag private">Private</span></div>
130
  <h3>Poems</h3>
131
  <p>Read Armenian poetry with background music, literary vocabulary, expressive phrasing, and text structures underrepresented in general ASR sets.</p>
132
  <dl class="dataset-meta"><div><dt>Samples</dt><dd>969</dd></div><div><dt>Audio</dt><dd>3.3 h</dd></div><div><dt>Style</dt><dd>Literary</dd></div></dl>
133
  </article>
134
+
135
  <article class="dataset-card">
136
+ <div class="dataset-card-top"><span>04</span><span class="dataset-tag private">Private</span></div>
137
  <h3>Movies</h3>
138
  <p>Armenian movie dialogue with conversational delivery, produced-media acoustics, and occasional background noise.</p>
139
  <dl class="dataset-meta"><div><dt>Samples</dt><dd>820</dd></div><div><dt>Audio</dt><dd>1.7 h</dd></div><div><dt>Style</dt><dd>Dialogue</dd></div></dl>
140
  </article>
141
+
142
  <article class="dataset-card">
143
+ <div class="dataset-card-top"><span>05</span><span class="dataset-tag private">Private</span></div>
144
+ <h3>News</h3>
145
  <p>A compact Armenian news dataset recited by a speaker, designed to evaluate clear narrated speech beyond standard public test sets.</p>
146
  <dl class="dataset-meta"><div><dt>Samples</dt><dd>500</dd></div><div><dt>Audio</dt><dd>1.9 h</dd></div><div><dt>Style</dt><dd>Narrated news</dd></div></dl>
147
  </article>
 
175
  <strong>API settings</strong>
176
  <p>Hosted systems use fixed model identifiers and transcription-only prompts. Evaluated Gemini runs use temperature 0, a 1,000-token output cap, and audio limited to 40 seconds. Thinking is disabled or set to the lowest applicable level: budget 0 for Gemini 2.5 Flash, budget 128 for Gemini 2.5 Pro, and <b>MINIMAL</b> for Gemini 3.1 Flash Lite, 3.5 Flash Lite, and 3.6 Flash.</p>
177
  </div>
178
+ <div class="note">
179
+ <strong>Text cleaning</strong>
180
+ <p>References and predictions are processed with the same deterministic pipeline. Strict scoring standardizes punctuation, spacing, capitalization, and Unicode; normalized scoring additionally reduces orthographic and formatting differences while preserving Armenian, English, Russian letters, digits, and recognized clock times. Both <b>WER / CER</b> and <b>WER-norm / CER-norm</b> are reported.</p>
181
+ </div>
182
 
183
+ <section class="lab-card">
184
+ <div>
185
+ <h2>About Us</h2>
186
+ <p>Armenian ASR Benchmark is a Metric AI Lab project. The applied AI research lab builds production systems across custom language and vision-language models, multilingual voice agents, Physical AI, retrieval, and analytics—while publishing frontier research and open benchmarks. We also gratefully acknowledge the model providers who made their systems available for evaluation at no cost, and Infocom for providing Armenian news data used in this benchmark.</p>
187
+ </div>
188
+ <a href="https://metricailab.com" target="_blank" rel="noreferrer">Visit Metric AI Lab <span>↗</span></a>
189
+ </section>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
190
  </section>
191
 
192
  <footer>