Download lib/text.js from ryusei-46/transcriber-device-bench: direct link, hf CLI and curl.
- Browser
- Download file 2.4 kB
-
https://huggingface.co/spaces/ryusei-46/transcriber-device-bench/resolve/main/lib/text.js
- Command line
-
hf download hf://spaces/ryusei-46/transcriber-device-bench/lib/text.js
-
curl -L -o text.js https://huggingface.co/spaces/ryusei-46/transcriber-device-bench/resolve/main/lib/text.js
2.4 kB
| // 文字の後始末と簡易 WER。正規化は 11-10・11-11 の採点(Whisper の BasicTextNormalizer、発音区別符号は残す)の写し: | |
| // 小文字 → [..] <..> と (..) を消す → NFKC → 記号・句読点・結合文字(Unicode の M・S・P)を空白に → 空白をまとめる。 | |
| export function normalize(s) { | |
| let t = (s ?? "").replace(/<[a-z]{2}-[A-Z]{2}>/g, "").toLowerCase(); | |
| t = t.replace(/[<[][^>\]]*[>\]]/g, ""); | |
| t = t.replace(/\(([^)]+?)\)/g, ""); | |
| t = t.normalize("NFKC").replace(/[\p{M}\p{S}\p{P}]/gu, " ").toLowerCase(); | |
| return t.replace(/\s+/g, " ").trim(); | |
| } | |
| /** 語の編集距離(置換・削除・挿入の数)と正解の語数。 */ | |
| export function wordErrors(ref, hyp) { | |
| const r = normalize(ref).split(" ").filter(Boolean); | |
| const h = normalize(hyp).split(" ").filter(Boolean); | |
| if (r.length === 0) return { errors: 0, words: 0 }; | |
| let prev = new Int32Array(h.length + 1); | |
| let cur = new Int32Array(h.length + 1); | |
| for (let j = 0; j <= h.length; j++) prev[j] = j; | |
| for (let i = 1; i <= r.length; i++) { | |
| cur[0] = i; | |
| for (let j = 1; j <= h.length; j++) { | |
| const sub = prev[j - 1] + (r[i - 1] === h[j - 1] ? 0 : 1); | |
| cur[j] = Math.min(sub, prev[j] + 1, cur[j - 1] + 1); | |
| } | |
| [prev, cur] = [cur, prev]; | |
| } | |
| return { errors: prev[h.length], words: r.length }; | |
| } | |
| /** | |
| * SentencePiece の片を文字にする("▁" は空白、"<0xNN>" は 1 バイト)。空白は onnx-asr と同じ規則で整える | |
| * (`\A\s|\s\B|(\s)\b`: 先頭の空白と、語の文字が続かない空白(句読点の前の "▁," など)を消す)。 | |
| */ | |
| const utf8 = new TextEncoder(); | |
| export function detokenize(pieces) { | |
| const bytes = []; | |
| for (const p of pieces) { | |
| const m = /^<0x([0-9A-Fa-f]{2})>$/.exec(p); | |
| if (m) bytes.push(Number.parseInt(m[1], 16)); | |
| else for (const b of utf8.encode(p.replaceAll("▁", " "))) bytes.push(b); | |
| } | |
| const s = new TextDecoder().decode(new Uint8Array(bytes)); | |
| return s.replace(/^\s/u, "").replace(/\s(?![\p{L}\p{N}_])/gu, "").replace(/\s/gu, " "); | |
| } | |
| /** onnx-asr の配置の vocab.txt("片 id" の行)を id 順の配列にする。 */ | |
| export function parseVocab(text) { | |
| const out = []; | |
| for (const line of text.split(/\r?\n/)) { | |
| const i = line.lastIndexOf(" "); | |
| if (i > 0) out[Number(line.slice(i + 1))] = line.slice(0, i); | |
| } | |
| return out; | |
| } | |