Spaces:

cfahlgren1
/

talking-pixel

Running

talking-pixel / tests /reference /py_frontend_fuzz.py

convert talking pixel to a sveltekit app that runs pivo-1.4m tts in the browser

3ec6b9e about 17 hours ago

1.87 kB

"""Generate a randomized frontend corpus with Python reference outputs."""
import json, random, sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]  # the workspace that holds model/ and ref/
sys.path.insert(0, str(ROOT / "model"))
import pivo.text as T

random.seed(1234)
pieces = ["Hello", "world", "NASA", "nasa's", "CPU", "GPUs", "USA's", "ABCDEF", "Mr.", "mrs.", "DR.", "Prof.", "St.", "vs.", "etc.", "etc",
          "$5", "$1.00", "$0.5", "£1.01", "£2", "€3.99", "$1,234.56", "$1.234", "1/2", "2024-02-29", "2023-02-29", "1999-12-31", "0000-01-01",
          "21st", "22ND", "3rd", "4th", "11th", "1th", "1,000", "12,34", "1,234,567", "3.14", "-2", "-0.5", "007", "1e5", "10%", "A&B", "x+y",
          "well-known", "(paren)", "[b]", "{c}", "\"q\"", "it's", "rock'n'roll", "'tis", "o'clock", "—", "–", "…", "’", "“hi”", "...", "?!", ";", ":",
          "٣", "٣rd", "１２", "Ⅻ", "café", "ﬁne", "Ⓐ", "x_y", "_ab", "hi there", "a\tb", "\n", "line sep", "12345678901234567", "99999999999999",
          "zorblax", "qwxyz", "I'm", "can't", "Pi", "Vo", "THE", "a", "I", "AI", "OK", "ok", "100th", "2nd-hand", "9:30", "@", "#", "émigré", "x2", "2x", "A1"]
cases = []
for _ in range(4000):
    n = random.randint(1, 8)
    seps = [" ", " ", " ", "", ", ", ". ", "-", "\n"]
    s = "".join(random.choice(pieces) + random.choice(seps) for _ in range(n))
    cases.append(s)
out = []
for s in cases:
    try:
        fr = T.phonemize(s, strict=False)
        out.append({"text": s, "normalized": fr.normalized_text, "ids": list(fr.token_ids), "unknown": list(fr.unknown_words)})
    except Exception as e:
        out.append({"text": s, "error": type(e).__name__, "message": str(e)})
(ROOT / "ref/frontend_fuzz.json").write_text(json.dumps(out, ensure_ascii=False))
print(len(out), sum("error" in o for o in out), "errors")

	"""Generate a randomized frontend corpus with Python reference outputs."""
	import json, random, sys
	from pathlib import Path
	ROOT = Path(__file__).resolve().parents[3] # the workspace that holds model/ and ref/
	sys.path.insert(0, str(ROOT / "model"))
	import pivo.text as T

	random.seed(1234)
	pieces = ["Hello", "world", "NASA", "nasa's", "CPU", "GPUs", "USA's", "ABCDEF", "Mr.", "mrs.", "DR.", "Prof.", "St.", "vs.", "etc.", "etc",
	"$5", "$1.00", "$0.5", "£1.01", "£2", "€3.99", "$1,234.56", "$1.234", "1/2", "2024-02-29", "2023-02-29", "1999-12-31", "0000-01-01",
	"21st", "22ND", "3rd", "4th", "11th", "1th", "1,000", "12,34", "1,234,567", "3.14", "-2", "-0.5", "007", "1e5", "10%", "A&B", "x+y",
	"well-known", "(paren)", "[b]", "{c}", "\"q\"", "it's", "rock'n'roll", "'tis", "o'clock", "—", "–", "…", "’", "“hi”", "...", "?!", ";", ":",
	"٣", "٣rd", "１２", "Ⅻ", "café", "ﬁne", "Ⓐ", "x_y", "_ab", "hi there", "a\tb", "\n", "line sep", "12345678901234567", "99999999999999",
	"zorblax", "qwxyz", "I'm", "can't", "Pi", "Vo", "THE", "a", "I", "AI", "OK", "ok", "100th", "2nd-hand", "9:30", "@", "#", "émigré", "x2", "2x", "A1"]
	cases = []
	for _ in range(4000):
	n = random.randint(1, 8)
	seps = [" ", " ", " ", "", ", ", ". ", "-", "\n"]
	s = "".join(random.choice(pieces) + random.choice(seps) for _ in range(n))
	cases.append(s)
	out = []
	for s in cases:
	try:
	fr = T.phonemize(s, strict=False)
	out.append({"text": s, "normalized": fr.normalized_text, "ids": list(fr.token_ids), "unknown": list(fr.unknown_words)})
	except Exception as e:
	out.append({"text": s, "error": type(e).__name__, "message": str(e)})
	(ROOT / "ref/frontend_fuzz.json").write_text(json.dumps(out, ensure_ascii=False))
	print(len(out), sum("error" in o for o in out), "errors")