splainer-voice / source /kokoro_server.py
Kevin200599's picture
Source of the voice program
736dbe2 verified
Raw History Blame Contribute Delete
14.5 kB
"""Splainer's local voice: Kokoro-82M (Apache-2.0) behind OpenAI's speech API on 127.0.0.1, so Splainer narrates with a natural
voice without the cloud. Runs on the processor (ONNX Runtime), leaving the graphics card to the drawing model.
python kokoro_server.py --models <folder with kokoro-v1.0.onnx and voices-v1.0.bin> --token-file <file> [--port 8880]
Every request must carry "Authorization: Bearer <token from the file>" (Splainer creates it per install). Browser requests
(any Origin header) and other Host names are refused, so web pages can't use the voice; input is capped at 4096 characters.
POST /v1/audio/speech {"model":"kokoro","input":"...","voice":"female"|"male"|<kokoro voice>,"response_format":"wav"|"pcm","speed":1.0}
-> 24 kHz mono 16-bit WAV, or raw PCM streamed sentence by sentence.
POST /v1/audio/transcriptions (multipart: file=<PCM WAV>, language, prompt) -> {"text": "..."}
Whisper (faster-whisper): large-v3-turbo on the graphics card (~0.2 s per question), else "small" on the processor.
The installed app runs this file as a packaged program (tools/voice/build.ps1) and puts the models in the --models folder:
kokoro-v1.0.onnx, voices-v1.0.bin, whisper-small/, and with an NVIDIA card whisper-large-v3-turbo/ and gpu/ (CUDA libraries).
GET /v1/models, /v1/audio/voices
"""
import argparse, glob, hmac, io, json, os, re, site, struct, sys, threading, time, wave
from email.parser import BytesParser
from email.policy import HTTP
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import numpy as np
from kokoro_onnx import Kokoro
# The two choices Splainer offers; any Kokoro voice name also works.
ALIASES = {"female": "af_heart", "male": "am_michael", "default": "af_heart"}
RATE = 24000
kokoro: Kokoro
lock = threading.Lock()
TOKEN = ""
HOSTS: set = set()
MAX_INPUT = 4096
MAX_AUDIO = 25 * 1024 * 1024
FOLDER = "."
whisper = None
whisper_ready = threading.Event()
whisper_lock = threading.Lock()
def gpu_folders(folder):
"""Where the CUDA libraries for Whisper live: the GPU pack Splainer downloads (<folder>/gpu), or the nvidia-* wheels of a
developer's Python environment."""
found = [os.path.join(folder, "gpu")] if os.path.isdir(os.path.join(folder, "gpu")) else []
if not getattr(sys, "frozen", False):
try: found += glob.glob(os.path.join(site.getsitepackages()[-1], "nvidia", "*", "bin"))
except Exception: pass
return found
def whisper_source(folder, name):
"""A model folder Splainer downloaded (<folder>/whisper-<name>, holding model.bin), else the model's name so a developer's
environment fetches it into <folder>/whisper."""
local = os.path.join(folder, "whisper-" + name)
return local if os.path.exists(os.path.join(local, "model.bin")) else name
def load_whisper(folder):
"""Loads Whisper in the background so speech is available at once: large-v3-turbo on the graphics card when the CUDA
libraries are there, else "small" on the processor. The installed app never downloads here: it only loads what is on disk."""
global whisper
try:
from faster_whisper import WhisperModel
frozen = getattr(sys, "frozen", False); model = None
dirs = gpu_folders(folder)
for d in dirs:
os.add_dll_directory(d); os.environ["PATH"] = d + os.pathsep + os.environ["PATH"]
turbo = whisper_source(folder, "large-v3-turbo")
if dirs and (not frozen or os.path.isdir(turbo)):
try:
model = WhisperModel(turbo, device="cuda", compute_type="float16", download_root=os.path.join(folder, "whisper"))
list(model.transcribe(np.zeros(16000, np.float32), beam_size=1)[0]) # warm up; fails here when CUDA is unusable
log("whisper: large-v3-turbo on the graphics card")
except Exception as e:
model = None; log(f"whisper: the graphics card could not be used ({str(e)[:160]}); using the processor")
if model is None:
small = whisper_source(folder, "small")
if frozen and not os.path.isdir(small): raise RuntimeError("no speech recognition model is installed")
model = WhisperModel(small, device="cpu", compute_type="int8", download_root=os.path.join(folder, "whisper"))
list(model.transcribe(np.zeros(16000, np.float32), beam_size=1)[0])
log("whisper: small on the processor")
whisper = model
except Exception as e:
log(f"whisper unavailable: {str(e)[:200]}")
finally:
whisper_ready.set()
def pcm_from_wav(data):
w = wave.open(io.BytesIO(data)); rate, channels = w.getframerate(), w.getnchannels()
if w.getsampwidth() != 2: raise ValueError("send 16-bit PCM WAV")
a = np.frombuffer(w.readframes(w.getnframes()), dtype="<i2").astype(np.float32) / 32768
if channels > 1: a = a.reshape(-1, channels).mean(1)
if rate != 16000: a = np.interp(np.arange(0, len(a), rate / 16000), np.arange(len(a)), a).astype(np.float32)
return a
def transcribe(data, language, prompt):
whisper_ready.wait(120)
if whisper is None: raise RuntimeError("speech recognition is not available")
lang = (language or "").split("-")[0].lower() or None
audio = pcm_from_wav(data)
level = f"{len(audio) / 16000:.1f} s, peak {np.abs(audio).max(initial=0):.3f}, rms {float(np.sqrt(np.mean(audio ** 2))) if len(audio) else 0:.4f}"
if os.path.exists(os.path.join(FOLDER, "debug")): open(os.path.join(FOLDER, "last-recording.wav"), "wb").write(data)
with whisper_lock:
segments, _ = whisper.transcribe(audio, language=lang if lang not in ("default", "auto") else None,
initial_prompt=prompt or None, beam_size=1, vad_filter=True)
text = " ".join(s.text.strip() for s in segments).strip()
log(f"heard {len(text)} chars from {level}")
return text
def log(line):
with open(os.path.join(FOLDER, "server.log"), "a", encoding="utf-8") as f: f.write(time.strftime("%H:%M:%S ") + line + "\n")
def resolve(voice):
name = ALIASES.get((voice or "default").lower(), voice)
return name if name in kokoro.get_voices() else ALIASES["default"]
def lang_for(voice):
return "en-gb" if voice[:1] == "b" else "en-us"
def sentences(text):
# Short pieces start playback sooner; very short ones are joined so the voice keeps its flow.
parts, cur = [], ""
for s in re.split(r"(?<=[.!?…])\s+", text.strip()):
cur = f"{cur} {s}".strip()
if len(cur) >= 60:
parts.append(cur); cur = ""
if cur: parts.append(cur)
return parts or [text]
def pcm16(samples):
return (np.clip(samples, -1, 1) * 32767).astype("<i2").tobytes()
def synth(text, voice, speed):
with lock: # one ONNX session; requests queue rather than fight over the cores
started = time.time()
samples, _ = kokoro.create(text, voice=voice, speed=speed, lang=lang_for(voice))
took = time.time() - started
# Lengths and timing only, never the words: shows at once when Windows has slowed the voice down.
log(f"spoke {len(text)} chars ({len(samples) / RATE:.1f} s of sound) in {took:.2f} s")
return pcm16(samples)
def full_speed():
"""Windows moves windowless background programs onto the processor's efficiency cores after a while, which made the
voice start three to four times later. Opting out of power throttling keeps speech on the fast cores."""
if os.name != "nt": return
try:
import ctypes
from ctypes import wintypes
class Throttling(ctypes.Structure):
_fields_ = [("Version", wintypes.ULONG), ("ControlMask", wintypes.ULONG), ("StateMask", wintypes.ULONG)]
k = ctypes.WinDLL("kernel32", use_last_error=True)
k.GetCurrentProcess.restype = wintypes.HANDLE
k.SetProcessInformation.argtypes = [wintypes.HANDLE, ctypes.c_int, ctypes.c_void_p, wintypes.DWORD]
k.SetProcessInformation.restype = wintypes.BOOL
k.SetPriorityClass.argtypes = [wintypes.HANDLE, wintypes.DWORD]
me = k.GetCurrentProcess()
# ProcessPowerThrottling (4): control execution speed, state 0 = never throttle (high quality of service).
state = Throttling(1, 0x1, 0)
ok = k.SetProcessInformation(me, 4, ctypes.byref(state), ctypes.sizeof(state))
k.SetPriorityClass(me, 0x00008000) # ABOVE_NORMAL_PRIORITY_CLASS: speech is short and the user is waiting for it
log("full speed: power throttling " + ("off" if ok else f"could not be turned off (error {ctypes.get_last_error()})"))
except Exception as e:
log(f"full speed: not set ({e})")
def wav(pcm):
return b"RIFF" + struct.pack("<I", 36 + len(pcm)) + b"WAVEfmt " + struct.pack("<IHHIIHH", 16, 1, 1, RATE, RATE * 2, 2, 16) + b"data" + struct.pack("<I", len(pcm)) + pcm
class Handler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
def log_message(self, *a): pass
def send(self, code, body, kind="application/json"):
self.send_response(code); self.send_header("Content-Type", kind); self.send_header("Content-Length", str(len(body))); self.end_headers(); self.wfile.write(body)
def allowed(self):
# Only Splainer: no web page (they always send Origin), only our own host name, and the per-install token.
if self.headers.get("Origin") is not None or self.headers.get("Host", "") not in HOSTS:
self.send(403, b'{"error":"forbidden"}'); return False
if not TOKEN or not hmac.compare_digest(self.headers.get("Authorization", "").encode(), ("Bearer " + TOKEN).encode()):
self.send(401, b'{"error":"unauthorized"}'); return False
return True
def do_GET(self):
if not self.allowed(): return
if self.path.rstrip("/") == "/v1/models":
return self.send(200, json.dumps({"object": "list", "data": [{"id": "kokoro", "object": "model", "owned_by": "local"}, {"id": "whisper", "object": "model", "owned_by": "local"}]}).encode())
if self.path.rstrip("/") == "/v1/audio/voices":
return self.send(200, json.dumps({"voices": kokoro.get_voices(), "aliases": ALIASES, "listening": whisper is not None, "listening_ready": whisper_ready.is_set()}).encode())
self.send(404, b"{}")
def body(self, limit=1_000_000):
# .NET's JsonContent sends the request chunked, without a Content-Length.
if "chunked" in self.headers.get("Transfer-Encoding", "").lower():
data = b""
while True:
size = int(self.rfile.readline().split(b";")[0].strip() or b"0", 16)
if size == 0: self.rfile.readline(); return data
data += self.rfile.read(size); self.rfile.readline()
if len(data) > limit: raise ValueError("request too large")
length = int(self.headers.get("Content-Length", 0))
if length > limit: raise ValueError("request too large")
return self.rfile.read(length)
def transcription(self):
kind = self.headers.get("Content-Type", "")
if not kind.lower().startswith("multipart/form-data"): return self.send(415, b'{"error":"send multipart/form-data"}')
message = BytesParser(policy=HTTP).parsebytes(b"Content-Type: " + kind.encode() + b"\r\n\r\n" + self.body(MAX_AUDIO + 65536))
fields = {p.get_param("name", header="content-disposition"): p.get_payload(decode=True) for p in message.iter_parts()}
if not fields.get("file"): return self.send(400, b'{"error":"file is missing"}')
text = transcribe(fields["file"], (fields.get("language") or b"").decode(), (fields.get("prompt") or b"").decode()[:500])
self.send(200, json.dumps({"text": text}).encode())
def do_POST(self):
if not self.allowed(): return
if self.path.rstrip("/") == "/v1/audio/transcriptions":
try: return self.transcription()
except Exception as e: return self.send(500, json.dumps({"error": str(e)}).encode())
if self.path.rstrip("/") != "/v1/audio/speech": return self.send(404, b"{}")
if not self.headers.get("Content-Type", "").lower().startswith("application/json"): return self.send(415, b'{"error":"send JSON"}')
try:
req = json.loads(self.body() or b"{}")
text = (req.get("input") or "").strip()
if not text: return self.send(400, b'{"error":"input is empty"}')
if len(text) > MAX_INPUT: return self.send(413, b'{"error":"input is longer than 4096 characters"}')
voice, speed = resolve(req.get("voice")), float(req.get("speed") or 1.0)
speed = min(2.0, max(0.5, speed))
if req.get("response_format") == "pcm":
# Stream: each sentence is sent as soon as it is spoken, so the first words play within a fraction of a second.
self.send_response(200); self.send_header("Content-Type", "audio/pcm"); self.send_header("Transfer-Encoding", "chunked"); self.end_headers()
for piece in sentences(text):
data = synth(piece, voice, speed)
self.wfile.write(f"{len(data):X}\r\n".encode() + data + b"\r\n"); self.wfile.flush()
self.wfile.write(b"0\r\n\r\n"); return
self.send(200, wav(b"".join(synth(p, voice, speed) for p in sentences(text))), "audio/wav")
except (BrokenPipeError, ConnectionResetError):
pass
except Exception as e:
self.send(500, json.dumps({"error": str(e)}).encode())
def main():
global kokoro, TOKEN, FOLDER
ap = argparse.ArgumentParser(); ap.add_argument("--models", required=True); ap.add_argument("--port", type=int, default=8880); ap.add_argument("--token-file", required=True)
a = ap.parse_args()
TOKEN = open(a.token_file, encoding="utf-8").read().strip()
if len(TOKEN) < 32: sys.exit("the token file must hold a secret of at least 32 characters")
FOLDER = a.models
HOSTS.update({f"127.0.0.1:{a.port}", f"localhost:{a.port}"})
full_speed()
kokoro = Kokoro(os.path.join(a.models, "kokoro-v1.0.onnx"), os.path.join(a.models, "voices-v1.0.bin"))
synth("Ready.", ALIASES["default"], 1.0) # warm up so the first answer is quick
threading.Thread(target=load_whisper, args=(a.models,), daemon=True).start()
print(f"Splainer voice ready on http://127.0.0.1:{a.port}/v1", flush=True)
ThreadingHTTPServer(("127.0.0.1", a.port), Handler).serve_forever()
if __name__ == "__main__": main()