File size: 14,522 Bytes
736dbe2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 | """Splainer's local voice: Kokoro-82M (Apache-2.0) behind OpenAI's speech API on 127.0.0.1, so Splainer narrates with a natural
voice without the cloud. Runs on the processor (ONNX Runtime), leaving the graphics card to the drawing model.
python kokoro_server.py --models <folder with kokoro-v1.0.onnx and voices-v1.0.bin> --token-file <file> [--port 8880]
Every request must carry "Authorization: Bearer <token from the file>" (Splainer creates it per install). Browser requests
(any Origin header) and other Host names are refused, so web pages can't use the voice; input is capped at 4096 characters.
POST /v1/audio/speech {"model":"kokoro","input":"...","voice":"female"|"male"|<kokoro voice>,"response_format":"wav"|"pcm","speed":1.0}
-> 24 kHz mono 16-bit WAV, or raw PCM streamed sentence by sentence.
POST /v1/audio/transcriptions (multipart: file=<PCM WAV>, language, prompt) -> {"text": "..."}
Whisper (faster-whisper): large-v3-turbo on the graphics card (~0.2 s per question), else "small" on the processor.
The installed app runs this file as a packaged program (tools/voice/build.ps1) and puts the models in the --models folder:
kokoro-v1.0.onnx, voices-v1.0.bin, whisper-small/, and with an NVIDIA card whisper-large-v3-turbo/ and gpu/ (CUDA libraries).
GET /v1/models, /v1/audio/voices
"""
import argparse, glob, hmac, io, json, os, re, site, struct, sys, threading, time, wave
from email.parser import BytesParser
from email.policy import HTTP
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import numpy as np
from kokoro_onnx import Kokoro
# The two choices Splainer offers; any Kokoro voice name also works.
ALIASES = {"female": "af_heart", "male": "am_michael", "default": "af_heart"}
RATE = 24000
kokoro: Kokoro
lock = threading.Lock()
TOKEN = ""
HOSTS: set = set()
MAX_INPUT = 4096
MAX_AUDIO = 25 * 1024 * 1024
FOLDER = "."
whisper = None
whisper_ready = threading.Event()
whisper_lock = threading.Lock()
def gpu_folders(folder):
"""Where the CUDA libraries for Whisper live: the GPU pack Splainer downloads (<folder>/gpu), or the nvidia-* wheels of a
developer's Python environment."""
found = [os.path.join(folder, "gpu")] if os.path.isdir(os.path.join(folder, "gpu")) else []
if not getattr(sys, "frozen", False):
try: found += glob.glob(os.path.join(site.getsitepackages()[-1], "nvidia", "*", "bin"))
except Exception: pass
return found
def whisper_source(folder, name):
"""A model folder Splainer downloaded (<folder>/whisper-<name>, holding model.bin), else the model's name so a developer's
environment fetches it into <folder>/whisper."""
local = os.path.join(folder, "whisper-" + name)
return local if os.path.exists(os.path.join(local, "model.bin")) else name
def load_whisper(folder):
"""Loads Whisper in the background so speech is available at once: large-v3-turbo on the graphics card when the CUDA
libraries are there, else "small" on the processor. The installed app never downloads here: it only loads what is on disk."""
global whisper
try:
from faster_whisper import WhisperModel
frozen = getattr(sys, "frozen", False); model = None
dirs = gpu_folders(folder)
for d in dirs:
os.add_dll_directory(d); os.environ["PATH"] = d + os.pathsep + os.environ["PATH"]
turbo = whisper_source(folder, "large-v3-turbo")
if dirs and (not frozen or os.path.isdir(turbo)):
try:
model = WhisperModel(turbo, device="cuda", compute_type="float16", download_root=os.path.join(folder, "whisper"))
list(model.transcribe(np.zeros(16000, np.float32), beam_size=1)[0]) # warm up; fails here when CUDA is unusable
log("whisper: large-v3-turbo on the graphics card")
except Exception as e:
model = None; log(f"whisper: the graphics card could not be used ({str(e)[:160]}); using the processor")
if model is None:
small = whisper_source(folder, "small")
if frozen and not os.path.isdir(small): raise RuntimeError("no speech recognition model is installed")
model = WhisperModel(small, device="cpu", compute_type="int8", download_root=os.path.join(folder, "whisper"))
list(model.transcribe(np.zeros(16000, np.float32), beam_size=1)[0])
log("whisper: small on the processor")
whisper = model
except Exception as e:
log(f"whisper unavailable: {str(e)[:200]}")
finally:
whisper_ready.set()
def pcm_from_wav(data):
w = wave.open(io.BytesIO(data)); rate, channels = w.getframerate(), w.getnchannels()
if w.getsampwidth() != 2: raise ValueError("send 16-bit PCM WAV")
a = np.frombuffer(w.readframes(w.getnframes()), dtype="<i2").astype(np.float32) / 32768
if channels > 1: a = a.reshape(-1, channels).mean(1)
if rate != 16000: a = np.interp(np.arange(0, len(a), rate / 16000), np.arange(len(a)), a).astype(np.float32)
return a
def transcribe(data, language, prompt):
whisper_ready.wait(120)
if whisper is None: raise RuntimeError("speech recognition is not available")
lang = (language or "").split("-")[0].lower() or None
audio = pcm_from_wav(data)
level = f"{len(audio) / 16000:.1f} s, peak {np.abs(audio).max(initial=0):.3f}, rms {float(np.sqrt(np.mean(audio ** 2))) if len(audio) else 0:.4f}"
if os.path.exists(os.path.join(FOLDER, "debug")): open(os.path.join(FOLDER, "last-recording.wav"), "wb").write(data)
with whisper_lock:
segments, _ = whisper.transcribe(audio, language=lang if lang not in ("default", "auto") else None,
initial_prompt=prompt or None, beam_size=1, vad_filter=True)
text = " ".join(s.text.strip() for s in segments).strip()
log(f"heard {len(text)} chars from {level}")
return text
def log(line):
with open(os.path.join(FOLDER, "server.log"), "a", encoding="utf-8") as f: f.write(time.strftime("%H:%M:%S ") + line + "\n")
def resolve(voice):
name = ALIASES.get((voice or "default").lower(), voice)
return name if name in kokoro.get_voices() else ALIASES["default"]
def lang_for(voice):
return "en-gb" if voice[:1] == "b" else "en-us"
def sentences(text):
# Short pieces start playback sooner; very short ones are joined so the voice keeps its flow.
parts, cur = [], ""
for s in re.split(r"(?<=[.!?…])\s+", text.strip()):
cur = f"{cur} {s}".strip()
if len(cur) >= 60:
parts.append(cur); cur = ""
if cur: parts.append(cur)
return parts or [text]
def pcm16(samples):
return (np.clip(samples, -1, 1) * 32767).astype("<i2").tobytes()
def synth(text, voice, speed):
with lock: # one ONNX session; requests queue rather than fight over the cores
started = time.time()
samples, _ = kokoro.create(text, voice=voice, speed=speed, lang=lang_for(voice))
took = time.time() - started
# Lengths and timing only, never the words: shows at once when Windows has slowed the voice down.
log(f"spoke {len(text)} chars ({len(samples) / RATE:.1f} s of sound) in {took:.2f} s")
return pcm16(samples)
def full_speed():
"""Windows moves windowless background programs onto the processor's efficiency cores after a while, which made the
voice start three to four times later. Opting out of power throttling keeps speech on the fast cores."""
if os.name != "nt": return
try:
import ctypes
from ctypes import wintypes
class Throttling(ctypes.Structure):
_fields_ = [("Version", wintypes.ULONG), ("ControlMask", wintypes.ULONG), ("StateMask", wintypes.ULONG)]
k = ctypes.WinDLL("kernel32", use_last_error=True)
k.GetCurrentProcess.restype = wintypes.HANDLE
k.SetProcessInformation.argtypes = [wintypes.HANDLE, ctypes.c_int, ctypes.c_void_p, wintypes.DWORD]
k.SetProcessInformation.restype = wintypes.BOOL
k.SetPriorityClass.argtypes = [wintypes.HANDLE, wintypes.DWORD]
me = k.GetCurrentProcess()
# ProcessPowerThrottling (4): control execution speed, state 0 = never throttle (high quality of service).
state = Throttling(1, 0x1, 0)
ok = k.SetProcessInformation(me, 4, ctypes.byref(state), ctypes.sizeof(state))
k.SetPriorityClass(me, 0x00008000) # ABOVE_NORMAL_PRIORITY_CLASS: speech is short and the user is waiting for it
log("full speed: power throttling " + ("off" if ok else f"could not be turned off (error {ctypes.get_last_error()})"))
except Exception as e:
log(f"full speed: not set ({e})")
def wav(pcm):
return b"RIFF" + struct.pack("<I", 36 + len(pcm)) + b"WAVEfmt " + struct.pack("<IHHIIHH", 16, 1, 1, RATE, RATE * 2, 2, 16) + b"data" + struct.pack("<I", len(pcm)) + pcm
class Handler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
def log_message(self, *a): pass
def send(self, code, body, kind="application/json"):
self.send_response(code); self.send_header("Content-Type", kind); self.send_header("Content-Length", str(len(body))); self.end_headers(); self.wfile.write(body)
def allowed(self):
# Only Splainer: no web page (they always send Origin), only our own host name, and the per-install token.
if self.headers.get("Origin") is not None or self.headers.get("Host", "") not in HOSTS:
self.send(403, b'{"error":"forbidden"}'); return False
if not TOKEN or not hmac.compare_digest(self.headers.get("Authorization", "").encode(), ("Bearer " + TOKEN).encode()):
self.send(401, b'{"error":"unauthorized"}'); return False
return True
def do_GET(self):
if not self.allowed(): return
if self.path.rstrip("/") == "/v1/models":
return self.send(200, json.dumps({"object": "list", "data": [{"id": "kokoro", "object": "model", "owned_by": "local"}, {"id": "whisper", "object": "model", "owned_by": "local"}]}).encode())
if self.path.rstrip("/") == "/v1/audio/voices":
return self.send(200, json.dumps({"voices": kokoro.get_voices(), "aliases": ALIASES, "listening": whisper is not None, "listening_ready": whisper_ready.is_set()}).encode())
self.send(404, b"{}")
def body(self, limit=1_000_000):
# .NET's JsonContent sends the request chunked, without a Content-Length.
if "chunked" in self.headers.get("Transfer-Encoding", "").lower():
data = b""
while True:
size = int(self.rfile.readline().split(b";")[0].strip() or b"0", 16)
if size == 0: self.rfile.readline(); return data
data += self.rfile.read(size); self.rfile.readline()
if len(data) > limit: raise ValueError("request too large")
length = int(self.headers.get("Content-Length", 0))
if length > limit: raise ValueError("request too large")
return self.rfile.read(length)
def transcription(self):
kind = self.headers.get("Content-Type", "")
if not kind.lower().startswith("multipart/form-data"): return self.send(415, b'{"error":"send multipart/form-data"}')
message = BytesParser(policy=HTTP).parsebytes(b"Content-Type: " + kind.encode() + b"\r\n\r\n" + self.body(MAX_AUDIO + 65536))
fields = {p.get_param("name", header="content-disposition"): p.get_payload(decode=True) for p in message.iter_parts()}
if not fields.get("file"): return self.send(400, b'{"error":"file is missing"}')
text = transcribe(fields["file"], (fields.get("language") or b"").decode(), (fields.get("prompt") or b"").decode()[:500])
self.send(200, json.dumps({"text": text}).encode())
def do_POST(self):
if not self.allowed(): return
if self.path.rstrip("/") == "/v1/audio/transcriptions":
try: return self.transcription()
except Exception as e: return self.send(500, json.dumps({"error": str(e)}).encode())
if self.path.rstrip("/") != "/v1/audio/speech": return self.send(404, b"{}")
if not self.headers.get("Content-Type", "").lower().startswith("application/json"): return self.send(415, b'{"error":"send JSON"}')
try:
req = json.loads(self.body() or b"{}")
text = (req.get("input") or "").strip()
if not text: return self.send(400, b'{"error":"input is empty"}')
if len(text) > MAX_INPUT: return self.send(413, b'{"error":"input is longer than 4096 characters"}')
voice, speed = resolve(req.get("voice")), float(req.get("speed") or 1.0)
speed = min(2.0, max(0.5, speed))
if req.get("response_format") == "pcm":
# Stream: each sentence is sent as soon as it is spoken, so the first words play within a fraction of a second.
self.send_response(200); self.send_header("Content-Type", "audio/pcm"); self.send_header("Transfer-Encoding", "chunked"); self.end_headers()
for piece in sentences(text):
data = synth(piece, voice, speed)
self.wfile.write(f"{len(data):X}\r\n".encode() + data + b"\r\n"); self.wfile.flush()
self.wfile.write(b"0\r\n\r\n"); return
self.send(200, wav(b"".join(synth(p, voice, speed) for p in sentences(text))), "audio/wav")
except (BrokenPipeError, ConnectionResetError):
pass
except Exception as e:
self.send(500, json.dumps({"error": str(e)}).encode())
def main():
global kokoro, TOKEN, FOLDER
ap = argparse.ArgumentParser(); ap.add_argument("--models", required=True); ap.add_argument("--port", type=int, default=8880); ap.add_argument("--token-file", required=True)
a = ap.parse_args()
TOKEN = open(a.token_file, encoding="utf-8").read().strip()
if len(TOKEN) < 32: sys.exit("the token file must hold a secret of at least 32 characters")
FOLDER = a.models
HOSTS.update({f"127.0.0.1:{a.port}", f"localhost:{a.port}"})
full_speed()
kokoro = Kokoro(os.path.join(a.models, "kokoro-v1.0.onnx"), os.path.join(a.models, "voices-v1.0.bin"))
synth("Ready.", ALIASES["default"], 1.0) # warm up so the first answer is quick
threading.Thread(target=load_whisper, args=(a.models,), daemon=True).start()
print(f"Splainer voice ready on http://127.0.0.1:{a.port}/v1", flush=True)
ThreadingHTTPServer(("127.0.0.1", a.port), Handler).serve_forever()
if __name__ == "__main__": main()
|