File size: 14,522 Bytes
736dbe2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
"""Splainer's local voice: Kokoro-82M (Apache-2.0) behind OpenAI's speech API on 127.0.0.1, so Splainer narrates with a natural
voice without the cloud. Runs on the processor (ONNX Runtime), leaving the graphics card to the drawing model.

  python kokoro_server.py --models <folder with kokoro-v1.0.onnx and voices-v1.0.bin> --token-file <file> [--port 8880]

Every request must carry "Authorization: Bearer <token from the file>" (Splainer creates it per install). Browser requests
(any Origin header) and other Host names are refused, so web pages can't use the voice; input is capped at 4096 characters.

POST /v1/audio/speech {"model":"kokoro","input":"...","voice":"female"|"male"|<kokoro voice>,"response_format":"wav"|"pcm","speed":1.0}
  -> 24 kHz mono 16-bit WAV, or raw PCM streamed sentence by sentence.
POST /v1/audio/transcriptions (multipart: file=<PCM WAV>, language, prompt) -> {"text": "..."}
  Whisper (faster-whisper): large-v3-turbo on the graphics card (~0.2 s per question), else "small" on the processor.
  The installed app runs this file as a packaged program (tools/voice/build.ps1) and puts the models in the --models folder:
  kokoro-v1.0.onnx, voices-v1.0.bin, whisper-small/, and with an NVIDIA card whisper-large-v3-turbo/ and gpu/ (CUDA libraries).
GET  /v1/models, /v1/audio/voices
"""
import argparse, glob, hmac, io, json, os, re, site, struct, sys, threading, time, wave
from email.parser import BytesParser
from email.policy import HTTP
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer

import numpy as np
from kokoro_onnx import Kokoro

# The two choices Splainer offers; any Kokoro voice name also works.
ALIASES = {"female": "af_heart", "male": "am_michael", "default": "af_heart"}
RATE = 24000
kokoro: Kokoro
lock = threading.Lock()
TOKEN = ""
HOSTS: set = set()
MAX_INPUT = 4096
MAX_AUDIO = 25 * 1024 * 1024
FOLDER = "."
whisper = None
whisper_ready = threading.Event()
whisper_lock = threading.Lock()


def gpu_folders(folder):
    """Where the CUDA libraries for Whisper live: the GPU pack Splainer downloads (<folder>/gpu), or the nvidia-* wheels of a
    developer's Python environment."""
    found = [os.path.join(folder, "gpu")] if os.path.isdir(os.path.join(folder, "gpu")) else []
    if not getattr(sys, "frozen", False):
        try: found += glob.glob(os.path.join(site.getsitepackages()[-1], "nvidia", "*", "bin"))
        except Exception: pass
    return found


def whisper_source(folder, name):
    """A model folder Splainer downloaded (<folder>/whisper-<name>, holding model.bin), else the model's name so a developer's
    environment fetches it into <folder>/whisper."""
    local = os.path.join(folder, "whisper-" + name)
    return local if os.path.exists(os.path.join(local, "model.bin")) else name


def load_whisper(folder):
    """Loads Whisper in the background so speech is available at once: large-v3-turbo on the graphics card when the CUDA
    libraries are there, else "small" on the processor. The installed app never downloads here: it only loads what is on disk."""
    global whisper
    try:
        from faster_whisper import WhisperModel
        frozen = getattr(sys, "frozen", False); model = None
        dirs = gpu_folders(folder)
        for d in dirs:
            os.add_dll_directory(d); os.environ["PATH"] = d + os.pathsep + os.environ["PATH"]
        turbo = whisper_source(folder, "large-v3-turbo")
        if dirs and (not frozen or os.path.isdir(turbo)):
            try:
                model = WhisperModel(turbo, device="cuda", compute_type="float16", download_root=os.path.join(folder, "whisper"))
                list(model.transcribe(np.zeros(16000, np.float32), beam_size=1)[0])  # warm up; fails here when CUDA is unusable
                log("whisper: large-v3-turbo on the graphics card")
            except Exception as e:
                model = None; log(f"whisper: the graphics card could not be used ({str(e)[:160]}); using the processor")
        if model is None:
            small = whisper_source(folder, "small")
            if frozen and not os.path.isdir(small): raise RuntimeError("no speech recognition model is installed")
            model = WhisperModel(small, device="cpu", compute_type="int8", download_root=os.path.join(folder, "whisper"))
            list(model.transcribe(np.zeros(16000, np.float32), beam_size=1)[0])
            log("whisper: small on the processor")
        whisper = model
    except Exception as e:
        log(f"whisper unavailable: {str(e)[:200]}")
    finally:
        whisper_ready.set()


def pcm_from_wav(data):
    w = wave.open(io.BytesIO(data)); rate, channels = w.getframerate(), w.getnchannels()
    if w.getsampwidth() != 2: raise ValueError("send 16-bit PCM WAV")
    a = np.frombuffer(w.readframes(w.getnframes()), dtype="<i2").astype(np.float32) / 32768
    if channels > 1: a = a.reshape(-1, channels).mean(1)
    if rate != 16000: a = np.interp(np.arange(0, len(a), rate / 16000), np.arange(len(a)), a).astype(np.float32)
    return a


def transcribe(data, language, prompt):
    whisper_ready.wait(120)
    if whisper is None: raise RuntimeError("speech recognition is not available")
    lang = (language or "").split("-")[0].lower() or None
    audio = pcm_from_wav(data)
    level = f"{len(audio) / 16000:.1f} s, peak {np.abs(audio).max(initial=0):.3f}, rms {float(np.sqrt(np.mean(audio ** 2))) if len(audio) else 0:.4f}"
    if os.path.exists(os.path.join(FOLDER, "debug")): open(os.path.join(FOLDER, "last-recording.wav"), "wb").write(data)
    with whisper_lock:
        segments, _ = whisper.transcribe(audio, language=lang if lang not in ("default", "auto") else None,
                                         initial_prompt=prompt or None, beam_size=1, vad_filter=True)
        text = " ".join(s.text.strip() for s in segments).strip()
    log(f"heard {len(text)} chars from {level}")
    return text


def log(line):
    with open(os.path.join(FOLDER, "server.log"), "a", encoding="utf-8") as f: f.write(time.strftime("%H:%M:%S ") + line + "\n")


def resolve(voice):
    name = ALIASES.get((voice or "default").lower(), voice)
    return name if name in kokoro.get_voices() else ALIASES["default"]


def lang_for(voice):
    return "en-gb" if voice[:1] == "b" else "en-us"


def sentences(text):
    # Short pieces start playback sooner; very short ones are joined so the voice keeps its flow.
    parts, cur = [], ""
    for s in re.split(r"(?<=[.!?…])\s+", text.strip()):
        cur = f"{cur} {s}".strip()
        if len(cur) >= 60:
            parts.append(cur); cur = ""
    if cur: parts.append(cur)
    return parts or [text]


def pcm16(samples):
    return (np.clip(samples, -1, 1) * 32767).astype("<i2").tobytes()


def synth(text, voice, speed):
    with lock:  # one ONNX session; requests queue rather than fight over the cores
        started = time.time()
        samples, _ = kokoro.create(text, voice=voice, speed=speed, lang=lang_for(voice))
        took = time.time() - started
    # Lengths and timing only, never the words: shows at once when Windows has slowed the voice down.
    log(f"spoke {len(text)} chars ({len(samples) / RATE:.1f} s of sound) in {took:.2f} s")
    return pcm16(samples)


def full_speed():
    """Windows moves windowless background programs onto the processor's efficiency cores after a while, which made the
    voice start three to four times later. Opting out of power throttling keeps speech on the fast cores."""
    if os.name != "nt": return
    try:
        import ctypes
        from ctypes import wintypes

        class Throttling(ctypes.Structure):
            _fields_ = [("Version", wintypes.ULONG), ("ControlMask", wintypes.ULONG), ("StateMask", wintypes.ULONG)]

        k = ctypes.WinDLL("kernel32", use_last_error=True)
        k.GetCurrentProcess.restype = wintypes.HANDLE
        k.SetProcessInformation.argtypes = [wintypes.HANDLE, ctypes.c_int, ctypes.c_void_p, wintypes.DWORD]
        k.SetProcessInformation.restype = wintypes.BOOL
        k.SetPriorityClass.argtypes = [wintypes.HANDLE, wintypes.DWORD]
        me = k.GetCurrentProcess()
        # ProcessPowerThrottling (4): control execution speed, state 0 = never throttle (high quality of service).
        state = Throttling(1, 0x1, 0)
        ok = k.SetProcessInformation(me, 4, ctypes.byref(state), ctypes.sizeof(state))
        k.SetPriorityClass(me, 0x00008000)  # ABOVE_NORMAL_PRIORITY_CLASS: speech is short and the user is waiting for it
        log("full speed: power throttling " + ("off" if ok else f"could not be turned off (error {ctypes.get_last_error()})"))
    except Exception as e:
        log(f"full speed: not set ({e})")


def wav(pcm):
    return b"RIFF" + struct.pack("<I", 36 + len(pcm)) + b"WAVEfmt " + struct.pack("<IHHIIHH", 16, 1, 1, RATE, RATE * 2, 2, 16) + b"data" + struct.pack("<I", len(pcm)) + pcm


class Handler(BaseHTTPRequestHandler):
    protocol_version = "HTTP/1.1"

    def log_message(self, *a): pass

    def send(self, code, body, kind="application/json"):
        self.send_response(code); self.send_header("Content-Type", kind); self.send_header("Content-Length", str(len(body))); self.end_headers(); self.wfile.write(body)

    def allowed(self):
        # Only Splainer: no web page (they always send Origin), only our own host name, and the per-install token.
        if self.headers.get("Origin") is not None or self.headers.get("Host", "") not in HOSTS:
            self.send(403, b'{"error":"forbidden"}'); return False
        if not TOKEN or not hmac.compare_digest(self.headers.get("Authorization", "").encode(), ("Bearer " + TOKEN).encode()):
            self.send(401, b'{"error":"unauthorized"}'); return False
        return True

    def do_GET(self):
        if not self.allowed(): return
        if self.path.rstrip("/") == "/v1/models":
            return self.send(200, json.dumps({"object": "list", "data": [{"id": "kokoro", "object": "model", "owned_by": "local"}, {"id": "whisper", "object": "model", "owned_by": "local"}]}).encode())
        if self.path.rstrip("/") == "/v1/audio/voices":
            return self.send(200, json.dumps({"voices": kokoro.get_voices(), "aliases": ALIASES, "listening": whisper is not None, "listening_ready": whisper_ready.is_set()}).encode())
        self.send(404, b"{}")

    def body(self, limit=1_000_000):
        # .NET's JsonContent sends the request chunked, without a Content-Length.
        if "chunked" in self.headers.get("Transfer-Encoding", "").lower():
            data = b""
            while True:
                size = int(self.rfile.readline().split(b";")[0].strip() or b"0", 16)
                if size == 0: self.rfile.readline(); return data
                data += self.rfile.read(size); self.rfile.readline()
                if len(data) > limit: raise ValueError("request too large")
        length = int(self.headers.get("Content-Length", 0))
        if length > limit: raise ValueError("request too large")
        return self.rfile.read(length)

    def transcription(self):
        kind = self.headers.get("Content-Type", "")
        if not kind.lower().startswith("multipart/form-data"): return self.send(415, b'{"error":"send multipart/form-data"}')
        message = BytesParser(policy=HTTP).parsebytes(b"Content-Type: " + kind.encode() + b"\r\n\r\n" + self.body(MAX_AUDIO + 65536))
        fields = {p.get_param("name", header="content-disposition"): p.get_payload(decode=True) for p in message.iter_parts()}
        if not fields.get("file"): return self.send(400, b'{"error":"file is missing"}')
        text = transcribe(fields["file"], (fields.get("language") or b"").decode(), (fields.get("prompt") or b"").decode()[:500])
        self.send(200, json.dumps({"text": text}).encode())

    def do_POST(self):
        if not self.allowed(): return
        if self.path.rstrip("/") == "/v1/audio/transcriptions":
            try: return self.transcription()
            except Exception as e: return self.send(500, json.dumps({"error": str(e)}).encode())
        if self.path.rstrip("/") != "/v1/audio/speech": return self.send(404, b"{}")
        if not self.headers.get("Content-Type", "").lower().startswith("application/json"): return self.send(415, b'{"error":"send JSON"}')
        try:
            req = json.loads(self.body() or b"{}")
            text = (req.get("input") or "").strip()
            if not text: return self.send(400, b'{"error":"input is empty"}')
            if len(text) > MAX_INPUT: return self.send(413, b'{"error":"input is longer than 4096 characters"}')
            voice, speed = resolve(req.get("voice")), float(req.get("speed") or 1.0)
            speed = min(2.0, max(0.5, speed))
            if req.get("response_format") == "pcm":
                # Stream: each sentence is sent as soon as it is spoken, so the first words play within a fraction of a second.
                self.send_response(200); self.send_header("Content-Type", "audio/pcm"); self.send_header("Transfer-Encoding", "chunked"); self.end_headers()
                for piece in sentences(text):
                    data = synth(piece, voice, speed)
                    self.wfile.write(f"{len(data):X}\r\n".encode() + data + b"\r\n"); self.wfile.flush()
                self.wfile.write(b"0\r\n\r\n"); return
            self.send(200, wav(b"".join(synth(p, voice, speed) for p in sentences(text))), "audio/wav")
        except (BrokenPipeError, ConnectionResetError):
            pass
        except Exception as e:
            self.send(500, json.dumps({"error": str(e)}).encode())


def main():
    global kokoro, TOKEN, FOLDER
    ap = argparse.ArgumentParser(); ap.add_argument("--models", required=True); ap.add_argument("--port", type=int, default=8880); ap.add_argument("--token-file", required=True)
    a = ap.parse_args()
    TOKEN = open(a.token_file, encoding="utf-8").read().strip()
    if len(TOKEN) < 32: sys.exit("the token file must hold a secret of at least 32 characters")
    FOLDER = a.models
    HOSTS.update({f"127.0.0.1:{a.port}", f"localhost:{a.port}"})
    full_speed()
    kokoro = Kokoro(os.path.join(a.models, "kokoro-v1.0.onnx"), os.path.join(a.models, "voices-v1.0.bin"))
    synth("Ready.", ALIASES["default"], 1.0)  # warm up so the first answer is quick
    threading.Thread(target=load_whisper, args=(a.models,), daemon=True).start()
    print(f"Splainer voice ready on http://127.0.0.1:{a.port}/v1", flush=True)
    ThreadingHTTPServer(("127.0.0.1", a.port), Handler).serve_forever()


if __name__ == "__main__": main()