ZkhanNewmetrics commited on
Commit
2375413
·
verified ·
1 Parent(s): c97f977

Add self-contained NeMo SALM server for canary-qwen-2.5b

Browse files
Files changed (1) hide show
  1. serve_canary.py +72 -0
serve_canary.py ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Minimal self-contained ASR server for nvidia/canary-qwen-2.5b, using NeMo's
2
+ own documented SALM API directly (this model is NeMo-native, not a
3
+ `transformers` pipeline model - the standard Endpoint ASR task container
4
+ can't load it at all).
5
+
6
+ Contract: POST raw audio bytes to "/", get back {"text": "..."}. GET "/health" -> 200.
7
+ """
8
+ import http.server
9
+ import json
10
+ import socketserver
11
+ import tempfile
12
+
13
+ from nemo.collections.speechlm2.models import SALM
14
+
15
+ MODEL_ID = "nvidia/canary-qwen-2.5b"
16
+
17
+ print("Loading SALM model (downloads ~5GB, can take a while)...", flush=True)
18
+ model = SALM.from_pretrained(MODEL_ID)
19
+ print("Model loaded, ready to serve.", flush=True)
20
+
21
+
22
+ def transcribe(raw_bytes: bytes) -> str:
23
+ with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
24
+ f.write(raw_bytes)
25
+ path = f.name
26
+ answer_ids = model.generate(
27
+ prompts=[
28
+ [{"role": "user", "content": f"Transcribe the following: {model.audio_locator_tag}", "audio": [path]}]
29
+ ],
30
+ max_new_tokens=128,
31
+ )
32
+ return model.tokenizer.ids_to_text(answer_ids[0].cpu())
33
+
34
+
35
+ class Handler(http.server.BaseHTTPRequestHandler):
36
+ def do_GET(self):
37
+ if self.path == "/health":
38
+ self.send_response(200)
39
+ self.end_headers()
40
+ self.wfile.write(b"ok")
41
+ else:
42
+ self.send_response(404)
43
+ self.end_headers()
44
+
45
+ def do_POST(self):
46
+ length = int(self.headers.get("Content-Length", 0))
47
+ raw = self.rfile.read(length)
48
+ try:
49
+ text = transcribe(raw)
50
+ body = json.dumps({"text": text}, ensure_ascii=False).encode("utf-8")
51
+ status = 200
52
+ except Exception as e: # noqa: BLE001 - surfaced to the caller, not swallowed
53
+ body = json.dumps({"error": str(e)}).encode("utf-8")
54
+ status = 500
55
+ self.send_response(status)
56
+ self.send_header("Content-Type", "application/json; charset=utf-8")
57
+ self.send_header("Content-Length", str(len(body)))
58
+ self.end_headers()
59
+ self.wfile.write(body)
60
+
61
+ def log_message(self, fmt, *args):
62
+ print(f"[serve] {self.address_string()} - {fmt % args}", flush=True)
63
+
64
+
65
+ class ThreadingServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
66
+ daemon_threads = True
67
+
68
+
69
+ if __name__ == "__main__":
70
+ port = 8000
71
+ print(f"Serving on 0.0.0.0:{port}", flush=True)
72
+ ThreadingServer(("0.0.0.0", port), Handler).serve_forever()