PRIYANSHUDHAKED commited on
Commit
df83106
·
verified ·
1 Parent(s): a152b32

fill the content in code file.

Browse files

by mistake file pushed without content.

Files changed (1) hide show
  1. mio_codec_wrapper.py +215 -0
mio_codec_wrapper.py CHANGED
@@ -0,0 +1,215 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """mio_codec_wrapper.py
3
+
4
+ Wrapper to run MioCodec in two modes:
5
+ - streaming: connects to a websocket (e.g. vLLM/your LLM) that emits
6
+ token embeddings (JSON messages) and decodes them to audio in near
7
+ real-time using the windowed streamer.
8
+ - offline: loads a JSONL file of embeddings and decodes to a WAV file.
9
+
10
+ Requirements (minimal): torch, numpy, websockets (for streaming), soundfile (for offline write), sounddevice (optional playback).
11
+
12
+ Install quickly without cloning full repo (if upstream exists):
13
+ pip install git+https://github.com/Aratako/MioCodec.git
14
+
15
+ If you already have local helper files (`streaming_istft_head.py`, `windowed_streamer.py`), the wrapper will import them; otherwise it will try to use `miocodec` package's model and swap in the streaming head.
16
+ """
17
+ from __future__ import annotations
18
+
19
+ import argparse
20
+ import asyncio
21
+ import json
22
+ import sys
23
+ from pathlib import Path
24
+
25
+ import numpy as np
26
+ import torch
27
+
28
+
29
+ def build_args():
30
+ p = argparse.ArgumentParser()
31
+ p.add_argument("--mode", choices=("stream", "offline"), default="stream")
32
+ p.add_argument("--uri", default="ws://localhost:8000/stream", help="Websocket URI for streaming mode")
33
+ p.add_argument("--input", help="JSONL file with embeddings for offline mode")
34
+ p.add_argument("--out", default="out.wav", help="Output WAV file for offline mode")
35
+ p.add_argument("--device", default="cpu")
36
+ p.add_argument("--repo-id", default="Aratako/MioCodec-25Hz-24kHz")
37
+ p.add_argument("--play", action="store_true", help="Play decoded audio while streaming")
38
+ p.add_argument("--chunk-size", type=int, default=20)
39
+ p.add_argument("--left", type=int, default=10)
40
+ p.add_argument("--right", type=int, default=10)
41
+ return p.parse_args()
42
+
43
+
44
+ def load_model_and_streamer(repo_id: str, device: str, chunk_size: int, left: int, right: int):
45
+ """Return (model, streamer, sample_rate). Attempts to import local windowed_streamer first."""
46
+ try:
47
+ # Prefer local helper module if present
48
+ from windowed_streamer import build_windowed_model, MioCodecWindowedStreamer, WindowedStreamConfig
49
+
50
+ model = build_windowed_model(repo_id=repo_id, device=device)
51
+ cfg = model.config
52
+ stream_cfg = WindowedStreamConfig(chunk_size=chunk_size, left_context_tokens=left, right_lookahead_tokens=right)
53
+ # create a default global embedding (zero) sized to model's expected dim
54
+ gdim = getattr(cfg, "global_embedding_dim", getattr(cfg, "wave_decoder_dim", 512))
55
+ global_embedding = torch.zeros(gdim, dtype=torch.float32).to(device)
56
+ streamer = MioCodecWindowedStreamer(model, global_embedding, config=stream_cfg)
57
+ return model, streamer, getattr(cfg, "sample_rate", 24000)
58
+ except Exception:
59
+ # Fallback: import MioCodecModel from installed package and swap head
60
+ from miocodec.model import MioCodecModel
61
+ try:
62
+ from streaming_istft_head import ISTFTHead as StreamingISTFTHead
63
+ except Exception:
64
+ StreamingISTFTHead = None
65
+
66
+ model = MioCodecModel.from_pretrained(repo_id).to(device)
67
+ cfg = model.config
68
+ if StreamingISTFTHead is not None:
69
+ old_head = model.istft_head
70
+ new_head = StreamingISTFTHead(dim=cfg.wave_decoder_dim, n_fft=cfg.n_fft, hop_length=cfg.hop_length, padding=cfg.istft_padding).to(device)
71
+ new_head.out.load_state_dict(old_head.out.state_dict())
72
+ model.istft_head = new_head
73
+
74
+ # create a simple streamer-like helper using WindowedStreamConfig from windowed_streamer if available
75
+ try:
76
+ from windowed_streamer import MioCodecWindowedStreamer, WindowedStreamConfig
77
+
78
+ stream_cfg = WindowedStreamConfig(chunk_size=chunk_size, left_context_tokens=left, right_lookahead_tokens=right)
79
+ gdim = getattr(cfg, "global_embedding_dim", getattr(cfg, "wave_decoder_dim", 512))
80
+ global_embedding = torch.zeros(gdim, dtype=torch.float32).to(device)
81
+ streamer = MioCodecWindowedStreamer(model, global_embedding, config=stream_cfg)
82
+ except Exception:
83
+ # Minimal fallback: use a naive one-shot decoder wrapper for full sequences
84
+ streamer = None
85
+
86
+ return model, streamer, getattr(cfg, "sample_rate", 24000)
87
+
88
+
89
+ async def stream_mode(uri: str, model, streamer, samplerate: int, play: bool, device: str):
90
+ """Connect to websocket; expect messages with {"embedding": [...]} or {"finished": true}."""
91
+ try:
92
+ import websockets
93
+ except Exception:
94
+ print("Install websockets: pip install websockets", file=sys.stderr)
95
+ return
96
+
97
+ sd = None
98
+ if play:
99
+ try:
100
+ import sounddevice as sd
101
+ except Exception:
102
+ sd = None
103
+
104
+ async with websockets.connect(uri) as ws:
105
+ print("connected", uri)
106
+ async for raw in ws:
107
+ try:
108
+ msg = json.loads(raw)
109
+ except Exception:
110
+ continue
111
+
112
+ if msg.get("finished"):
113
+ if streamer is not None:
114
+ final = streamer.finish()
115
+ if final is not None:
116
+ arr = final.cpu().numpy().astype(np.float32)
117
+ if sd is not None:
118
+ sd.play(arr, samplerate)
119
+ sd.wait()
120
+ else:
121
+ print("final chunk ready (not played)")
122
+ break
123
+
124
+ emb = msg.get("embedding")
125
+ if emb is None:
126
+ continue
127
+ tensor = torch.tensor(emb, dtype=torch.float32).to(device)
128
+ if streamer is not None:
129
+ audio = streamer.push_token(tensor)
130
+ if audio is not None:
131
+ arr = audio.cpu().numpy().astype(np.float32)
132
+ if sd is not None:
133
+ sd.play(arr, samplerate)
134
+ # do not wait to allow continuous streaming
135
+ else:
136
+ sys.stdout.buffer.write(arr.tobytes())
137
+ else:
138
+ # no streamer: buffer and decode once finished
139
+ # simple approach: accumulate embeddings on model directly if model supports method
140
+ if not hasattr(stream_mode, "buffer"):
141
+ stream_mode.buffer = []
142
+ stream_mode.buffer.append(tensor.cpu().numpy())
143
+
144
+
145
+ def write_wav(path: Path, arr: np.ndarray, samplerate: int):
146
+ try:
147
+ import soundfile as sf
148
+
149
+ sf.write(str(path), arr, samplerate)
150
+ except Exception:
151
+ # fallback: write raw float32
152
+ path.write_bytes(arr.astype(np.float32).tobytes())
153
+
154
+
155
+ def offline_mode(input_path: Path, out_path: Path, model, streamer, samplerate: int, device: str):
156
+ """Read JSONL file of {embedding: [...]} lines and decode offline."""
157
+ embeddings = []
158
+ with input_path.open("r") as f:
159
+ for line in f:
160
+ try:
161
+ m = json.loads(line)
162
+ except Exception:
163
+ continue
164
+ if "embedding" in m:
165
+ embeddings.append(torch.tensor(m["embedding"], dtype=torch.float32).to(device))
166
+
167
+ if streamer is not None:
168
+ # push all and finish
169
+ for emb in embeddings:
170
+ streamer.push_token(emb)
171
+ final = streamer.finish()
172
+ if final is not None:
173
+ arr = final.cpu().numpy().astype(np.float32)
174
+ write_wav(out_path, arr, samplerate)
175
+ print("Wrote", out_path)
176
+ return
177
+
178
+ # fallback one-shot: try to concatenate and run model-specific decode path
179
+ if hasattr(model, "istft_head") and hasattr(model, "wave_prenet"):
180
+ # build a content embedding and run forward once (best-effort)
181
+ all_tokens = torch.stack(embeddings, dim=0).unsqueeze(0).to(device)
182
+ with torch.no_grad():
183
+ local_latent = model.wave_prenet(all_tokens)
184
+ if model.wave_conv_upsample is not None:
185
+ local_latent = model.wave_conv_upsample(local_latent.transpose(1, 2)).transpose(1, 2)
186
+ # approximate frames
187
+ frames = local_latent.shape[1] * 100 # heuristic; prefer streamer for correctness
188
+ local_latent = torch.nn.functional.interpolate(local_latent.transpose(1, 2), size=frames, mode=model.config.wave_interpolation_mode).transpose(1, 2)
189
+ local_latent = model.wave_prior_net(local_latent.transpose(1, 2)).transpose(1, 2)
190
+ local_latent = model.wave_decoder(local_latent, condition=torch.zeros(1, 1, local_latent.shape[-1], device=device))
191
+ local_latent = model.wave_post_net(local_latent.transpose(1, 2)).transpose(1, 2)
192
+ audio = model.istft_head(local_latent).squeeze(0).cpu().numpy().astype(np.float32)
193
+ write_wav(out_path, audio, samplerate)
194
+ print("Wrote", out_path)
195
+ return
196
+
197
+ print("Could not decode offline: no streamer and model lacks decode path.")
198
+
199
+
200
+ def main():
201
+ args = build_args()
202
+ device = args.device
203
+ model, streamer, samplerate = load_model_and_streamer(args.repo_id, device, args.chunk_size, args.left, args.right)
204
+
205
+ if args.mode == "stream":
206
+ asyncio.run(stream_mode(args.uri, model, streamer, samplerate, args.play, device))
207
+ else:
208
+ if not args.input:
209
+ print("--input required for offline mode", file=sys.stderr)
210
+ return
211
+ offline_mode(Path(args.input), Path(args.out), model, streamer, samplerate, device)
212
+
213
+
214
+ if __name__ == "__main__":
215
+ main()