| """Expose VoiceMem-Jev through Studio's routing lifecycle.""" |
| from .cuda_runtime import BUCKETS, CompiledCudaRouter |
| from .adapter import VoiceDepthRouter |
|
|
|
|
| class CompiledVoiceDepthRouter(VoiceDepthRouter): |
| """Keep host cancellation, caching and GPU admission around compiled calls.""" |
|
|
| def _load(self): |
| import torch |
| from voicemem.engine import cpu_thread_budget |
|
|
| threads = cpu_thread_budget() |
| if torch.get_num_threads() != threads: |
| torch.set_num_threads(threads) |
| print(f'[jev] Torch CPU thread budget fixed before compilation: {threads}', flush=True) |
|
|
| runtime = CompiledCudaRouter(self.path, device=self.device) |
| runtime._load() |
| self._model, self._tokenizer = runtime.model, runtime.tokenizer |
| with torch.inference_mode(): |
| for bucket in BUCKETS: |
| ids = torch.full((2, bucket), self._tokenizer.pad_token_id, |
| dtype=torch.long, device=self.device) |
| ids[:, 0] = self._tokenizer.cls_token_id |
| ids[:, -1] = self._tokenizer.sep_token_id |
| mask = torch.ones_like(ids) |
| observed = torch.zeros((1, 7), dtype=torch.float32, device=self.device) |
| for _ in range(3): |
| self._model(ids, mask, observed).float().cpu().numpy().copy() |
| if self._model.execution_mode == 'compiled-cuda': |
| print('[jev] VoiceMem-Jev: compiled CUDA runtime ready; all five buckets warm', flush=True) |
| else: |
| print('[jev] VoiceMem-Jev: same-weight eager CUDA fallback ready', flush=True) |
|
|
|
|
| def create(*, model, base_url='', api_key='', options=None): |
| """Select the existing eager/ONNX adapter or opt into compiled CUDA.""" |
| options = dict(options or {}) |
| compiled = options.pop('compiled', False) |
| if type(compiled) is not bool: |
| raise ValueError('compiled must be a boolean') |
| if compiled and (options.get('backend', 'onnx') != 'eager' |
| or not str(options.get('device', 'cpu')).startswith('cuda')): |
| raise ValueError('Compiled inference requires eager CUDA') |
| from .adapter import create as ordinary |
| adapter = ordinary(model=model, base_url=base_url, api_key=api_key, options=options) |
| if not compiled: |
| return adapter |
| if adapter.backend != 'eager' or not adapter.device.startswith('cuda'): |
| raise ValueError('Compiled inference requires eager CUDA') |
| return CompiledVoiceDepthRouter(model, **options) |
|
|
