"""Native CUDA GGUF generation, loaded only inside a ZeroGPU worker.""" import time import spaces @spaces.GPU(duration=120) def generate_gguf(path, messages, max_tokens, temperature): import ctypes import glob import importlib # PyTorch's CUDA libraries are already installed; expose them to ctypes loaders. for module in ("nvidia.cuda_runtime", "nvidia.cublas", "nvidia.cuda_nvrtc"): package = importlib.import_module(module) for location in package.__path__: for library in glob.glob(location + "/lib/*.so*"): try: ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL) except OSError: pass from llama_cpp import Llama, llama_supports_gpu_offload if not llama_supports_gpu_offload(): raise RuntimeError("The GGUF runtime has no GPU support; CPU fallback is disabled.") started = time.monotonic() llm = Llama(model_path=path, n_gpu_layers=-1, n_ctx=8192, n_batch=256, n_threads=2, verbose=False) try: if time.monotonic() - started > 20: raise RuntimeError("This model takes too long to load within the 120-second quota budget. Use a smaller GGUF.") if len(llm.tokenize(('\n'.join(m['content'] for m in messages)).encode())) > 1600: raise ValueError("Conversation too long for this GGUF demo. Start a new chat or shorten the message.") text = "" for event in llm.create_chat_completion(messages=messages, max_tokens=min(max(int(max_tokens), 1), 4096), temperature=float(temperature), top_p=0.9, stream=True): text += event['choices'][0].get('delta', {}).get('content', '') or '' yield text if time.monotonic() - started >= 110: break finally: llm.close()