AstheralPlus / gguf_runtime.py
SEE7's picture
Allow 4096-token replies and 110-second generation with 180-second quota reserve
7244dd9 verified
Raw History Blame Contribute Delete
1.85 kB
"""Native CUDA GGUF generation, loaded only inside a ZeroGPU worker."""
import time
import spaces
@spaces.GPU(duration=120)
def generate_gguf(path, messages, max_tokens, temperature):
import ctypes
import glob
import importlib
# PyTorch's CUDA libraries are already installed; expose them to ctypes loaders.
for module in ("nvidia.cuda_runtime", "nvidia.cublas", "nvidia.cuda_nvrtc"):
package = importlib.import_module(module)
for location in package.__path__:
for library in glob.glob(location + "/lib/*.so*"):
try:
ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL)
except OSError:
pass
from llama_cpp import Llama, llama_supports_gpu_offload
if not llama_supports_gpu_offload():
raise RuntimeError("The GGUF runtime has no GPU support; CPU fallback is disabled.")
started = time.monotonic()
llm = Llama(model_path=path, n_gpu_layers=-1, n_ctx=8192,
n_batch=256, n_threads=2, verbose=False)
try:
if time.monotonic() - started > 20:
raise RuntimeError("This model takes too long to load within the 120-second quota budget. Use a smaller GGUF.")
if len(llm.tokenize(('\n'.join(m['content'] for m in messages)).encode())) > 1600:
raise ValueError("Conversation too long for this GGUF demo. Start a new chat or shorten the message.")
text = ""
for event in llm.create_chat_completion(messages=messages, max_tokens=min(max(int(max_tokens), 1), 4096),
temperature=float(temperature), top_p=0.9, stream=True):
text += event['choices'][0].get('delta', {}).get('content', '') or ''
yield text
if time.monotonic() - started >= 110:
break
finally:
llm.close()