Spaces:
Paused
Paused
Download gguf_runtime.py from SEE7/AstheralPlus: direct link, hf CLI and curl.
- Browser
- Download file 1.85 kB
-
https://huggingface.co/spaces/SEE7/AstheralPlus/resolve/main/gguf_runtime.py
- Command line
-
hf download hf://spaces/SEE7/AstheralPlus/gguf_runtime.py
-
curl -L -o gguf_runtime.py https://huggingface.co/spaces/SEE7/AstheralPlus/resolve/main/gguf_runtime.py
1.85 kB
| """Native CUDA GGUF generation, loaded only inside a ZeroGPU worker.""" | |
| import time | |
| import spaces | |
| def generate_gguf(path, messages, max_tokens, temperature): | |
| import ctypes | |
| import glob | |
| import importlib | |
| # PyTorch's CUDA libraries are already installed; expose them to ctypes loaders. | |
| for module in ("nvidia.cuda_runtime", "nvidia.cublas", "nvidia.cuda_nvrtc"): | |
| package = importlib.import_module(module) | |
| for location in package.__path__: | |
| for library in glob.glob(location + "/lib/*.so*"): | |
| try: | |
| ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL) | |
| except OSError: | |
| pass | |
| from llama_cpp import Llama, llama_supports_gpu_offload | |
| if not llama_supports_gpu_offload(): | |
| raise RuntimeError("The GGUF runtime has no GPU support; CPU fallback is disabled.") | |
| started = time.monotonic() | |
| llm = Llama(model_path=path, n_gpu_layers=-1, n_ctx=8192, | |
| n_batch=256, n_threads=2, verbose=False) | |
| try: | |
| if time.monotonic() - started > 20: | |
| raise RuntimeError("This model takes too long to load within the 120-second quota budget. Use a smaller GGUF.") | |
| if len(llm.tokenize(('\n'.join(m['content'] for m in messages)).encode())) > 1600: | |
| raise ValueError("Conversation too long for this GGUF demo. Start a new chat or shorten the message.") | |
| text = "" | |
| for event in llm.create_chat_completion(messages=messages, max_tokens=min(max(int(max_tokens), 1), 4096), | |
| temperature=float(temperature), top_p=0.9, stream=True): | |
| text += event['choices'][0].get('delta', {}).get('content', '') or '' | |
| yield text | |
| if time.monotonic() - started >= 110: | |
| break | |
| finally: | |
| llm.close() | |