Spaces:
Running on Zero
Running on Zero
| import os | |
| import sys | |
| import subprocess | |
| import re | |
| # ============================================================= | |
| # 1. Bootstrap prebuilt llama-cpp-python | |
| # ============================================================= | |
| def bootstrap_llama_cpp(): | |
| print("Bootstrapping llama-cpp-python...") | |
| from huggingface_hub import HfApi, hf_hub_download | |
| api = HfApi() | |
| files = api.list_repo_files("AIencoder/llama-cpp-wheels", repo_type="dataset") | |
| py_tag = f"cp{sys.version_info.major}{sys.version_info.minor}" | |
| candidates = [] | |
| for p in files: | |
| if not p.endswith(".whl") or "linux/x86_64/cpu/" not in p or "cuda" in p.lower(): | |
| continue | |
| if py_tag not in p and "py3-none" not in p: | |
| continue | |
| m = re.search(r"/0\.3\.(\d+)/", p) | |
| if not m or int(m.group(1)) < 20: | |
| continue | |
| candidates.append((int(m.group(1)), p)) | |
| candidates.sort(key=lambda x: x[0], reverse=True) | |
| chosen = candidates[0][1] | |
| print(f"Selected wheel: {chosen}") | |
| whl_path = hf_hub_download("AIencoder/llama-cpp-wheels", chosen, repo_type="dataset") | |
| subprocess.check_call([ | |
| sys.executable, "-m", "pip", "install", "--force-reinstall", whl_path | |
| ]) | |
| print("llama-cpp-python ready!") | |
| try: | |
| from llama_cpp import Llama | |
| print("llama-cpp-python already available.") | |
| except (ImportError, RuntimeError, OSError): | |
| bootstrap_llama_cpp() | |
| from llama_cpp import Llama | |
| import spaces | |
| from typing import List | |
| import gradio as gr | |
| # ============================================================= | |
| # 2. Load GGUF Model | |
| # ============================================================= | |
| MODEL_PATH = "Qwen3.5-0.8B.Q4_K_M.gguf" | |
| print(f"Loading {MODEL_PATH} via llama-cpp-python ...") | |
| llm = Llama( | |
| model_path=MODEL_PATH, | |
| n_ctx=4096, | |
| n_threads=2, | |
| verbose=False | |
| ) | |
| print("Model loaded successfully!") | |
| SYSTEM_PROMPT = ( | |
| "You are Sherpa, an AI driving assistant for the CommitCon ride-hailing " | |
| "repositioning platform in Bengaluru. Provide concise, direct, helpful " | |
| "recommendations to drivers about where to reposition for high demand, " | |
| "earnings, and low traffic delays." | |
| ) | |
| def run_inference(prompt: str, history: List[dict] = None) -> str: | |
| messages = [{"role": "system", "content": SYSTEM_PROMPT}] | |
| if history: | |
| for m in history: | |
| messages.append({"role": m["role"], "content": m["content"]}) | |
| messages.append({"role": "user", "content": prompt}) | |
| response = llm.create_chat_completion( | |
| messages=messages, temperature=0.7, max_tokens=256 | |
| ) | |
| return response["choices"][0]["message"]["content"] | |
| # ============================================================= | |
| # 3. Gradio Interface (duration=15 ensures ZeroGPU quota is preserved) | |
| # ============================================================= | |
| def gradio_chat(user_message, history): | |
| hist_formatted = [] | |
| if history: | |
| for item in history: | |
| if isinstance(item, (list, tuple)) and len(item) == 2: | |
| u, b = item | |
| hist_formatted.append({"role": "user", "content": str(u)}) | |
| if b: | |
| hist_formatted.append({"role": "assistant", "content": str(b)}) | |
| elif isinstance(item, dict): | |
| hist_formatted.append(item) | |
| return run_inference(user_message, hist_formatted) | |
| demo = gr.ChatInterface( | |
| fn=gradio_chat, | |
| title="🧭 CommitCon Sherpa Driver Copilot", | |
| description="Fine-tuned Qwen 0.8B Q4_K_M — llama-cpp-python inference", | |
| api_name="chat" | |
| ) | |
| # ============================================================= | |
| # 4. Native ZeroGPU Launch | |
| # ============================================================= | |
| if __name__ == "__main__": | |
| demo.launch() |