Switch Analyse to free ZeroGPU local Qwen (no Inference Providers)

README.md

@@ -8,6 +8,9 @@ sdk_version: 6.13.0
 app_file: app.py
 pinned: true
 license: other
+suggested_hardware: zero-a10g
+tags:
+  - zerogpu
 ---
 
 # ⚡ The Metabolic Phoenix Framework

app.py

@@ -19,84 +19,55 @@ from docx.enum.text import WD_ALIGN_PARAGRAPH
 SAVE_DIR = Path("/data/saved_docs")
 SAVE_DIR.mkdir(parents=True, exist_ok=True)
 
-# Llama 3.3 70B is supported via the Hugging Face Router
-MODEL = "meta-llama/Llama-3.3-70B-Instruct"
+# Local instruct model on ZeroGPU — no paid Inference Providers
+LOCAL_MODEL_ID = "Qwen/Qwen2.5-1.5B-Instruct"
 
 
 # =========================
-# SHARED: TOKEN / MODEL / TTS
+# SHARED: LOCAL MODEL / TTS
 # =========================
 
-def _resolve_hf_token(user_token=None):
-    """Resolve HF token from Space/env secrets only. Strip quotes accidentally pasted into secrets."""
-    candidates = [
-        user_token,
-        os.environ.get("HF_TOKEN"),
-        os.environ.get("HUGGINGFACE_HUB_TOKEN"),
-        os.environ.get("HUGGING_FACE_HUB_TOKEN"),
-    ]
-    for raw in candidates:
-        if not raw:
-            continue
-        token = str(raw).strip().strip('"').strip("'")
-        if token:
-            return token
-    return ""
+import spaces
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
 
+_tokenizer = AutoTokenizer.from_pretrained(LOCAL_MODEL_ID, trust_remote_code=True)
+_model = AutoModelForCausalLM.from_pretrained(
+    LOCAL_MODEL_ID,
+    torch_dtype=torch.bfloat16,
+    device_map="cuda",
+    trust_remote_code=True,
+)
 
+
+@spaces.GPU(duration=120)
 def call_llama(prompt, user_token=None, max_tokens=1200):
     """
-    Pulls Llama directly via Hugging Face using HF_TOKEN.
-    Prioritizes user_token from UI if provided, otherwise looks for HF_TOKEN env var.
-    Uses the OpenAI-compatible Router API for Llama 3.3 70B.
+    Runs Qwen2.5-1.5B on the Space's free ZeroGPU slice.
+    Does not call Hugging Face Inference Providers (no extra credit spend).
+    user_token is ignored and kept only for call-site compatibility.
     """
-    token = _resolve_hf_token(user_token)
-
-    if not token:
-        return (
-            "Model error: HF_TOKEN missing. Create a Hugging Face access token at "
-            "https://huggingface.co/settings/tokens (needs Inference permission), "
-            "then add it as a Space secret named HF_TOKEN."
-        )
-
-    # Do not hard-fail on prefix: classic tokens start with hf_, but some valid
-    # Hub credentials do not. Let the Router reject bad credentials instead.
-
     try:
-        # Use the Hugging Face Router endpoint which supports Llama 3.3 70B
-        url = "https://router.huggingface.co/v1/chat/completions"
-        headers = {
-            "Authorization": f"Bearer {token}",
-            "Content-Type": "application/json",
-        }
-        
-        # Router uses OpenAI-style chat completions payload
-        payload = {
-            "model": MODEL,
-            "messages": [
-                {"role": "user", "content": prompt}
-            ],
-            "max_tokens": max_tokens,
-            "temperature": 0.4,
-            "stream": False
-        }
-
-        # Increased timeout for large case summaries
-        response = requests.post(url, headers=headers, json=payload, timeout=180)
-
-        if response.status_code != 200:
-            return f"Model error: Hugging Face Router returned {response.status_code}: {response.text[:1200]}"
-
-        data = response.json()
-
-        if "choices" in data and len(data["choices"]) > 0:
-            return data["choices"][0]["message"]["content"]
-        
-        if "error" in data:
-            return f"Model error: {data['error']}"
-            
-        return str(data)
-
+        messages = [{"role": "user", "content": str(prompt)[:12000]}]
+        chat = _tokenizer.apply_chat_template(
+            messages,
+            tokenize=False,
+            add_generation_prompt=True,
+        )
+        inputs = _tokenizer(chat, return_tensors="pt", truncation=True, max_length=4096)
+        inputs = {k: v.to(_model.device) for k, v in inputs.items()}
+        with torch.no_grad():
+            out = _model.generate(
+                **inputs,
+                max_new_tokens=min(int(max_tokens or 1200), 1024),
+                do_sample=True,
+                temperature=0.4,
+                top_p=0.9,
+                pad_token_id=_tokenizer.eos_token_id,
+            )
+        gen = out[0][inputs["input_ids"].shape[-1]:]
+        text = _tokenizer.decode(gen, skip_special_tokens=True).strip()
+        return text or "[Empty model response]"
     except Exception as e:
         return f"Model error: {e}"
 

requirements.txt

@@ -1,4 +1,10 @@
 gradio
+spaces
+torch
+transformers>=4.45.0
+accelerate
+sentencepiece
+einops
 requests
 gTTS
 pypdf