Upload folder using huggingface_hub

README.md

@@ -15,15 +15,18 @@ tags:
 
 # GSE Labs Chat (ZeroGPU)
 
-A Gradio chat UI running **Qwen2.5-14B-Instruct** on Hugging Face's free
-**ZeroGPU** hardware (time-sliced Nvidia GPU, allocated per request).
+A personal Gradio chat UI running **Qwen2.5-32B-Instruct** (4-bit) on
+Hugging Face's free **ZeroGPU** hardware (time-sliced Nvidia GPU, allocated
+per request). Personal-use tool, not part of the GSE product surface.
 
 No API key, no per-token billing — inference runs on the Space's shared free
-GPU quota instead of a paid hosted API.
+GPU quota instead of a paid hosted API. Responses stream token-by-token, and
+the GPU time requested per reply scales with how many tokens you ask for
+(`_duration()` in `app.py`), instead of a fixed cap.
 
 ## Swapping the model
 
 Change `MODEL_ID` in `app.py` to any other instruct-tuned model on the Hub.
-Bigger models cost more GPU-seconds per reply and are more likely to hit
-ZeroGPU's per-call time limit (currently a few minutes) — if you go bigger,
-test with short prompts first.
+Current footprint (~16GB in 4-bit) leaves headroom under the ~29.5GB that
+already loaded successfully on this Space's hardware — going meaningfully
+bigger than that is unverified and worth testing with short prompts first.

app.py

@@ -1,19 +1,34 @@
+import threading
+
 import spaces
 import torch
 import gradio as gr
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import (
+    AutoModelForCausalLM,
+    AutoTokenizer,
+    BitsAndBytesConfig,
+    TextIteratorStreamer,
+)
 
-MODEL_ID = "Qwen/Qwen2.5-14B-Instruct"
+MODEL_ID = "Qwen/Qwen2.5-32B-Instruct"
 
 tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
 model = AutoModelForCausalLM.from_pretrained(
     MODEL_ID,
-    torch_dtype=torch.bfloat16,
+    quantization_config=BitsAndBytesConfig(
+        load_in_4bit=True,
+        bnb_4bit_quant_type="nf4",
+        bnb_4bit_compute_dtype=torch.bfloat16,
+    ),
     device_map="auto",
 )
 
 
-@spaces.GPU(duration=90)
+def _duration(message, history, system_prompt, max_new_tokens, temperature):
+    return min(120, 40 + int(max_new_tokens) // 10)
+
+
+@spaces.GPU(duration=_duration)
 def generate(message, history, system_prompt, max_new_tokens, temperature):
     messages = [{"role": "system", "content": system_prompt}] if system_prompt else []
     for turn in history:
@@ -24,17 +39,25 @@ def generate(message, history, system_prompt, max_new_tokens, temperature):
         messages, add_generation_prompt=True, return_tensors="pt", return_dict=True
     ).to(model.device)
 
-    output = model.generate(
+    streamer = TextIteratorStreamer(
+        tokenizer, skip_prompt=True, skip_special_tokens=True
+    )
+    gen_kwargs = dict(
         **inputs,
         max_new_tokens=int(max_new_tokens),
         temperature=float(temperature),
         do_sample=temperature > 0,
         pad_token_id=tokenizer.eos_token_id,
+        streamer=streamer,
     )
-    reply = tokenizer.decode(
-        output[0][inputs["input_ids"].shape[-1]:], skip_special_tokens=True
-    )
-    return reply
+    thread = threading.Thread(target=model.generate, kwargs=gen_kwargs)
+    thread.start()
+
+    partial = ""
+    for token in streamer:
+        partial += token
+        yield partial
+    thread.join()
 
 
 demo = gr.ChatInterface(
@@ -46,7 +69,7 @@ demo = gr.ChatInterface(
         gr.Slider(0.0, 1.5, value=0.7, step=0.1, label="Temperature"),
     ],
     title="GSE Labs Chat (ZeroGPU)",
-    description=f"Running `{MODEL_ID}` on Hugging Face's free ZeroGPU hardware.",
+    description=f"Personal free-GPU chat — `{MODEL_ID}` (4-bit), streaming, on ZeroGPU.",
 )
 
 if __name__ == "__main__":

requirements.txt

@@ -3,3 +3,4 @@ gradio>=5.9.1
 transformers>=4.46.0
 accelerate>=1.1.0
 torch>=2.4.0
+bitsandbytes>=0.44.0