myspace / app.py
AADIplayz23's picture
Upload 3 files
bcce2c9 verified
Raw History Blame Contribute Delete
7.35 kB
import os
import torch
import gradio as gr
import spaces
from transformers import (
AutoTokenizer,
AutoModelForCausalLM,
BitsAndBytesConfig,
TextIteratorStreamer,
)
from diffusers import AutoPipelineForText2Image
from threading import Thread
# ============================================================
# MODEL CONFIGURATION
# ============================================================
LLM_MODEL_ID = "Qwen/Qwen3.8-27B"
IMAGE_MODEL_ID = "stabilityai/stable-diffusion-xl-base-1.0"
print(f"[BOOT] Initializing Nova AI Backend on Hugging Face ZeroGPU...")
# Pre-load tokenizer on CPU (lightweight, instant)
tokenizer = None
try:
print(f"[BOOT] Loading Tokenizer: {LLM_MODEL_ID}...")
tokenizer = AutoTokenizer.from_pretrained(LLM_MODEL_ID, trust_remote_code=True)
print(f"[BOOT] Tokenizer loaded successfully!")
except Exception as e:
print(f"[WARN] Tokenizer deferred loading: {e}")
# Global references for lazy loading inside @spaces.GPU
llm_model = None
pipe_image = None
def get_tokenizer():
global tokenizer
if tokenizer is None:
tokenizer = AutoTokenizer.from_pretrained(LLM_MODEL_ID, trust_remote_code=True)
return tokenizer
def get_llm():
global llm_model
if llm_model is None:
print(f"[GPU] Loading LLM Model to ZeroGPU: {LLM_MODEL_ID} (4-bit)...")
bnb_config = BitsAndBytesConfig(
load_in_4bit=True,
bnb_4bit_compute_dtype=torch.bfloat16,
bnb_4bit_quant_type="nf4",
bnb_4bit_use_double_quant=True,
llm_int8_enable_fp32_cpu_offload=True,
)
llm_model = AutoModelForCausalLM.from_pretrained(
LLM_MODEL_ID,
quantization_config=bnb_config,
device_map="auto",
torch_dtype=torch.bfloat16,
trust_remote_code=True,
)
print(f"[GPU] LLM Model loaded successfully onto ZeroGPU!")
return llm_model
def get_image_pipe():
global pipe_image
if pipe_image is None:
print(f"[GPU] Loading SDXL Image Model to ZeroGPU: {IMAGE_MODEL_ID}...")
pipe_image = AutoPipelineForText2Image.from_pretrained(
IMAGE_MODEL_ID,
torch_dtype=torch.bfloat16,
use_safetensors=True,
)
pipe_image.to("cuda")
print(f"[GPU] Image Pipeline loaded successfully!")
return pipe_image
# ============================================================
# INFERENCE ENDPOINTS (ZeroGPU Dynamic Allocation)
# ============================================================
def chat_inference(message, history, system_prompt, temperature, max_tokens, gpu_duration=60):
"""
Dynamic ZeroGPU chat inference.
gpu_duration can be customized (e.g., 10s to 120s).
"""
req_duration = max(10, min(120, int(gpu_duration)))
@spaces.GPU(duration=req_duration)
def _inner_chat_gen():
tok = get_tokenizer()
model = get_llm()
messages = []
if system_prompt:
messages.append({"role": "system", "content": system_prompt})
if history:
for item in history:
if isinstance(item, (list, tuple)) and len(item) == 2:
u, b = item
if u: messages.append({"role": "user", "content": str(u)})
if b: messages.append({"role": "assistant", "content": str(b)})
elif isinstance(item, dict):
messages.append(item)
messages.append({"role": "user", "content": str(message)})
prompt_text = tok.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True
)
inputs = tok([prompt_text], return_tensors="pt").to(model.device)
streamer = TextIteratorStreamer(tok, skip_prompt=True, skip_special_tokens=True)
generation_kwargs = dict(
inputs,
streamer=streamer,
max_new_tokens=max(64, min(16384, int(max_tokens))),
temperature=float(temperature) if float(temperature) > 0 else 0.01,
do_sample=float(temperature) > 0.0,
)
thread = Thread(target=model.generate, kwargs=generation_kwargs)
thread.start()
partial_text = ""
for new_text in streamer:
partial_text += new_text
yield partial_text
return _inner_chat_gen()
def generate_image(prompt, width, height, num_inference_steps, gpu_duration=30):
"""
Dynamic ZeroGPU SDXL image generation.
"""
req_duration = max(10, min(60, int(gpu_duration)))
@spaces.GPU(duration=req_duration)
def _inner_image_gen():
pipe = get_image_pipe()
image = pipe(
prompt=str(prompt),
width=int(width),
height=int(height),
num_inference_steps=int(num_inference_steps),
).images[0]
return image
return _inner_image_gen()
# ============================================================
# GRADIO UI & API ENDPOINTS
# ============================================================
with gr.Blocks(title="Nova AI Backend — Qwen3.8-27B + SDXL (ZeroGPU)") as demo:
gr.Markdown(
"""
# ⚡ Nova AI Compute Backend (ZeroGPU)
Exposes high-performance ZeroGPU endpoints for Nova AI Dashboard:
- **LLM Chat**: `Qwen/Qwen3.8-27B` (4-bit Quantized, up to 16,384 tokens)
- **Image Generation**: `SDXL 1.0`
"""
)
with gr.Tab("Chat Inference API"):
msg_in = gr.Textbox(label="User Message")
hist_in = gr.JSON(value=[], label="History JSON")
sys_in = gr.Textbox(
value="You are Nova, a fast, intelligent AI Assistant.",
label="System Prompt",
)
temp_in = gr.Slider(minimum=0.1, maximum=1.5, value=0.7, label="Temperature")
max_in = gr.Slider(minimum=512, maximum=16384, value=4096, step=512, label="Max Tokens")
gpu_dur_in = gr.Slider(minimum=10, maximum=120, value=60, step=5, label="GPU Duration (Sec)")
txt_out = gr.Textbox(label="Assistant Response")
chat_btn = gr.Button("Run Chat Inference")
chat_btn.click(
chat_inference,
inputs=[msg_in, hist_in, sys_in, temp_in, max_in, gpu_dur_in],
outputs=[txt_out],
api_name="chat",
)
with gr.Tab("Image Generation API"):
img_prompt = gr.Textbox(
label="Prompt",
value="A futuristic cyberpunk city skyline with neon lights, cinematic lighting",
)
img_width = gr.Slider(
minimum=512, maximum=1024, value=1024, step=64, label="Width"
)
img_height = gr.Slider(
minimum=512, maximum=1024, value=1024, step=64, label="Height"
)
img_steps = gr.Slider(
minimum=10, maximum=50, value=25, step=5, label="Inference Steps"
)
img_gpu_dur_in = gr.Slider(minimum=10, maximum=60, value=30, step=5, label="GPU Duration (Sec)")
img_out = gr.Image(label="Generated Output")
gen_btn = gr.Button("Generate Image")
gen_btn.click(
generate_image,
inputs=[img_prompt, img_width, img_height, img_steps, img_gpu_dur_in],
outputs=[img_out],
api_name="generate_image",
)
demo.queue()
if __name__ == "__main__":
demo.launch()