import os import torch import gradio as gr import spaces from transformers import ( AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig, TextIteratorStreamer, ) from diffusers import AutoPipelineForText2Image from threading import Thread # ============================================================ # MODEL CONFIGURATION # ============================================================ LLM_MODEL_ID = "Qwen/Qwen3.8-27B" IMAGE_MODEL_ID = "stabilityai/stable-diffusion-xl-base-1.0" print(f"[BOOT] Initializing Nova AI Backend on Hugging Face ZeroGPU...") # Pre-load tokenizer on CPU (lightweight, instant) tokenizer = None try: print(f"[BOOT] Loading Tokenizer: {LLM_MODEL_ID}...") tokenizer = AutoTokenizer.from_pretrained(LLM_MODEL_ID, trust_remote_code=True) print(f"[BOOT] Tokenizer loaded successfully!") except Exception as e: print(f"[WARN] Tokenizer deferred loading: {e}") # Global references for lazy loading inside @spaces.GPU llm_model = None pipe_image = None def get_tokenizer(): global tokenizer if tokenizer is None: tokenizer = AutoTokenizer.from_pretrained(LLM_MODEL_ID, trust_remote_code=True) return tokenizer def get_llm(): global llm_model if llm_model is None: print(f"[GPU] Loading LLM Model to ZeroGPU: {LLM_MODEL_ID} (4-bit)...") bnb_config = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_quant_type="nf4", bnb_4bit_use_double_quant=True, llm_int8_enable_fp32_cpu_offload=True, ) llm_model = AutoModelForCausalLM.from_pretrained( LLM_MODEL_ID, quantization_config=bnb_config, device_map="auto", torch_dtype=torch.bfloat16, trust_remote_code=True, ) print(f"[GPU] LLM Model loaded successfully onto ZeroGPU!") return llm_model def get_image_pipe(): global pipe_image if pipe_image is None: print(f"[GPU] Loading SDXL Image Model to ZeroGPU: {IMAGE_MODEL_ID}...") pipe_image = AutoPipelineForText2Image.from_pretrained( IMAGE_MODEL_ID, torch_dtype=torch.bfloat16, use_safetensors=True, ) pipe_image.to("cuda") print(f"[GPU] Image Pipeline loaded successfully!") return pipe_image # ============================================================ # INFERENCE ENDPOINTS (ZeroGPU Dynamic Allocation) # ============================================================ def chat_inference(message, history, system_prompt, temperature, max_tokens, gpu_duration=60): """ Dynamic ZeroGPU chat inference. gpu_duration can be customized (e.g., 10s to 120s). """ req_duration = max(10, min(120, int(gpu_duration))) @spaces.GPU(duration=req_duration) def _inner_chat_gen(): tok = get_tokenizer() model = get_llm() messages = [] if system_prompt: messages.append({"role": "system", "content": system_prompt}) if history: for item in history: if isinstance(item, (list, tuple)) and len(item) == 2: u, b = item if u: messages.append({"role": "user", "content": str(u)}) if b: messages.append({"role": "assistant", "content": str(b)}) elif isinstance(item, dict): messages.append(item) messages.append({"role": "user", "content": str(message)}) prompt_text = tok.apply_chat_template( messages, tokenize=False, add_generation_prompt=True ) inputs = tok([prompt_text], return_tensors="pt").to(model.device) streamer = TextIteratorStreamer(tok, skip_prompt=True, skip_special_tokens=True) generation_kwargs = dict( inputs, streamer=streamer, max_new_tokens=max(64, min(16384, int(max_tokens))), temperature=float(temperature) if float(temperature) > 0 else 0.01, do_sample=float(temperature) > 0.0, ) thread = Thread(target=model.generate, kwargs=generation_kwargs) thread.start() partial_text = "" for new_text in streamer: partial_text += new_text yield partial_text return _inner_chat_gen() def generate_image(prompt, width, height, num_inference_steps, gpu_duration=30): """ Dynamic ZeroGPU SDXL image generation. """ req_duration = max(10, min(60, int(gpu_duration))) @spaces.GPU(duration=req_duration) def _inner_image_gen(): pipe = get_image_pipe() image = pipe( prompt=str(prompt), width=int(width), height=int(height), num_inference_steps=int(num_inference_steps), ).images[0] return image return _inner_image_gen() # ============================================================ # GRADIO UI & API ENDPOINTS # ============================================================ with gr.Blocks(title="Nova AI Backend — Qwen3.8-27B + SDXL (ZeroGPU)") as demo: gr.Markdown( """ # ⚡ Nova AI Compute Backend (ZeroGPU) Exposes high-performance ZeroGPU endpoints for Nova AI Dashboard: - **LLM Chat**: `Qwen/Qwen3.8-27B` (4-bit Quantized, up to 16,384 tokens) - **Image Generation**: `SDXL 1.0` """ ) with gr.Tab("Chat Inference API"): msg_in = gr.Textbox(label="User Message") hist_in = gr.JSON(value=[], label="History JSON") sys_in = gr.Textbox( value="You are Nova, a fast, intelligent AI Assistant.", label="System Prompt", ) temp_in = gr.Slider(minimum=0.1, maximum=1.5, value=0.7, label="Temperature") max_in = gr.Slider(minimum=512, maximum=16384, value=4096, step=512, label="Max Tokens") gpu_dur_in = gr.Slider(minimum=10, maximum=120, value=60, step=5, label="GPU Duration (Sec)") txt_out = gr.Textbox(label="Assistant Response") chat_btn = gr.Button("Run Chat Inference") chat_btn.click( chat_inference, inputs=[msg_in, hist_in, sys_in, temp_in, max_in, gpu_dur_in], outputs=[txt_out], api_name="chat", ) with gr.Tab("Image Generation API"): img_prompt = gr.Textbox( label="Prompt", value="A futuristic cyberpunk city skyline with neon lights, cinematic lighting", ) img_width = gr.Slider( minimum=512, maximum=1024, value=1024, step=64, label="Width" ) img_height = gr.Slider( minimum=512, maximum=1024, value=1024, step=64, label="Height" ) img_steps = gr.Slider( minimum=10, maximum=50, value=25, step=5, label="Inference Steps" ) img_gpu_dur_in = gr.Slider(minimum=10, maximum=60, value=30, step=5, label="GPU Duration (Sec)") img_out = gr.Image(label="Generated Output") gen_btn = gr.Button("Generate Image") gen_btn.click( generate_image, inputs=[img_prompt, img_width, img_height, img_steps, img_gpu_dur_in], outputs=[img_out], api_name="generate_image", ) demo.queue() if __name__ == "__main__": demo.launch()