Spaces:
Runtime error
Runtime error
Download app.py from AADIplayz23/myspace: direct link, hf CLI and curl.
- Browser
- Download file 7.35 kB
-
https://huggingface.co/spaces/AADIplayz23/myspace/resolve/main/app.py
- Command line
-
hf download hf://spaces/AADIplayz23/myspace/app.py
-
curl -L -o app.py https://huggingface.co/spaces/AADIplayz23/myspace/resolve/main/app.py
7.35 kB
| import os | |
| import torch | |
| import gradio as gr | |
| import spaces | |
| from transformers import ( | |
| AutoTokenizer, | |
| AutoModelForCausalLM, | |
| BitsAndBytesConfig, | |
| TextIteratorStreamer, | |
| ) | |
| from diffusers import AutoPipelineForText2Image | |
| from threading import Thread | |
| # ============================================================ | |
| # MODEL CONFIGURATION | |
| # ============================================================ | |
| LLM_MODEL_ID = "Qwen/Qwen3.8-27B" | |
| IMAGE_MODEL_ID = "stabilityai/stable-diffusion-xl-base-1.0" | |
| print(f"[BOOT] Initializing Nova AI Backend on Hugging Face ZeroGPU...") | |
| # Pre-load tokenizer on CPU (lightweight, instant) | |
| tokenizer = None | |
| try: | |
| print(f"[BOOT] Loading Tokenizer: {LLM_MODEL_ID}...") | |
| tokenizer = AutoTokenizer.from_pretrained(LLM_MODEL_ID, trust_remote_code=True) | |
| print(f"[BOOT] Tokenizer loaded successfully!") | |
| except Exception as e: | |
| print(f"[WARN] Tokenizer deferred loading: {e}") | |
| # Global references for lazy loading inside @spaces.GPU | |
| llm_model = None | |
| pipe_image = None | |
| def get_tokenizer(): | |
| global tokenizer | |
| if tokenizer is None: | |
| tokenizer = AutoTokenizer.from_pretrained(LLM_MODEL_ID, trust_remote_code=True) | |
| return tokenizer | |
| def get_llm(): | |
| global llm_model | |
| if llm_model is None: | |
| print(f"[GPU] Loading LLM Model to ZeroGPU: {LLM_MODEL_ID} (4-bit)...") | |
| bnb_config = BitsAndBytesConfig( | |
| load_in_4bit=True, | |
| bnb_4bit_compute_dtype=torch.bfloat16, | |
| bnb_4bit_quant_type="nf4", | |
| bnb_4bit_use_double_quant=True, | |
| llm_int8_enable_fp32_cpu_offload=True, | |
| ) | |
| llm_model = AutoModelForCausalLM.from_pretrained( | |
| LLM_MODEL_ID, | |
| quantization_config=bnb_config, | |
| device_map="auto", | |
| torch_dtype=torch.bfloat16, | |
| trust_remote_code=True, | |
| ) | |
| print(f"[GPU] LLM Model loaded successfully onto ZeroGPU!") | |
| return llm_model | |
| def get_image_pipe(): | |
| global pipe_image | |
| if pipe_image is None: | |
| print(f"[GPU] Loading SDXL Image Model to ZeroGPU: {IMAGE_MODEL_ID}...") | |
| pipe_image = AutoPipelineForText2Image.from_pretrained( | |
| IMAGE_MODEL_ID, | |
| torch_dtype=torch.bfloat16, | |
| use_safetensors=True, | |
| ) | |
| pipe_image.to("cuda") | |
| print(f"[GPU] Image Pipeline loaded successfully!") | |
| return pipe_image | |
| # ============================================================ | |
| # INFERENCE ENDPOINTS (ZeroGPU Dynamic Allocation) | |
| # ============================================================ | |
| def chat_inference(message, history, system_prompt, temperature, max_tokens, gpu_duration=60): | |
| """ | |
| Dynamic ZeroGPU chat inference. | |
| gpu_duration can be customized (e.g., 10s to 120s). | |
| """ | |
| req_duration = max(10, min(120, int(gpu_duration))) | |
| def _inner_chat_gen(): | |
| tok = get_tokenizer() | |
| model = get_llm() | |
| messages = [] | |
| if system_prompt: | |
| messages.append({"role": "system", "content": system_prompt}) | |
| if history: | |
| for item in history: | |
| if isinstance(item, (list, tuple)) and len(item) == 2: | |
| u, b = item | |
| if u: messages.append({"role": "user", "content": str(u)}) | |
| if b: messages.append({"role": "assistant", "content": str(b)}) | |
| elif isinstance(item, dict): | |
| messages.append(item) | |
| messages.append({"role": "user", "content": str(message)}) | |
| prompt_text = tok.apply_chat_template( | |
| messages, tokenize=False, add_generation_prompt=True | |
| ) | |
| inputs = tok([prompt_text], return_tensors="pt").to(model.device) | |
| streamer = TextIteratorStreamer(tok, skip_prompt=True, skip_special_tokens=True) | |
| generation_kwargs = dict( | |
| inputs, | |
| streamer=streamer, | |
| max_new_tokens=max(64, min(16384, int(max_tokens))), | |
| temperature=float(temperature) if float(temperature) > 0 else 0.01, | |
| do_sample=float(temperature) > 0.0, | |
| ) | |
| thread = Thread(target=model.generate, kwargs=generation_kwargs) | |
| thread.start() | |
| partial_text = "" | |
| for new_text in streamer: | |
| partial_text += new_text | |
| yield partial_text | |
| return _inner_chat_gen() | |
| def generate_image(prompt, width, height, num_inference_steps, gpu_duration=30): | |
| """ | |
| Dynamic ZeroGPU SDXL image generation. | |
| """ | |
| req_duration = max(10, min(60, int(gpu_duration))) | |
| def _inner_image_gen(): | |
| pipe = get_image_pipe() | |
| image = pipe( | |
| prompt=str(prompt), | |
| width=int(width), | |
| height=int(height), | |
| num_inference_steps=int(num_inference_steps), | |
| ).images[0] | |
| return image | |
| return _inner_image_gen() | |
| # ============================================================ | |
| # GRADIO UI & API ENDPOINTS | |
| # ============================================================ | |
| with gr.Blocks(title="Nova AI Backend — Qwen3.8-27B + SDXL (ZeroGPU)") as demo: | |
| gr.Markdown( | |
| """ | |
| # ⚡ Nova AI Compute Backend (ZeroGPU) | |
| Exposes high-performance ZeroGPU endpoints for Nova AI Dashboard: | |
| - **LLM Chat**: `Qwen/Qwen3.8-27B` (4-bit Quantized, up to 16,384 tokens) | |
| - **Image Generation**: `SDXL 1.0` | |
| """ | |
| ) | |
| with gr.Tab("Chat Inference API"): | |
| msg_in = gr.Textbox(label="User Message") | |
| hist_in = gr.JSON(value=[], label="History JSON") | |
| sys_in = gr.Textbox( | |
| value="You are Nova, a fast, intelligent AI Assistant.", | |
| label="System Prompt", | |
| ) | |
| temp_in = gr.Slider(minimum=0.1, maximum=1.5, value=0.7, label="Temperature") | |
| max_in = gr.Slider(minimum=512, maximum=16384, value=4096, step=512, label="Max Tokens") | |
| gpu_dur_in = gr.Slider(minimum=10, maximum=120, value=60, step=5, label="GPU Duration (Sec)") | |
| txt_out = gr.Textbox(label="Assistant Response") | |
| chat_btn = gr.Button("Run Chat Inference") | |
| chat_btn.click( | |
| chat_inference, | |
| inputs=[msg_in, hist_in, sys_in, temp_in, max_in, gpu_dur_in], | |
| outputs=[txt_out], | |
| api_name="chat", | |
| ) | |
| with gr.Tab("Image Generation API"): | |
| img_prompt = gr.Textbox( | |
| label="Prompt", | |
| value="A futuristic cyberpunk city skyline with neon lights, cinematic lighting", | |
| ) | |
| img_width = gr.Slider( | |
| minimum=512, maximum=1024, value=1024, step=64, label="Width" | |
| ) | |
| img_height = gr.Slider( | |
| minimum=512, maximum=1024, value=1024, step=64, label="Height" | |
| ) | |
| img_steps = gr.Slider( | |
| minimum=10, maximum=50, value=25, step=5, label="Inference Steps" | |
| ) | |
| img_gpu_dur_in = gr.Slider(minimum=10, maximum=60, value=30, step=5, label="GPU Duration (Sec)") | |
| img_out = gr.Image(label="Generated Output") | |
| gen_btn = gr.Button("Generate Image") | |
| gen_btn.click( | |
| generate_image, | |
| inputs=[img_prompt, img_width, img_height, img_steps, img_gpu_dur_in], | |
| outputs=[img_out], | |
| api_name="generate_image", | |
| ) | |
| demo.queue() | |
| if __name__ == "__main__": | |
| demo.launch() | |