import gradio as gr import torch import spaces from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig from peft import PeftModel import os # --- Configuration --- BASE_MODEL_ID = "Qwen/Qwen3-4B" ADAPTER_MODEL_ID = "afnotes/Qwen3-4B-YeI-LoRA" # --- Model Loading (Optimized for ZeroGPU) --- @spaces.GPU def load_model(): print(f"Loading tokenizer from: {ADAPTER_MODEL_ID}") # Load tokenizer without triggering CUDA tokenizer = AutoTokenizer.from_pretrained(ADAPTER_MODEL_ID, use_fast=False) if tokenizer.eos_token is None: tokenizer.eos_token = "<|endoftext|>" print(f"Setting tokenizer.eos_token to: {tokenizer.eos_token}") if tokenizer.pad_token is None: tokenizer.pad_token = tokenizer.eos_token print(f"Setting tokenizer.pad_token to: {tokenizer.pad_token}") print(f"Loading base model: {BASE_MODEL_ID}") # Check for CUDA availability inside the @spaces.GPU context device_to_load = "cuda" if torch.cuda.is_available() else "cpu" print(f"Using device: {device_to_load}") # Quantization configuration is no longer needed for 16-bit bnb_config = None try: # Load base model in float16 if on CUDA, otherwise float32 on CPU base_model = AutoModelForCausalLM.from_pretrained( BASE_MODEL_ID, # Use torch.float16 for CUDA, torch.float32 for CPU torch_dtype=torch.float16 if device_to_load == "cuda" else torch.float32, device_map="auto" if device_to_load == "cuda" else "cpu", trust_remote_code=True ) print(f"Loading LoRA adapter from {ADAPTER_MODEL_ID} on top of base model...") model = PeftModel.from_pretrained(base_model, ADAPTER_MODEL_ID) model = model.merge_and_unload() print(f"Successfully loaded LoRA adapter from {ADAPTER_MODEL_ID} on top of {BASE_MODEL_ID}") except Exception as e: print(f"Error loading LoRA adapter with 16-bit/GPU setup ({e}). Attempting to load without specific device_map or in full precision on CPU...") try: model = AutoModelForCausalLM.from_pretrained( ADAPTER_MODEL_ID, torch_dtype=torch.float32, device_map="cpu", trust_remote_code=True ) print(f"Successfully loaded full merged model directly from {ADAPTER_MODEL_ID} on CPU.") except Exception as e_full: print(f"Failed to load model in fallback: {e_full}") raise RuntimeError("Model loading failed completely.") model.eval() if device_to_load == "cuda": model = model.to("cuda") print(f"Model loaded and moved to CUDA device: {model.device}") print(f"CUDA memory allocated: {torch.cuda.memory_allocated() / (1024**3):.2f} GB") print(f"CUDA memory cached: {torch.cuda.memory_reserved() / (1024**3):.2f} GB") else: print(f"Model loaded and running on CPU device: {model.device}") print("Model loaded successfully and set to evaluation mode.") return tokenizer, model # --- Generation Function for Gradio --- @spaces.GPU def generate_kanye_tweet(prompt_text, max_new_tokens, temperature, top_p, repetition_penalty): print(f"--- Generating for prompt ---\n'{prompt_text}'") # Load model and tokenizer within the @spaces.GPU context tokenizer, model = load_model() inputs = tokenizer(prompt_text, return_tensors="pt").to(model.device) input_token_len = inputs.input_ids.shape[1] with torch.no_grad(): outputs = model.generate( **inputs, max_new_tokens=max_new_tokens, temperature=temperature, top_p=top_p, do_sample=True, repetition_penalty=repetition_penalty, pad_token_id=tokenizer.eos_token_id, eos_token_id=tokenizer.eos_token_id, ) full_generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True) if full_generated_text.startswith(prompt_text): completion_text = full_generated_text[len(prompt_text):].strip() else: print("Warning: Decoded text did not start exactly with the prompt. Showing full output.") completion_text = full_generated_text print(f"\n--- Generated Completion (Ye-style) ---") print(completion_text) print("------------------------------------------") # Clear model from memory to comply with ZeroGPU del model del tokenizer if torch.cuda.is_available(): torch.cuda.empty_cache() return completion_text # --- Gradio Interface --- iface = gr.Interface( fn=generate_kanye_tweet, inputs=[ gr.Textbox(lines=2, label="Enter your prompt for Ye-style tweet:", placeholder="My next album will be called..."), gr.Slider(minimum=10, maximum=250, value=100, step=5, label="Max New Tokens"), gr.Slider(minimum=0.1, maximum=1.0, value=0.9, step=0.05, label="Temperature"), gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top P"), gr.Slider(minimum=1.0, maximum=2.0, value=1.0, step=0.05, label="Repetition Penalty") ], outputs=gr.Textbox(label="Generated Ye Tweet"), title="🎙️ Ye-I: Ye Tweet Generator (4B Model) 🎤", description="Generate Ye-style tweet snippets using a fine-tuned Qwen3-4B model. Optimized for ZeroGPU.", examples=[ ["I HATE", 50, 0.9, 0.95, 1.0], ["My next album will be called", 50, 0.9, 0.95, 1.0], ["Fashion is dead without", 50, 0.9, 0.95, 1.0], ], allow_flagging="never" ) if __name__ == "__main__": # Disable SSR mode to avoid potential main process issues iface.launch(ssr_mode=False)