afnotes
Removed 'West'
669a731
Raw History Blame Contribute Delete
5.72 kB
import gradio as gr
import torch
import spaces
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
from peft import PeftModel
import os
# --- Configuration ---
BASE_MODEL_ID = "Qwen/Qwen3-4B"
ADAPTER_MODEL_ID = "afnotes/Qwen3-4B-YeI-LoRA"
# --- Model Loading (Optimized for ZeroGPU) ---
@spaces.GPU
def load_model():
print(f"Loading tokenizer from: {ADAPTER_MODEL_ID}")
# Load tokenizer without triggering CUDA
tokenizer = AutoTokenizer.from_pretrained(ADAPTER_MODEL_ID, use_fast=False)
if tokenizer.eos_token is None:
tokenizer.eos_token = "<|endoftext|>"
print(f"Setting tokenizer.eos_token to: {tokenizer.eos_token}")
if tokenizer.pad_token is None:
tokenizer.pad_token = tokenizer.eos_token
print(f"Setting tokenizer.pad_token to: {tokenizer.pad_token}")
print(f"Loading base model: {BASE_MODEL_ID}")
# Check for CUDA availability inside the @spaces.GPU context
device_to_load = "cuda" if torch.cuda.is_available() else "cpu"
print(f"Using device: {device_to_load}")
# Quantization configuration is no longer needed for 16-bit
bnb_config = None
try:
# Load base model in float16 if on CUDA, otherwise float32 on CPU
base_model = AutoModelForCausalLM.from_pretrained(
BASE_MODEL_ID,
# Use torch.float16 for CUDA, torch.float32 for CPU
torch_dtype=torch.float16 if device_to_load == "cuda" else torch.float32,
device_map="auto" if device_to_load == "cuda" else "cpu",
trust_remote_code=True
)
print(f"Loading LoRA adapter from {ADAPTER_MODEL_ID} on top of base model...")
model = PeftModel.from_pretrained(base_model, ADAPTER_MODEL_ID)
model = model.merge_and_unload()
print(f"Successfully loaded LoRA adapter from {ADAPTER_MODEL_ID} on top of {BASE_MODEL_ID}")
except Exception as e:
print(f"Error loading LoRA adapter with 16-bit/GPU setup ({e}). Attempting to load without specific device_map or in full precision on CPU...")
try:
model = AutoModelForCausalLM.from_pretrained(
ADAPTER_MODEL_ID,
torch_dtype=torch.float32,
device_map="cpu",
trust_remote_code=True
)
print(f"Successfully loaded full merged model directly from {ADAPTER_MODEL_ID} on CPU.")
except Exception as e_full:
print(f"Failed to load model in fallback: {e_full}")
raise RuntimeError("Model loading failed completely.")
model.eval()
if device_to_load == "cuda":
model = model.to("cuda")
print(f"Model loaded and moved to CUDA device: {model.device}")
print(f"CUDA memory allocated: {torch.cuda.memory_allocated() / (1024**3):.2f} GB")
print(f"CUDA memory cached: {torch.cuda.memory_reserved() / (1024**3):.2f} GB")
else:
print(f"Model loaded and running on CPU device: {model.device}")
print("Model loaded successfully and set to evaluation mode.")
return tokenizer, model
# --- Generation Function for Gradio ---
@spaces.GPU
def generate_kanye_tweet(prompt_text, max_new_tokens, temperature, top_p, repetition_penalty):
print(f"--- Generating for prompt ---\n'{prompt_text}'")
# Load model and tokenizer within the @spaces.GPU context
tokenizer, model = load_model()
inputs = tokenizer(prompt_text, return_tensors="pt").to(model.device)
input_token_len = inputs.input_ids.shape[1]
with torch.no_grad():
outputs = model.generate(
**inputs,
max_new_tokens=max_new_tokens,
temperature=temperature,
top_p=top_p,
do_sample=True,
repetition_penalty=repetition_penalty,
pad_token_id=tokenizer.eos_token_id,
eos_token_id=tokenizer.eos_token_id,
)
full_generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
if full_generated_text.startswith(prompt_text):
completion_text = full_generated_text[len(prompt_text):].strip()
else:
print("Warning: Decoded text did not start exactly with the prompt. Showing full output.")
completion_text = full_generated_text
print(f"\n--- Generated Completion (Ye-style) ---")
print(completion_text)
print("------------------------------------------")
# Clear model from memory to comply with ZeroGPU
del model
del tokenizer
if torch.cuda.is_available():
torch.cuda.empty_cache()
return completion_text
# --- Gradio Interface ---
iface = gr.Interface(
fn=generate_kanye_tweet,
inputs=[
gr.Textbox(lines=2, label="Enter your prompt for Ye-style tweet:", placeholder="My next album will be called..."),
gr.Slider(minimum=10, maximum=250, value=100, step=5, label="Max New Tokens"),
gr.Slider(minimum=0.1, maximum=1.0, value=0.9, step=0.05, label="Temperature"),
gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top P"),
gr.Slider(minimum=1.0, maximum=2.0, value=1.0, step=0.05, label="Repetition Penalty")
],
outputs=gr.Textbox(label="Generated Ye Tweet"),
title="🎙️ Ye-I: Ye Tweet Generator (4B Model) 🎤",
description="Generate Ye-style tweet snippets using a fine-tuned Qwen3-4B model. Optimized for ZeroGPU.",
examples=[
["I HATE", 50, 0.9, 0.95, 1.0],
["My next album will be called", 50, 0.9, 0.95, 1.0],
["Fashion is dead without", 50, 0.9, 0.95, 1.0],
],
allow_flagging="never"
)
if __name__ == "__main__":
# Disable SSR mode to avoid potential main process issues
iface.launch(ssr_mode=False)