Spaces:
Sleeping
Sleeping
File size: 5,718 Bytes
3caefac e93f31c ebb8aa3 3caefac fe2af5a e93f31c fe2af5a c0d02bd 23342bb 3caefac 2569a42 fe2af5a 3caefac fe2af5a 3caefac fe2af5a 3caefac 8170a49 2569a42 c0d02bd 8416e9b fe2af5a 8416e9b fe2af5a 8416e9b c0d02bd fe2af5a 8416e9b fe2af5a c0d02bd fe2af5a c0d02bd fe2af5a c0d02bd fe2af5a c0d02bd fe2af5a 8170a49 c0d02bd 8170a49 fe2af5a 8170a49 3caefac 2569a42 fe2af5a 3caefac 2569a42 c0d02bd 3caefac 3869c1d 3caefac c0d02bd 3caefac fe2af5a 3caefac 3869c1d 1d5d111 3caefac 3869c1d 669a731 3caefac 1d5d111 3caefac fe2af5a 3caefac c0d02bd 2569a42 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 | import gradio as gr
import torch
import spaces
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
from peft import PeftModel
import os
# --- Configuration ---
BASE_MODEL_ID = "Qwen/Qwen3-4B"
ADAPTER_MODEL_ID = "afnotes/Qwen3-4B-YeI-LoRA"
# --- Model Loading (Optimized for ZeroGPU) ---
@spaces.GPU
def load_model():
print(f"Loading tokenizer from: {ADAPTER_MODEL_ID}")
# Load tokenizer without triggering CUDA
tokenizer = AutoTokenizer.from_pretrained(ADAPTER_MODEL_ID, use_fast=False)
if tokenizer.eos_token is None:
tokenizer.eos_token = "<|endoftext|>"
print(f"Setting tokenizer.eos_token to: {tokenizer.eos_token}")
if tokenizer.pad_token is None:
tokenizer.pad_token = tokenizer.eos_token
print(f"Setting tokenizer.pad_token to: {tokenizer.pad_token}")
print(f"Loading base model: {BASE_MODEL_ID}")
# Check for CUDA availability inside the @spaces.GPU context
device_to_load = "cuda" if torch.cuda.is_available() else "cpu"
print(f"Using device: {device_to_load}")
# Quantization configuration is no longer needed for 16-bit
bnb_config = None
try:
# Load base model in float16 if on CUDA, otherwise float32 on CPU
base_model = AutoModelForCausalLM.from_pretrained(
BASE_MODEL_ID,
# Use torch.float16 for CUDA, torch.float32 for CPU
torch_dtype=torch.float16 if device_to_load == "cuda" else torch.float32,
device_map="auto" if device_to_load == "cuda" else "cpu",
trust_remote_code=True
)
print(f"Loading LoRA adapter from {ADAPTER_MODEL_ID} on top of base model...")
model = PeftModel.from_pretrained(base_model, ADAPTER_MODEL_ID)
model = model.merge_and_unload()
print(f"Successfully loaded LoRA adapter from {ADAPTER_MODEL_ID} on top of {BASE_MODEL_ID}")
except Exception as e:
print(f"Error loading LoRA adapter with 16-bit/GPU setup ({e}). Attempting to load without specific device_map or in full precision on CPU...")
try:
model = AutoModelForCausalLM.from_pretrained(
ADAPTER_MODEL_ID,
torch_dtype=torch.float32,
device_map="cpu",
trust_remote_code=True
)
print(f"Successfully loaded full merged model directly from {ADAPTER_MODEL_ID} on CPU.")
except Exception as e_full:
print(f"Failed to load model in fallback: {e_full}")
raise RuntimeError("Model loading failed completely.")
model.eval()
if device_to_load == "cuda":
model = model.to("cuda")
print(f"Model loaded and moved to CUDA device: {model.device}")
print(f"CUDA memory allocated: {torch.cuda.memory_allocated() / (1024**3):.2f} GB")
print(f"CUDA memory cached: {torch.cuda.memory_reserved() / (1024**3):.2f} GB")
else:
print(f"Model loaded and running on CPU device: {model.device}")
print("Model loaded successfully and set to evaluation mode.")
return tokenizer, model
# --- Generation Function for Gradio ---
@spaces.GPU
def generate_kanye_tweet(prompt_text, max_new_tokens, temperature, top_p, repetition_penalty):
print(f"--- Generating for prompt ---\n'{prompt_text}'")
# Load model and tokenizer within the @spaces.GPU context
tokenizer, model = load_model()
inputs = tokenizer(prompt_text, return_tensors="pt").to(model.device)
input_token_len = inputs.input_ids.shape[1]
with torch.no_grad():
outputs = model.generate(
**inputs,
max_new_tokens=max_new_tokens,
temperature=temperature,
top_p=top_p,
do_sample=True,
repetition_penalty=repetition_penalty,
pad_token_id=tokenizer.eos_token_id,
eos_token_id=tokenizer.eos_token_id,
)
full_generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
if full_generated_text.startswith(prompt_text):
completion_text = full_generated_text[len(prompt_text):].strip()
else:
print("Warning: Decoded text did not start exactly with the prompt. Showing full output.")
completion_text = full_generated_text
print(f"\n--- Generated Completion (Ye-style) ---")
print(completion_text)
print("------------------------------------------")
# Clear model from memory to comply with ZeroGPU
del model
del tokenizer
if torch.cuda.is_available():
torch.cuda.empty_cache()
return completion_text
# --- Gradio Interface ---
iface = gr.Interface(
fn=generate_kanye_tweet,
inputs=[
gr.Textbox(lines=2, label="Enter your prompt for Ye-style tweet:", placeholder="My next album will be called..."),
gr.Slider(minimum=10, maximum=250, value=100, step=5, label="Max New Tokens"),
gr.Slider(minimum=0.1, maximum=1.0, value=0.9, step=0.05, label="Temperature"),
gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top P"),
gr.Slider(minimum=1.0, maximum=2.0, value=1.0, step=0.05, label="Repetition Penalty")
],
outputs=gr.Textbox(label="Generated Ye Tweet"),
title="🎙️ Ye-I: Ye Tweet Generator (4B Model) 🎤",
description="Generate Ye-style tweet snippets using a fine-tuned Qwen3-4B model. Optimized for ZeroGPU.",
examples=[
["I HATE", 50, 0.9, 0.95, 1.0],
["My next album will be called", 50, 0.9, 0.95, 1.0],
["Fashion is dead without", 50, 0.9, 0.95, 1.0],
],
allow_flagging="never"
)
if __name__ == "__main__":
# Disable SSR mode to avoid potential main process issues
iface.launch(ssr_mode=False)
|