File size: 5,718 Bytes
3caefac
 
e93f31c
ebb8aa3
3caefac
 
 
fe2af5a
e93f31c
 
fe2af5a
c0d02bd
23342bb
3caefac
 
2569a42
 
fe2af5a
3caefac
 
fe2af5a
3caefac
 
fe2af5a
3caefac
 
8170a49
2569a42
c0d02bd
 
 
8416e9b
fe2af5a
 
 
8416e9b
fe2af5a
 
8416e9b
c0d02bd
 
fe2af5a
 
 
 
 
 
 
 
 
8416e9b
fe2af5a
 
c0d02bd
 
 
fe2af5a
 
 
 
 
c0d02bd
fe2af5a
c0d02bd
fe2af5a
c0d02bd
fe2af5a
8170a49
 
 
c0d02bd
8170a49
fe2af5a
8170a49
 
3caefac
2569a42
fe2af5a
3caefac
 
2569a42
c0d02bd
 
3caefac
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3869c1d
3caefac
 
c0d02bd
 
 
 
 
 
 
3caefac
 
 
 
fe2af5a
3caefac
3869c1d
1d5d111
3caefac
 
 
 
3869c1d
 
669a731
3caefac
1d5d111
 
 
3caefac
fe2af5a
3caefac
 
c0d02bd
2569a42
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
import gradio as gr
import torch
import spaces
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
from peft import PeftModel
import os

# --- Configuration ---
BASE_MODEL_ID = "Qwen/Qwen3-4B"
ADAPTER_MODEL_ID = "afnotes/Qwen3-4B-YeI-LoRA"

# --- Model Loading (Optimized for ZeroGPU) ---
@spaces.GPU
def load_model():
    print(f"Loading tokenizer from: {ADAPTER_MODEL_ID}")
    # Load tokenizer without triggering CUDA
    tokenizer = AutoTokenizer.from_pretrained(ADAPTER_MODEL_ID, use_fast=False)

    if tokenizer.eos_token is None:
        tokenizer.eos_token = "<|endoftext|>"
        print(f"Setting tokenizer.eos_token to: {tokenizer.eos_token}")
    if tokenizer.pad_token is None:
        tokenizer.pad_token = tokenizer.eos_token
        print(f"Setting tokenizer.pad_token to: {tokenizer.pad_token}")

    print(f"Loading base model: {BASE_MODEL_ID}")

    # Check for CUDA availability inside the @spaces.GPU context
    device_to_load = "cuda" if torch.cuda.is_available() else "cpu"
    print(f"Using device: {device_to_load}")

    # Quantization configuration is no longer needed for 16-bit
    bnb_config = None

    try:
        # Load base model in float16 if on CUDA, otherwise float32 on CPU
        base_model = AutoModelForCausalLM.from_pretrained(
            BASE_MODEL_ID,
            # Use torch.float16 for CUDA, torch.float32 for CPU
            torch_dtype=torch.float16 if device_to_load == "cuda" else torch.float32,
            device_map="auto" if device_to_load == "cuda" else "cpu",
            trust_remote_code=True
        )

        print(f"Loading LoRA adapter from {ADAPTER_MODEL_ID} on top of base model...")
        model = PeftModel.from_pretrained(base_model, ADAPTER_MODEL_ID)
        model = model.merge_and_unload()
        print(f"Successfully loaded LoRA adapter from {ADAPTER_MODEL_ID} on top of {BASE_MODEL_ID}")

    except Exception as e:
        print(f"Error loading LoRA adapter with 16-bit/GPU setup ({e}). Attempting to load without specific device_map or in full precision on CPU...")
        try:
            model = AutoModelForCausalLM.from_pretrained(
                ADAPTER_MODEL_ID,
                torch_dtype=torch.float32,
                device_map="cpu",
                trust_remote_code=True
            )
            print(f"Successfully loaded full merged model directly from {ADAPTER_MODEL_ID} on CPU.")
        except Exception as e_full:
            print(f"Failed to load model in fallback: {e_full}")
            raise RuntimeError("Model loading failed completely.")

    model.eval()
    if device_to_load == "cuda":
        model = model.to("cuda")
        print(f"Model loaded and moved to CUDA device: {model.device}")
        print(f"CUDA memory allocated: {torch.cuda.memory_allocated() / (1024**3):.2f} GB")
        print(f"CUDA memory cached: {torch.cuda.memory_reserved() / (1024**3):.2f} GB")
    else:
        print(f"Model loaded and running on CPU device: {model.device}")

    print("Model loaded successfully and set to evaluation mode.")
    return tokenizer, model

# --- Generation Function for Gradio ---
@spaces.GPU
def generate_kanye_tweet(prompt_text, max_new_tokens, temperature, top_p, repetition_penalty):
    print(f"--- Generating for prompt ---\n'{prompt_text}'")

    # Load model and tokenizer within the @spaces.GPU context
    tokenizer, model = load_model()

    inputs = tokenizer(prompt_text, return_tensors="pt").to(model.device)
    input_token_len = inputs.input_ids.shape[1]

    with torch.no_grad():
        outputs = model.generate(
            **inputs,
            max_new_tokens=max_new_tokens,
            temperature=temperature,
            top_p=top_p,
            do_sample=True,
            repetition_penalty=repetition_penalty,
            pad_token_id=tokenizer.eos_token_id,
            eos_token_id=tokenizer.eos_token_id,
        )

    full_generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)

    if full_generated_text.startswith(prompt_text):
        completion_text = full_generated_text[len(prompt_text):].strip()
    else:
        print("Warning: Decoded text did not start exactly with the prompt. Showing full output.")
        completion_text = full_generated_text

    print(f"\n--- Generated Completion (Ye-style) ---")
    print(completion_text)
    print("------------------------------------------")

    # Clear model from memory to comply with ZeroGPU
    del model
    del tokenizer
    if torch.cuda.is_available():
        torch.cuda.empty_cache()

    return completion_text

# --- Gradio Interface ---
iface = gr.Interface(
    fn=generate_kanye_tweet,
    inputs=[
        gr.Textbox(lines=2, label="Enter your prompt for Ye-style tweet:", placeholder="My next album will be called..."),
        gr.Slider(minimum=10, maximum=250, value=100, step=5, label="Max New Tokens"),
        gr.Slider(minimum=0.1, maximum=1.0, value=0.9, step=0.05, label="Temperature"),
        gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top P"),
        gr.Slider(minimum=1.0, maximum=2.0, value=1.0, step=0.05, label="Repetition Penalty")
    ],
    outputs=gr.Textbox(label="Generated Ye Tweet"),
    title="🎙️ Ye-I: Ye Tweet Generator (4B Model) 🎤",
    description="Generate Ye-style tweet snippets using a fine-tuned Qwen3-4B model. Optimized for ZeroGPU.",
    examples=[
        ["I HATE", 50, 0.9, 0.95, 1.0],
        ["My next album will be called", 50, 0.9, 0.95, 1.0],
        ["Fashion is dead without", 50, 0.9, 0.95, 1.0],
    ],
    allow_flagging="never"
)

if __name__ == "__main__":
    # Disable SSR mode to avoid potential main process issues
    iface.launch(ssr_mode=False)