afnotes commited on
Commit
3caefac
·
1 Parent(s): 037ad30

Added app.py

Browse files
Files changed (2) hide show
  1. app.py +97 -0
  2. requirements.txt +10 -0
app.py ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ import torch
3
+ from transformers import AutoModelForCausalLM, AutoTokenizer
4
+ from peft import PeftModel
5
+ import os
6
+
7
+ # --- Configuration ---
8
+ # The base Qwen model
9
+ BASE_MODEL_ID = "Qwen/Qwen3-4B"
10
+ # Your uploaded adapter model ID on Hugging Face Hub
11
+ ADAPTER_MODEL_ID = "afnotes/Qwen3-4B-YeI-LoRA" # IMPORTANT: Replace with your actual repo ID
12
+
13
+ # --- Model Loading (Optimized for Space) ---
14
+ # This function will be called once when the app starts
15
+ def load_model():
16
+ print(f"Loading tokenizer from: {ADAPTER_MODEL_ID}")
17
+ tokenizer = AutoTokenizer.from_pretrained(ADAPTER_MODEL_ID)
18
+
19
+ if tokenizer.eos_token is None:
20
+ tokenizer.eos_token = "<|endoftext|>"
21
+ print(f"Setting tokenizer.eos_token to: {tokenizer.eos_token}")
22
+ if tokenizer.pad_token is None:
23
+ tokenizer.pad_token = tokenizer.eos_token
24
+ print(f"Setting tokenizer.pad_token to: {tokenizer.pad_token}")
25
+
26
+ print(f"Loading base model: {BASE_MODEL_ID}")
27
+ base_model = AutoModelForCausalLM.from_pretrained(
28
+ BASE_MODEL_ID,
29
+ torch_dtype=torch.float16, # Use float16 for memory efficiency on GPU
30
+ device_map="auto",
31
+ trust_remote_code=True
32
+ )
33
+
34
+ print(f"Loading LoRA adapter from {ADAPTER_MODEL_ID} on top of base model...")
35
+ model = PeftModel.from_pretrained(base_model, ADAPTER_MODEL_ID)
36
+ # Merge LoRA weights for faster inference
37
+ model = model.merge_and_unload()
38
+ model.eval() # Set model to evaluation mode
39
+ print("Model loaded and LoRA merged successfully!")
40
+ return tokenizer, model.to("cuda" if torch.cuda.is_available() else "cpu") # Ensure model is on GPU if available
41
+
42
+ tokenizer, model = load_model() # Load model globally when app starts
43
+
44
+ # --- Generation Function for Gradio ---
45
+ def generate_ye_tweet(prompt_text, max_new_tokens, temperature, top_p, repetition_penalty):
46
+ print(f"--- Generating for prompt ---\n'{prompt_text}'")
47
+
48
+ inputs = tokenizer(prompt_text, return_tensors="pt").to(model.device)
49
+ input_token_len = inputs.input_ids.shape[1]
50
+
51
+ with torch.no_grad():
52
+ outputs = model.generate(
53
+ **inputs,
54
+ max_new_tokens=max_new_tokens,
55
+ temperature=temperature,
56
+ top_p=top_p,
57
+ do_sample=True,
58
+ repetition_penalty=repetition_penalty,
59
+ pad_token_id=tokenizer.eos_token_id,
60
+ eos_token_id=tokenizer.eos_token_id,
61
+ )
62
+
63
+ full_generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
64
+
65
+ if full_generated_text.startswith(prompt_text):
66
+ completion_text = full_generated_text[len(prompt_text):].strip()
67
+ else:
68
+ print("Warning: Decoded text did not start exactly with the prompt. Showing full output.")
69
+ completion_text = full_generated_text
70
+
71
+ print(f"\n--- Generated Completion (Kanye-style) ---")
72
+ print(completion_text)
73
+ print("------------------------------------------")
74
+ return completion_text
75
+
76
+ # --- Gradio Interface ---
77
+ iface = gr.Interface(
78
+ fn=generate_ye_tweet,
79
+ inputs=[
80
+ gr.Textbox(lines=2, label="Enter your prompt for Ye-style tweet:", placeholder="My next album will be called..."),
81
+ gr.Slider(minimum=10, maximum=100, value=50, step=5, label="Max New Tokens"),
82
+ gr.Slider(minimum=0.1, maximum=1.0, value=0.9, step=0.05, label="Temperature"),
83
+ gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top P"),
84
+ gr.Slider(minimum=1.0, maximum=2.0, value=1.0, step=0.05, label="Repetition Penalty")
85
+ ],
86
+ outputs=gr.Textbox(label="Generated Ye Tweet"),
87
+ title="🎙️ Ye-I: Ye Tweet Generator 🎤",
88
+ description="Generate Ye West-style tweet snippets using a fine-tuned Qwen3-4B model. Remember the ZeroGPU limitations!",
89
+ examples=[
90
+ ["I HATE"],
91
+ ["My next album will be called"],
92
+ ["Fashion is dead without"],
93
+ ],
94
+ allow_flagging="never" # You can change this to "auto" if you want to allow flagging of bad outputs
95
+ )
96
+
97
+ iface.launch()
requirements.txt ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ bitsandbytes
2
+ hf-xet
3
+ peft
4
+ requests
5
+ safetensors
6
+ tokenizers
7
+ torch
8
+ transformers
9
+ gradio
10
+ accelerate