afnotes commited on
Commit
c0d02bd
ยท
1 Parent(s): 23342bb

fixed stupid thing

Browse files
Files changed (1) hide show
  1. app.py +29 -26
app.py CHANGED
@@ -6,12 +6,10 @@ from peft import PeftModel
6
  import os
7
 
8
  # --- Configuration ---
9
- # The base Qwen model
10
  BASE_MODEL_ID = "Qwen/Qwen3-4B"
11
  ADAPTER_MODEL_ID = "afnotes/Qwen3-4B-YeI-LoRA"
12
 
13
- # --- Model Loading (Optimized for Space) ---
14
- # This function will be called once when the app starts
15
  @spaces.GPU
16
  def load_model():
17
  print(f"Loading tokenizer from: {ADAPTER_MODEL_ID}")
@@ -26,8 +24,11 @@ def load_model():
26
 
27
  print(f"Loading base model: {BASE_MODEL_ID}")
28
 
29
- # Quantization configuration for 4-bit loading
30
- # Only apply if we are loading on CUDA
 
 
 
31
  bnb_config = None
32
  if device_to_load == "cuda":
33
  bnb_config = BitsAndBytesConfig(
@@ -38,59 +39,53 @@ def load_model():
38
  )
39
 
40
  try:
41
- # Load base model with or without quantization based on device_to_load
42
  base_model = AutoModelForCausalLM.from_pretrained(
43
  BASE_MODEL_ID,
44
  quantization_config=bnb_config if device_to_load == "cuda" else None,
45
- torch_dtype=torch.float16 if device_to_load == "cuda" else torch.float32, # Use float16 for GPU, float32 for CPU
46
- device_map="auto" if device_to_load == "cuda" else "cpu", # Use auto for GPU, explicitly CPU otherwise
47
  trust_remote_code=True
48
  )
49
 
50
  print(f"Loading LoRA adapter from {ADAPTER_MODEL_ID} on top of base model...")
51
  model = PeftModel.from_pretrained(base_model, ADAPTER_MODEL_ID)
52
- # Merge LoRA weights for faster inference
53
  model = model.merge_and_unload()
54
  print(f"Successfully loaded LoRA adapter from {ADAPTER_MODEL_ID} on top of {BASE_MODEL_ID}")
55
 
56
  except Exception as e:
57
  print(f"Error loading LoRA adapter with quantization/GPU setup ({e}). Attempting to load without specific quantization or device_map...")
58
  try:
59
- # Fallback for base model loading if initial attempt fails (less common with small models)
60
  model = AutoModelForCausalLM.from_pretrained(
61
- ADAPTER_MODEL_ID, # Try loading the adapter model directly as a full model if it was merged and pushed
62
- torch_dtype=torch.float32, # Default to float32 for robustness if GPU issues
63
- device_map="cpu", # Force CPU
64
  trust_remote_code=True
65
  )
66
  print(f"Successfully loaded full merged model directly from {ADAPTER_MODEL_ID} on CPU.")
67
  except Exception as e_full:
68
  print(f"Failed to load model in fallback: {e_full}")
69
- print("Please ensure your model IDs are correct and accessible.")
70
- raise RuntimeError("Model loading failed completely.") # Re-raise to crash Space
71
-
72
- model.eval() # Set model to evaluation mode
73
 
74
- # Ensure model is on the correct device for generation
75
  if device_to_load == "cuda":
76
- model = model.to("cuda") # Ensure it's on CUDA if available
77
  print(f"Model loaded and moved to CUDA device: {model.device}")
78
  print(f"CUDA memory allocated: {torch.cuda.memory_allocated() / (1024**3):.2f} GB")
79
  print(f"CUDA memory cached: {torch.cuda.memory_reserved() / (1024**3):.2f} GB")
80
  else:
81
- print(f"Model loaded and is running on CPU device: {model.device}")
82
-
83
 
84
  print("Model loaded successfully and set to evaluation mode.")
85
  return tokenizer, model
86
 
87
- tokenizer, model = load_model() # Load model globally when app starts
88
-
89
  # --- Generation Function for Gradio ---
90
  def generate_kanye_tweet(prompt_text, max_new_tokens, temperature, top_p, repetition_penalty):
91
  print(f"--- Generating for prompt ---\n'{prompt_text}'")
92
 
93
- # Ensure inputs are on the same device as the model
 
 
94
  inputs = tokenizer(prompt_text, return_tensors="pt").to(model.device)
95
  input_token_len = inputs.input_ids.shape[1]
96
 
@@ -117,6 +112,13 @@ def generate_kanye_tweet(prompt_text, max_new_tokens, temperature, top_p, repeti
117
  print(f"\n--- Generated Completion (Kanye-style) ---")
118
  print(completion_text)
119
  print("------------------------------------------")
 
 
 
 
 
 
 
120
  return completion_text
121
 
122
  # --- Gradio Interface ---
@@ -131,7 +133,7 @@ iface = gr.Interface(
131
  ],
132
  outputs=gr.Textbox(label="Generated Kanye Tweet"),
133
  title="๐ŸŽ™๏ธ Ye-I: Kanye Tweet Generator (4B Model) ๐ŸŽค",
134
- description="Generate Kanye West-style tweet snippets using a fine-tuned Qwen3-4B model. This smaller model has a better chance of running on free tiers. Remember ZeroGPU limitations!",
135
  examples=[
136
  ["I HATE"],
137
  ["My next album will be called"],
@@ -141,4 +143,5 @@ iface = gr.Interface(
141
  allow_flagging="never"
142
  )
143
 
144
- iface.launch()
 
 
6
  import os
7
 
8
  # --- Configuration ---
 
9
  BASE_MODEL_ID = "Qwen/Qwen3-4B"
10
  ADAPTER_MODEL_ID = "afnotes/Qwen3-4B-YeI-LoRA"
11
 
12
+ # --- Model Loading (Optimized for ZeroGPU) ---
 
13
  @spaces.GPU
14
  def load_model():
15
  print(f"Loading tokenizer from: {ADAPTER_MODEL_ID}")
 
24
 
25
  print(f"Loading base model: {BASE_MODEL_ID}")
26
 
27
+ # Determine device (CUDA if available, else CPU)
28
+ device_to_load = "cuda" if torch.cuda.is_available() else "cpu"
29
+ print(f"Using device: {device_to_load}")
30
+
31
+ # Quantization configuration for 4-bit loading (only for CUDA)
32
  bnb_config = None
33
  if device_to_load == "cuda":
34
  bnb_config = BitsAndBytesConfig(
 
39
  )
40
 
41
  try:
42
+ # Load base model with or without quantization based on device
43
  base_model = AutoModelForCausalLM.from_pretrained(
44
  BASE_MODEL_ID,
45
  quantization_config=bnb_config if device_to_load == "cuda" else None,
46
+ torch_dtype=torch.float16 if device_to_load == "cuda" else torch.float32,
47
+ device_map="auto" if device_to_load == "cuda" else "cpu",
48
  trust_remote_code=True
49
  )
50
 
51
  print(f"Loading LoRA adapter from {ADAPTER_MODEL_ID} on top of base model...")
52
  model = PeftModel.from_pretrained(base_model, ADAPTER_MODEL_ID)
 
53
  model = model.merge_and_unload()
54
  print(f"Successfully loaded LoRA adapter from {ADAPTER_MODEL_ID} on top of {BASE_MODEL_ID}")
55
 
56
  except Exception as e:
57
  print(f"Error loading LoRA adapter with quantization/GPU setup ({e}). Attempting to load without specific quantization or device_map...")
58
  try:
 
59
  model = AutoModelForCausalLM.from_pretrained(
60
+ ADAPTER_MODEL_ID,
61
+ torch_dtype=torch.float32,
62
+ device_map="cpu",
63
  trust_remote_code=True
64
  )
65
  print(f"Successfully loaded full merged model directly from {ADAPTER_MODEL_ID} on CPU.")
66
  except Exception as e_full:
67
  print(f"Failed to load model in fallback: {e_full}")
68
+ raise RuntimeError("Model loading failed completely.")
 
 
 
69
 
70
+ model.eval()
71
  if device_to_load == "cuda":
72
+ model = model.to("cuda")
73
  print(f"Model loaded and moved to CUDA device: {model.device}")
74
  print(f"CUDA memory allocated: {torch.cuda.memory_allocated() / (1024**3):.2f} GB")
75
  print(f"CUDA memory cached: {torch.cuda.memory_reserved() / (1024**3):.2f} GB")
76
  else:
77
+ print(f"Model loaded and running on CPU device: {model.device}")
 
78
 
79
  print("Model loaded successfully and set to evaluation mode.")
80
  return tokenizer, model
81
 
 
 
82
  # --- Generation Function for Gradio ---
83
  def generate_kanye_tweet(prompt_text, max_new_tokens, temperature, top_p, repetition_penalty):
84
  print(f"--- Generating for prompt ---\n'{prompt_text}'")
85
 
86
+ # Load model and tokenizer dynamically for each request
87
+ tokenizer, model = load_model()
88
+
89
  inputs = tokenizer(prompt_text, return_tensors="pt").to(model.device)
90
  input_token_len = inputs.input_ids.shape[1]
91
 
 
112
  print(f"\n--- Generated Completion (Kanye-style) ---")
113
  print(completion_text)
114
  print("------------------------------------------")
115
+
116
+ # Clear model from memory to comply with ZeroGPU
117
+ del model
118
+ del tokenizer
119
+ if torch.cuda.is_available():
120
+ torch.cuda.empty_cache()
121
+
122
  return completion_text
123
 
124
  # --- Gradio Interface ---
 
133
  ],
134
  outputs=gr.Textbox(label="Generated Kanye Tweet"),
135
  title="๐ŸŽ™๏ธ Ye-I: Kanye Tweet Generator (4B Model) ๐ŸŽค",
136
+ description="Generate Kanye West-style tweet snippets using a fine-tuned Qwen3-4B model. Optimized for ZeroGPU.",
137
  examples=[
138
  ["I HATE"],
139
  ["My next album will be called"],
 
143
  allow_flagging="never"
144
  )
145
 
146
+ if __name__ == "__main__":
147
+ iface.launch()