afnotes commited on
Commit
8170a49
·
1 Parent(s): 3caefac

Edited app.py

Browse files
Files changed (1) hide show
  1. app.py +28 -14
app.py CHANGED
@@ -1,45 +1,59 @@
1
  import gradio as gr
2
  import torch
3
- from transformers import AutoModelForCausalLM, AutoTokenizer
4
  from peft import PeftModel
5
  import os
6
 
7
  # --- Configuration ---
8
- # The base Qwen model
9
  BASE_MODEL_ID = "Qwen/Qwen3-4B"
10
- # Your uploaded adapter model ID on Hugging Face Hub
11
- ADAPTER_MODEL_ID = "afnotes/Qwen3-4B-YeI-LoRA" # IMPORTANT: Replace with your actual repo ID
12
 
13
- # --- Model Loading (Optimized for Space) ---
14
- # This function will be called once when the app starts
15
  def load_model():
16
  print(f"Loading tokenizer from: {ADAPTER_MODEL_ID}")
17
  tokenizer = AutoTokenizer.from_pretrained(ADAPTER_MODEL_ID)
18
-
19
  if tokenizer.eos_token is None:
20
  tokenizer.eos_token = "<|endoftext|>"
21
- print(f"Setting tokenizer.eos_token to: {tokenizer.eos_token}")
22
  if tokenizer.pad_token is None:
23
  tokenizer.pad_token = tokenizer.eos_token
24
- print(f"Setting tokenizer.pad_token to: {tokenizer.pad_token}")
25
 
26
  print(f"Loading base model: {BASE_MODEL_ID}")
 
 
 
 
 
 
 
 
 
 
27
  base_model = AutoModelForCausalLM.from_pretrained(
28
  BASE_MODEL_ID,
29
- torch_dtype=torch.float16, # Use float16 for memory efficiency on GPU
 
30
  device_map="auto",
31
  trust_remote_code=True
32
  )
33
 
34
  print(f"Loading LoRA adapter from {ADAPTER_MODEL_ID} on top of base model...")
35
  model = PeftModel.from_pretrained(base_model, ADAPTER_MODEL_ID)
36
- # Merge LoRA weights for faster inference
37
  model = model.merge_and_unload()
38
- model.eval() # Set model to evaluation mode
39
  print("Model loaded and LoRA merged successfully!")
40
- return tokenizer, model.to("cuda" if torch.cuda.is_available() else "cpu") # Ensure model is on GPU if available
41
 
42
- tokenizer, model = load_model() # Load model globally when app starts
 
 
 
 
 
 
 
 
 
 
 
 
43
 
44
  # --- Generation Function for Gradio ---
45
  def generate_ye_tweet(prompt_text, max_new_tokens, temperature, top_p, repetition_penalty):
 
1
  import gradio as gr
2
  import torch
3
+ from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig # Import BitsAndBytesConfig
4
  from peft import PeftModel
5
  import os
6
 
7
  # --- Configuration ---
 
8
  BASE_MODEL_ID = "Qwen/Qwen3-4B"
9
+ ADAPTER_MODEL_ID = "afnotes/Qwen3-4B-YeI-LoRA"
 
10
 
 
 
11
  def load_model():
12
  print(f"Loading tokenizer from: {ADAPTER_MODEL_ID}")
13
  tokenizer = AutoTokenizer.from_pretrained(ADAPTER_MODEL_ID)
 
14
  if tokenizer.eos_token is None:
15
  tokenizer.eos_token = "<|endoftext|>"
 
16
  if tokenizer.pad_token is None:
17
  tokenizer.pad_token = tokenizer.eos_token
 
18
 
19
  print(f"Loading base model: {BASE_MODEL_ID}")
20
+
21
+ # Define quantization configuration for 4-bit loading
22
+ # This requires bitsandbytes to be installed with GPU support
23
+ bnb_config = BitsAndBytesConfig(
24
+ load_in_4bit=True,
25
+ bnb_4bit_quant_type="nf4", # Or "fp4"
26
+ bnb_4bit_use_double_quant=True,
27
+ bnb_4bit_compute_dtype=torch.float16,
28
+ )
29
+
30
  base_model = AutoModelForCausalLM.from_pretrained(
31
  BASE_MODEL_ID,
32
+ # torch_dtype=torch.float16, # This might be too large for GPU without quantization
33
+ quantization_config=bnb_config, # Apply quantization
34
  device_map="auto",
35
  trust_remote_code=True
36
  )
37
 
38
  print(f"Loading LoRA adapter from {ADAPTER_MODEL_ID} on top of base model...")
39
  model = PeftModel.from_pretrained(base_model, ADAPTER_MODEL_ID)
 
40
  model = model.merge_and_unload()
41
+ model.eval()
42
  print("Model loaded and LoRA merged successfully!")
 
43
 
44
+ # Verify GPU usage
45
+ print(f"Model device: {model.device}")
46
+ if torch.cuda.is_available():
47
+ print(f"CUDA is available. Current CUDA device: {torch.cuda.current_device()}")
48
+ print(f"CUDA device name: {torch.cuda.get_device_name(torch.cuda.current_device())}")
49
+ print(f"CUDA memory allocated: {torch.cuda.memory_allocated() / (1024**3):.2f} GB")
50
+ print(f"CUDA memory cached: {torch.cuda.memory_reserved() / (1024**3):.2f} GB")
51
+ else:
52
+ print("CUDA is NOT available. Model might be running on CPU.")
53
+
54
+ return tokenizer, model
55
+
56
+ tokenizer, model = load_model()
57
 
58
  # --- Generation Function for Gradio ---
59
  def generate_ye_tweet(prompt_text, max_new_tokens, temperature, top_p, repetition_penalty):