PranjalZetsu commited on
Commit
07aacbb
Β·
verified Β·
1 Parent(s): 2470dfc

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +9 -4
app.py CHANGED
@@ -1,21 +1,26 @@
1
  import gradio as gr
2
  import torch
3
- from transformers import AutoModelForCausalLM, AutoTokenizer
4
  from peft import PeftModel
5
 
6
  # --- CONFIGURATION ---
7
- # Replace with your actual Hugging Face username and the model repo you just made
8
  ADAPTER_REPO = "PranjalZetsu/Fab_Yield_Agent_Qwen-q4"
9
- # Assuming you used Qwen 2.5 7B. Change if you used a different base!
10
  BASE_MODEL = "unsloth/Qwen2.5-7B-Instruct"
11
 
12
  print("Loading tokenizer...")
13
  tokenizer = AutoTokenizer.from_pretrained(ADAPTER_REPO)
14
 
 
 
 
 
 
 
 
15
  print("Loading base model in 4-bit to save VRAM...")
16
  base_model = AutoModelForCausalLM.from_pretrained(
17
  BASE_MODEL,
18
- load_in_4bit=True,
19
  device_map="auto",
20
  torch_dtype=torch.float16,
21
  )
 
1
  import gradio as gr
2
  import torch
3
+ from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
4
  from peft import PeftModel
5
 
6
  # --- CONFIGURATION ---
 
7
  ADAPTER_REPO = "PranjalZetsu/Fab_Yield_Agent_Qwen-q4"
 
8
  BASE_MODEL = "unsloth/Qwen2.5-7B-Instruct"
9
 
10
  print("Loading tokenizer...")
11
  tokenizer = AutoTokenizer.from_pretrained(ADAPTER_REPO)
12
 
13
+ # --- NEW: BitsAndBytes Configuration ---
14
+ print("Setting up 4-bit quantization config...")
15
+ bnb_config = BitsAndBytesConfig(
16
+ load_in_4bit=True,
17
+ bnb_4bit_compute_dtype=torch.float16
18
+ )
19
+
20
  print("Loading base model in 4-bit to save VRAM...")
21
  base_model = AutoModelForCausalLM.from_pretrained(
22
  BASE_MODEL,
23
+ quantization_config=bnb_config, # <-- This is the crucial change
24
  device_map="auto",
25
  torch_dtype=torch.float16,
26
  )