bugaku commited on
Commit
4e3d3cf
Β·
verified Β·
1 Parent(s): 7beba88

Upload app.py

Browse files
Files changed (1) hide show
  1. app.py +8 -3
app.py CHANGED
@@ -7,7 +7,7 @@ Make sure your model is uploaded to HF first using export_model.py.
7
  """
8
 
9
  import gradio as gr
10
- from transformers import AutoModelForCausalLM, AutoTokenizer
11
  import torch
12
 
13
  # ──────────────────────────────────────────────
@@ -18,11 +18,16 @@ MODEL_REPO = "bugaku/fufufafa" # your HF model repo
18
 
19
  print(f"Loading model: {MODEL_REPO}")
20
  tokenizer = AutoTokenizer.from_pretrained(MODEL_REPO)
 
 
 
 
 
21
  model = AutoModelForCausalLM.from_pretrained(
22
  MODEL_REPO,
 
23
  dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
24
  device_map="auto",
25
- quantization_config=None, # ignore any saved quantization config
26
  )
27
  model.eval()
28
  print("Model loaded!")
@@ -66,4 +71,4 @@ with gr.Blocks(title="Style Model") as demo:
66
  chatbot=gr.Chatbot(height=500),
67
  )
68
 
69
- demo.launch()
 
7
  """
8
 
9
  import gradio as gr
10
+ from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig
11
  import torch
12
 
13
  # ──────────────────────────────────────────────
 
18
 
19
  print(f"Loading model: {MODEL_REPO}")
20
  tokenizer = AutoTokenizer.from_pretrained(MODEL_REPO)
21
+
22
+ # Load config and strip quantization so it loads cleanly without bitsandbytes
23
+ config = AutoConfig.from_pretrained(MODEL_REPO)
24
+ config.quantization_config = None
25
+
26
  model = AutoModelForCausalLM.from_pretrained(
27
  MODEL_REPO,
28
+ config=config,
29
  dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
30
  device_map="auto",
 
31
  )
32
  model.eval()
33
  print("Model loaded!")
 
71
  chatbot=gr.Chatbot(height=500),
72
  )
73
 
74
+ demo.launch()