Upload app.py
Browse files
app.py
CHANGED
|
@@ -7,7 +7,7 @@ Make sure your model is uploaded to HF first using export_model.py.
|
|
| 7 |
"""
|
| 8 |
|
| 9 |
import gradio as gr
|
| 10 |
-
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 11 |
import torch
|
| 12 |
|
| 13 |
# ββββββββββββββββββββββββββββββββββββββββββββββ
|
|
@@ -18,11 +18,16 @@ MODEL_REPO = "bugaku/fufufafa" # your HF model repo
|
|
| 18 |
|
| 19 |
print(f"Loading model: {MODEL_REPO}")
|
| 20 |
tokenizer = AutoTokenizer.from_pretrained(MODEL_REPO)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
model = AutoModelForCausalLM.from_pretrained(
|
| 22 |
MODEL_REPO,
|
|
|
|
| 23 |
dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
|
| 24 |
device_map="auto",
|
| 25 |
-
quantization_config=None, # ignore any saved quantization config
|
| 26 |
)
|
| 27 |
model.eval()
|
| 28 |
print("Model loaded!")
|
|
@@ -66,4 +71,4 @@ with gr.Blocks(title="Style Model") as demo:
|
|
| 66 |
chatbot=gr.Chatbot(height=500),
|
| 67 |
)
|
| 68 |
|
| 69 |
-
demo.launch()
|
|
|
|
| 7 |
"""
|
| 8 |
|
| 9 |
import gradio as gr
|
| 10 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig
|
| 11 |
import torch
|
| 12 |
|
| 13 |
# ββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 18 |
|
| 19 |
print(f"Loading model: {MODEL_REPO}")
|
| 20 |
tokenizer = AutoTokenizer.from_pretrained(MODEL_REPO)
|
| 21 |
+
|
| 22 |
+
# Load config and strip quantization so it loads cleanly without bitsandbytes
|
| 23 |
+
config = AutoConfig.from_pretrained(MODEL_REPO)
|
| 24 |
+
config.quantization_config = None
|
| 25 |
+
|
| 26 |
model = AutoModelForCausalLM.from_pretrained(
|
| 27 |
MODEL_REPO,
|
| 28 |
+
config=config,
|
| 29 |
dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
|
| 30 |
device_map="auto",
|
|
|
|
| 31 |
)
|
| 32 |
model.eval()
|
| 33 |
print("Model loaded!")
|
|
|
|
| 71 |
chatbot=gr.Chatbot(height=500),
|
| 72 |
)
|
| 73 |
|
| 74 |
+
demo.launch()
|