Neon-tech commited on
Commit
01630c4
·
verified ·
1 Parent(s): fd8889f

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +2 -1
app.py CHANGED
@@ -1,4 +1,5 @@
1
  import os
 
2
  import gradio as gr
3
  import psutil
4
  from transformers import AutoModelForCausalLM, AutoTokenizer
@@ -6,7 +7,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
6
  os.environ["HF_HOME"] = "/data/hf_cache"
7
 
8
  tokenizer = AutoTokenizer.from_pretrained("/data/model1")
9
- model = AutoModelForCausalLM.from_pretrained("/data/model1", device_map="auto", offload_folder="/tmp/offload")
10
 
11
  def get_stats():
12
  process = psutil.Process(os.getpid())
 
1
  import os
2
+ import torch
3
  import gradio as gr
4
  import psutil
5
  from transformers import AutoModelForCausalLM, AutoTokenizer
 
7
  os.environ["HF_HOME"] = "/data/hf_cache"
8
 
9
  tokenizer = AutoTokenizer.from_pretrained("/data/model1")
10
+ model = AutoModelForCausalLM.from_pretrained("/data/model1", device_map="cpu", offload_folder="/tmp/offload", torch_dtype=torch.bfloat16)
11
 
12
  def get_stats():
13
  process = psutil.Process(os.getpid())