SHK4K commited on
Commit
2c71b9c
·
verified ·
1 Parent(s): 9407bd2

Add @spaces.GPU inference function for ZeroGPU

Browse files
Files changed (1) hide show
  1. app.py +8 -1
app.py CHANGED
@@ -6,6 +6,7 @@ import re
6
  import sys
7
 
8
  import gradio as gr
 
9
  import torch
10
  from transformers import AutoModelForTokenClassification, AutoTokenizer
11
 
@@ -35,7 +36,9 @@ def load():
35
  if _model is None:
36
  _tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
37
  _model = AutoModelForTokenClassification.from_pretrained(MODEL_ID).eval()
38
- _model.to("cuda" if torch.cuda.is_available() else "cpu")
 
 
39
  return _tokenizer, _model
40
 
41
 
@@ -121,6 +124,7 @@ def _highlight(text, cut):
121
  )
122
 
123
 
 
124
  def run(text):
125
  try:
126
  r = segment(text)
@@ -129,6 +133,9 @@ def run(text):
129
  return r["html"], r["sanad"], r["matn"], r["note"]
130
 
131
 
 
 
 
132
  CSS = """
133
  #hadith-input textarea { direction: rtl; font-size: 1.05rem; line-height: 1.9; }
134
  .hadith { font-size: 1.15rem; line-height: 2.1; padding: 1rem; border-radius: .5rem;
 
6
  import sys
7
 
8
  import gradio as gr
9
+ import spaces
10
  import torch
11
  from transformers import AutoModelForTokenClassification, AutoTokenizer
12
 
 
36
  if _model is None:
37
  _tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
38
  _model = AutoModelForTokenClassification.from_pretrained(MODEL_ID).eval()
39
+ # Called from inside an @GPU scope, where the CUDA device is attached.
40
+ if torch.cuda.is_available() and _model.device.type == "cpu":
41
+ _model.to("cuda")
42
  return _tokenizer, _model
43
 
44
 
 
124
  )
125
 
126
 
127
+ @spaces.GPU(duration=60)
128
  def run(text):
129
  try:
130
  r = segment(text)
 
133
  return r["html"], r["sanad"], r["matn"], r["note"]
134
 
135
 
136
+ load() # warm the Hub cache at startup, outside any GPU allocation
137
+
138
+
139
  CSS = """
140
  #hadith-input textarea { direction: rtl; font-size: 1.05rem; line-height: 1.9; }
141
  .hadith { font-size: 1.15rem; line-height: 2.1; padding: 1rem; border-radius: .5rem;