AnimoFlow commited on
Commit
160b869
·
verified ·
1 Parent(s): a63413b

Initial probe deploy 2026-06-14 18:02:50

Browse files
Files changed (4) hide show
  1. README.md +2 -2
  2. app.py +18 -5
  3. requirements.txt +1 -0
  4. rewriter.py +39 -12
README.md CHANGED
@@ -6,8 +6,8 @@ colorTo: indigo
6
  sdk: docker
7
  app_port: 7860
8
  pinned: false
9
- short_description: Multilingual prompt rewriter probe (Gemma-3-1B-it + RAFSL)
10
- hardware: cpu-basic
11
  ---
12
 
13
  # AnimoFlow Rewriter Probe
 
6
  sdk: docker
7
  app_port: 7860
8
  pinned: false
9
+ short_description: Multilingual prompt rewriter probe (Qwen + RAFSL)
10
+ hardware: zero-a10g
11
  ---
12
 
13
  # AnimoFlow Rewriter Probe
app.py CHANGED
@@ -13,6 +13,12 @@ from fastapi import FastAPI, HTTPException
13
  from fastapi.middleware.cors import CORSMiddleware
14
  from pydantic import BaseModel, Field
15
 
 
 
 
 
 
 
16
  from rewriter import Rewriter
17
 
18
  DATA_DIR = Path(__file__).parent / "data"
@@ -78,12 +84,20 @@ def healthz():
78
  }
79
 
80
 
 
 
 
 
 
 
 
 
 
81
  @app.post("/rewrite", response_model=RewriteResponse)
82
  def rewrite(req: RewriteRequest):
83
  global _FIRST_INFERENCE_AT
84
- rw = get_rewriter()
85
  try:
86
- out = rw.rewrite(req.prompt)
87
  except Exception as e:
88
  raise HTTPException(status_code=500, detail=f"rewriter failed: {type(e).__name__}: {e}")
89
  if _FIRST_INFERENCE_AT is None:
@@ -91,12 +105,11 @@ def rewrite(req: RewriteRequest):
91
  return RewriteResponse(**out)
92
 
93
 
94
- # Gradio UI (mounted at /)
95
  def gradio_rewrite(prompt: str):
96
  if not prompt.strip():
97
  return "(empty)", "(empty)", 0.0
98
- rw = get_rewriter()
99
- out = rw.rewrite(prompt)
100
  examples_str = "\n".join(f" · {e}" for e in out["examples"])
101
  return out["rewritten"], examples_str, out["latency_s"]
102
 
 
13
  from fastapi.middleware.cors import CORSMiddleware
14
  from pydantic import BaseModel, Field
15
 
16
+ try:
17
+ import spaces # HF ZeroGPU decorator
18
+ _HAS_SPACES = True
19
+ except ImportError:
20
+ _HAS_SPACES = False
21
+
22
  from rewriter import Rewriter
23
 
24
  DATA_DIR = Path(__file__).parent / "data"
 
84
  }
85
 
86
 
87
+ def _do_rewrite(prompt: str) -> dict:
88
+ return get_rewriter().rewrite(prompt)
89
+
90
+
91
+ # On ZeroGPU, wrap the inference call so HF allocates a GPU burst for it.
92
+ if _HAS_SPACES:
93
+ _do_rewrite = spaces.GPU(duration=30)(_do_rewrite)
94
+
95
+
96
  @app.post("/rewrite", response_model=RewriteResponse)
97
  def rewrite(req: RewriteRequest):
98
  global _FIRST_INFERENCE_AT
 
99
  try:
100
+ out = _do_rewrite(req.prompt)
101
  except Exception as e:
102
  raise HTTPException(status_code=500, detail=f"rewriter failed: {type(e).__name__}: {e}")
103
  if _FIRST_INFERENCE_AT is None:
 
105
  return RewriteResponse(**out)
106
 
107
 
108
+ # Gradio UI (mounted at /ui)
109
  def gradio_rewrite(prompt: str):
110
  if not prompt.strip():
111
  return "(empty)", "(empty)", 0.0
112
+ out = _do_rewrite(prompt)
 
113
  examples_str = "\n".join(f" · {e}" for e in out["examples"])
114
  return out["rewritten"], examples_str, out["latency_s"]
115
 
requirements.txt CHANGED
@@ -7,3 +7,4 @@ sentence-transformers>=3.0
7
  huggingface_hub>=0.25.0
8
  gradio>=5.6.0
9
  accelerate>=1.0
 
 
7
  huggingface_hub>=0.25.0
8
  gradio>=5.6.0
9
  accelerate>=1.0
10
+ spaces>=0.30.0
rewriter.py CHANGED
@@ -11,10 +11,17 @@ import torch
11
  from sentence_transformers import SentenceTransformer
12
  from transformers import AutoModelForCausalLM, AutoTokenizer
13
 
 
 
 
 
 
 
14
  # Switchable via MODEL_REPO env var. Default Qwen2.5-1.5B-Instruct (Apache 2.0, ungated).
15
  # Gemma-3-1B-it can be selected once HF Space has license acceptance via HF_TOKEN.
16
  MODEL_ID = os.environ.get("MODEL_REPO", "Qwen/Qwen2.5-1.5B-Instruct")
17
  RETRIEVER_ID = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
 
18
 
19
  SYSTEM_PROMPT = """You are a prompt rewriter for a text-to-motion model that was trained on the HumanML3D dataset. Convert the user's input (in any language, any phrasing) into ONE short English caption that describes the motion only, in the exact style of HumanML3D captions.
20
 
@@ -49,12 +56,19 @@ class Retriever:
49
 
50
  class Rewriter:
51
  def __init__(self, data_dir: Path, device: str | None = None, dtype: str | None = None):
52
- if device is None:
 
 
 
 
 
53
  device = "cuda" if torch.cuda.is_available() else ("mps" if torch.backends.mps.is_available() else "cpu")
54
  if dtype is None:
55
  dtype = "bfloat16" if device != "cpu" else "float32"
56
  self.device = device
57
  self.model_id = MODEL_ID
 
 
58
  t0 = time.time()
59
  self.tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
60
  self.model = AutoModelForCausalLM.from_pretrained(
@@ -63,7 +77,24 @@ class Rewriter:
63
  device_map=device,
64
  ).eval()
65
  self.load_latency = time.time() - t0
66
- self.retriever = Retriever(data_dir, device=device if device != "cuda" else "cpu") # MiniLM on CPU is fine
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
67
 
68
  def rewrite(self, user_input: str, max_new_tokens: int = 64) -> dict:
69
  t0 = time.time()
@@ -81,18 +112,14 @@ class Rewriter:
81
  messages, add_generation_prompt=True, return_tensors="pt", return_dict=False
82
  )
83
  if hasattr(encoded, "shape"):
84
- prompt_ids = encoded.to(self.device)
85
  else:
86
- prompt_ids = encoded["input_ids"].to(self.device)
87
  input_len = prompt_ids.shape[-1]
88
- with torch.no_grad():
89
- out = self.model.generate(
90
- prompt_ids,
91
- max_new_tokens=max_new_tokens,
92
- do_sample=False,
93
- temperature=None,
94
- top_p=None,
95
- )
96
  latency = time.time() - t0
97
  new_tokens = out[0, input_len:]
98
  decoded = self.tokenizer.decode(new_tokens, skip_special_tokens=True).strip()
 
11
  from sentence_transformers import SentenceTransformer
12
  from transformers import AutoModelForCausalLM, AutoTokenizer
13
 
14
+ try:
15
+ import spaces # HF ZeroGPU decorator
16
+ _HAS_SPACES = True
17
+ except ImportError:
18
+ _HAS_SPACES = False
19
+
20
  # Switchable via MODEL_REPO env var. Default Qwen2.5-1.5B-Instruct (Apache 2.0, ungated).
21
  # Gemma-3-1B-it can be selected once HF Space has license acceptance via HF_TOKEN.
22
  MODEL_ID = os.environ.get("MODEL_REPO", "Qwen/Qwen2.5-1.5B-Instruct")
23
  RETRIEVER_ID = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
24
+ ON_ZEROGPU = os.environ.get("SPACES_ZERO_GPU") == "true"
25
 
26
  SYSTEM_PROMPT = """You are a prompt rewriter for a text-to-motion model that was trained on the HumanML3D dataset. Convert the user's input (in any language, any phrasing) into ONE short English caption that describes the motion only, in the exact style of HumanML3D captions.
27
 
 
56
 
57
  class Rewriter:
58
  def __init__(self, data_dir: Path, device: str | None = None, dtype: str | None = None):
59
+ # On ZeroGPU, load the model on CPU at boot (GPU is only available inside @spaces.GPU calls).
60
+ # We move it to CUDA on first inference inside the decorated method.
61
+ if ON_ZEROGPU:
62
+ device = "cpu"
63
+ dtype = "bfloat16"
64
+ elif device is None:
65
  device = "cuda" if torch.cuda.is_available() else ("mps" if torch.backends.mps.is_available() else "cpu")
66
  if dtype is None:
67
  dtype = "bfloat16" if device != "cpu" else "float32"
68
  self.device = device
69
  self.model_id = MODEL_ID
70
+ self._on_zerogpu = ON_ZEROGPU
71
+ self._moved_to_cuda = False
72
  t0 = time.time()
73
  self.tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
74
  self.model = AutoModelForCausalLM.from_pretrained(
 
77
  device_map=device,
78
  ).eval()
79
  self.load_latency = time.time() - t0
80
+ self.retriever = Retriever(data_dir, device="cpu") # MiniLM on CPU is fine
81
+
82
+ def _generate_core(self, prompt_ids: torch.Tensor, max_new_tokens: int) -> torch.Tensor:
83
+ with torch.no_grad():
84
+ return self.model.generate(
85
+ prompt_ids,
86
+ max_new_tokens=max_new_tokens,
87
+ do_sample=False,
88
+ temperature=None,
89
+ top_p=None,
90
+ )
91
+
92
+ def _generate_on_gpu(self, prompt_ids: torch.Tensor, max_new_tokens: int) -> torch.Tensor:
93
+ # ZeroGPU: move model + inputs to CUDA inside this call, run, optionally leave on GPU.
94
+ if not self._moved_to_cuda:
95
+ self.model = self.model.to("cuda")
96
+ self._moved_to_cuda = True
97
+ return self._generate_core(prompt_ids.to("cuda"), max_new_tokens)
98
 
99
  def rewrite(self, user_input: str, max_new_tokens: int = 64) -> dict:
100
  t0 = time.time()
 
112
  messages, add_generation_prompt=True, return_tensors="pt", return_dict=False
113
  )
114
  if hasattr(encoded, "shape"):
115
+ prompt_ids = encoded
116
  else:
117
+ prompt_ids = encoded["input_ids"]
118
  input_len = prompt_ids.shape[-1]
119
+ if self._on_zerogpu:
120
+ out = self._generate_on_gpu(prompt_ids, max_new_tokens)
121
+ else:
122
+ out = self._generate_core(prompt_ids.to(self.device), max_new_tokens)
 
 
 
 
123
  latency = time.time() - t0
124
  new_tokens = out[0, input_len:]
125
  decoded = self.tokenizer.decode(new_tokens, skip_special_tokens=True).strip()