Spaces:
Sleeping
Sleeping
Initial probe deploy 2026-06-14 18:02:50
Browse files- README.md +2 -2
- app.py +18 -5
- requirements.txt +1 -0
- rewriter.py +39 -12
README.md
CHANGED
|
@@ -6,8 +6,8 @@ colorTo: indigo
|
|
| 6 |
sdk: docker
|
| 7 |
app_port: 7860
|
| 8 |
pinned: false
|
| 9 |
-
short_description: Multilingual prompt rewriter probe (
|
| 10 |
-
hardware:
|
| 11 |
---
|
| 12 |
|
| 13 |
# AnimoFlow Rewriter Probe
|
|
|
|
| 6 |
sdk: docker
|
| 7 |
app_port: 7860
|
| 8 |
pinned: false
|
| 9 |
+
short_description: Multilingual prompt rewriter probe (Qwen + RAFSL)
|
| 10 |
+
hardware: zero-a10g
|
| 11 |
---
|
| 12 |
|
| 13 |
# AnimoFlow Rewriter Probe
|
app.py
CHANGED
|
@@ -13,6 +13,12 @@ from fastapi import FastAPI, HTTPException
|
|
| 13 |
from fastapi.middleware.cors import CORSMiddleware
|
| 14 |
from pydantic import BaseModel, Field
|
| 15 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
from rewriter import Rewriter
|
| 17 |
|
| 18 |
DATA_DIR = Path(__file__).parent / "data"
|
|
@@ -78,12 +84,20 @@ def healthz():
|
|
| 78 |
}
|
| 79 |
|
| 80 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
@app.post("/rewrite", response_model=RewriteResponse)
|
| 82 |
def rewrite(req: RewriteRequest):
|
| 83 |
global _FIRST_INFERENCE_AT
|
| 84 |
-
rw = get_rewriter()
|
| 85 |
try:
|
| 86 |
-
out =
|
| 87 |
except Exception as e:
|
| 88 |
raise HTTPException(status_code=500, detail=f"rewriter failed: {type(e).__name__}: {e}")
|
| 89 |
if _FIRST_INFERENCE_AT is None:
|
|
@@ -91,12 +105,11 @@ def rewrite(req: RewriteRequest):
|
|
| 91 |
return RewriteResponse(**out)
|
| 92 |
|
| 93 |
|
| 94 |
-
# Gradio UI (mounted at /)
|
| 95 |
def gradio_rewrite(prompt: str):
|
| 96 |
if not prompt.strip():
|
| 97 |
return "(empty)", "(empty)", 0.0
|
| 98 |
-
|
| 99 |
-
out = rw.rewrite(prompt)
|
| 100 |
examples_str = "\n".join(f" · {e}" for e in out["examples"])
|
| 101 |
return out["rewritten"], examples_str, out["latency_s"]
|
| 102 |
|
|
|
|
| 13 |
from fastapi.middleware.cors import CORSMiddleware
|
| 14 |
from pydantic import BaseModel, Field
|
| 15 |
|
| 16 |
+
try:
|
| 17 |
+
import spaces # HF ZeroGPU decorator
|
| 18 |
+
_HAS_SPACES = True
|
| 19 |
+
except ImportError:
|
| 20 |
+
_HAS_SPACES = False
|
| 21 |
+
|
| 22 |
from rewriter import Rewriter
|
| 23 |
|
| 24 |
DATA_DIR = Path(__file__).parent / "data"
|
|
|
|
| 84 |
}
|
| 85 |
|
| 86 |
|
| 87 |
+
def _do_rewrite(prompt: str) -> dict:
|
| 88 |
+
return get_rewriter().rewrite(prompt)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
# On ZeroGPU, wrap the inference call so HF allocates a GPU burst for it.
|
| 92 |
+
if _HAS_SPACES:
|
| 93 |
+
_do_rewrite = spaces.GPU(duration=30)(_do_rewrite)
|
| 94 |
+
|
| 95 |
+
|
| 96 |
@app.post("/rewrite", response_model=RewriteResponse)
|
| 97 |
def rewrite(req: RewriteRequest):
|
| 98 |
global _FIRST_INFERENCE_AT
|
|
|
|
| 99 |
try:
|
| 100 |
+
out = _do_rewrite(req.prompt)
|
| 101 |
except Exception as e:
|
| 102 |
raise HTTPException(status_code=500, detail=f"rewriter failed: {type(e).__name__}: {e}")
|
| 103 |
if _FIRST_INFERENCE_AT is None:
|
|
|
|
| 105 |
return RewriteResponse(**out)
|
| 106 |
|
| 107 |
|
| 108 |
+
# Gradio UI (mounted at /ui)
|
| 109 |
def gradio_rewrite(prompt: str):
|
| 110 |
if not prompt.strip():
|
| 111 |
return "(empty)", "(empty)", 0.0
|
| 112 |
+
out = _do_rewrite(prompt)
|
|
|
|
| 113 |
examples_str = "\n".join(f" · {e}" for e in out["examples"])
|
| 114 |
return out["rewritten"], examples_str, out["latency_s"]
|
| 115 |
|
requirements.txt
CHANGED
|
@@ -7,3 +7,4 @@ sentence-transformers>=3.0
|
|
| 7 |
huggingface_hub>=0.25.0
|
| 8 |
gradio>=5.6.0
|
| 9 |
accelerate>=1.0
|
|
|
|
|
|
| 7 |
huggingface_hub>=0.25.0
|
| 8 |
gradio>=5.6.0
|
| 9 |
accelerate>=1.0
|
| 10 |
+
spaces>=0.30.0
|
rewriter.py
CHANGED
|
@@ -11,10 +11,17 @@ import torch
|
|
| 11 |
from sentence_transformers import SentenceTransformer
|
| 12 |
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 13 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
# Switchable via MODEL_REPO env var. Default Qwen2.5-1.5B-Instruct (Apache 2.0, ungated).
|
| 15 |
# Gemma-3-1B-it can be selected once HF Space has license acceptance via HF_TOKEN.
|
| 16 |
MODEL_ID = os.environ.get("MODEL_REPO", "Qwen/Qwen2.5-1.5B-Instruct")
|
| 17 |
RETRIEVER_ID = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
|
|
|
|
| 18 |
|
| 19 |
SYSTEM_PROMPT = """You are a prompt rewriter for a text-to-motion model that was trained on the HumanML3D dataset. Convert the user's input (in any language, any phrasing) into ONE short English caption that describes the motion only, in the exact style of HumanML3D captions.
|
| 20 |
|
|
@@ -49,12 +56,19 @@ class Retriever:
|
|
| 49 |
|
| 50 |
class Rewriter:
|
| 51 |
def __init__(self, data_dir: Path, device: str | None = None, dtype: str | None = None):
|
| 52 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
device = "cuda" if torch.cuda.is_available() else ("mps" if torch.backends.mps.is_available() else "cpu")
|
| 54 |
if dtype is None:
|
| 55 |
dtype = "bfloat16" if device != "cpu" else "float32"
|
| 56 |
self.device = device
|
| 57 |
self.model_id = MODEL_ID
|
|
|
|
|
|
|
| 58 |
t0 = time.time()
|
| 59 |
self.tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
|
| 60 |
self.model = AutoModelForCausalLM.from_pretrained(
|
|
@@ -63,7 +77,24 @@ class Rewriter:
|
|
| 63 |
device_map=device,
|
| 64 |
).eval()
|
| 65 |
self.load_latency = time.time() - t0
|
| 66 |
-
self.retriever = Retriever(data_dir, device=
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
|
| 68 |
def rewrite(self, user_input: str, max_new_tokens: int = 64) -> dict:
|
| 69 |
t0 = time.time()
|
|
@@ -81,18 +112,14 @@ class Rewriter:
|
|
| 81 |
messages, add_generation_prompt=True, return_tensors="pt", return_dict=False
|
| 82 |
)
|
| 83 |
if hasattr(encoded, "shape"):
|
| 84 |
-
prompt_ids = encoded
|
| 85 |
else:
|
| 86 |
-
prompt_ids = encoded["input_ids"]
|
| 87 |
input_len = prompt_ids.shape[-1]
|
| 88 |
-
|
| 89 |
-
out = self.
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
do_sample=False,
|
| 93 |
-
temperature=None,
|
| 94 |
-
top_p=None,
|
| 95 |
-
)
|
| 96 |
latency = time.time() - t0
|
| 97 |
new_tokens = out[0, input_len:]
|
| 98 |
decoded = self.tokenizer.decode(new_tokens, skip_special_tokens=True).strip()
|
|
|
|
| 11 |
from sentence_transformers import SentenceTransformer
|
| 12 |
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 13 |
|
| 14 |
+
try:
|
| 15 |
+
import spaces # HF ZeroGPU decorator
|
| 16 |
+
_HAS_SPACES = True
|
| 17 |
+
except ImportError:
|
| 18 |
+
_HAS_SPACES = False
|
| 19 |
+
|
| 20 |
# Switchable via MODEL_REPO env var. Default Qwen2.5-1.5B-Instruct (Apache 2.0, ungated).
|
| 21 |
# Gemma-3-1B-it can be selected once HF Space has license acceptance via HF_TOKEN.
|
| 22 |
MODEL_ID = os.environ.get("MODEL_REPO", "Qwen/Qwen2.5-1.5B-Instruct")
|
| 23 |
RETRIEVER_ID = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
|
| 24 |
+
ON_ZEROGPU = os.environ.get("SPACES_ZERO_GPU") == "true"
|
| 25 |
|
| 26 |
SYSTEM_PROMPT = """You are a prompt rewriter for a text-to-motion model that was trained on the HumanML3D dataset. Convert the user's input (in any language, any phrasing) into ONE short English caption that describes the motion only, in the exact style of HumanML3D captions.
|
| 27 |
|
|
|
|
| 56 |
|
| 57 |
class Rewriter:
|
| 58 |
def __init__(self, data_dir: Path, device: str | None = None, dtype: str | None = None):
|
| 59 |
+
# On ZeroGPU, load the model on CPU at boot (GPU is only available inside @spaces.GPU calls).
|
| 60 |
+
# We move it to CUDA on first inference inside the decorated method.
|
| 61 |
+
if ON_ZEROGPU:
|
| 62 |
+
device = "cpu"
|
| 63 |
+
dtype = "bfloat16"
|
| 64 |
+
elif device is None:
|
| 65 |
device = "cuda" if torch.cuda.is_available() else ("mps" if torch.backends.mps.is_available() else "cpu")
|
| 66 |
if dtype is None:
|
| 67 |
dtype = "bfloat16" if device != "cpu" else "float32"
|
| 68 |
self.device = device
|
| 69 |
self.model_id = MODEL_ID
|
| 70 |
+
self._on_zerogpu = ON_ZEROGPU
|
| 71 |
+
self._moved_to_cuda = False
|
| 72 |
t0 = time.time()
|
| 73 |
self.tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
|
| 74 |
self.model = AutoModelForCausalLM.from_pretrained(
|
|
|
|
| 77 |
device_map=device,
|
| 78 |
).eval()
|
| 79 |
self.load_latency = time.time() - t0
|
| 80 |
+
self.retriever = Retriever(data_dir, device="cpu") # MiniLM on CPU is fine
|
| 81 |
+
|
| 82 |
+
def _generate_core(self, prompt_ids: torch.Tensor, max_new_tokens: int) -> torch.Tensor:
|
| 83 |
+
with torch.no_grad():
|
| 84 |
+
return self.model.generate(
|
| 85 |
+
prompt_ids,
|
| 86 |
+
max_new_tokens=max_new_tokens,
|
| 87 |
+
do_sample=False,
|
| 88 |
+
temperature=None,
|
| 89 |
+
top_p=None,
|
| 90 |
+
)
|
| 91 |
+
|
| 92 |
+
def _generate_on_gpu(self, prompt_ids: torch.Tensor, max_new_tokens: int) -> torch.Tensor:
|
| 93 |
+
# ZeroGPU: move model + inputs to CUDA inside this call, run, optionally leave on GPU.
|
| 94 |
+
if not self._moved_to_cuda:
|
| 95 |
+
self.model = self.model.to("cuda")
|
| 96 |
+
self._moved_to_cuda = True
|
| 97 |
+
return self._generate_core(prompt_ids.to("cuda"), max_new_tokens)
|
| 98 |
|
| 99 |
def rewrite(self, user_input: str, max_new_tokens: int = 64) -> dict:
|
| 100 |
t0 = time.time()
|
|
|
|
| 112 |
messages, add_generation_prompt=True, return_tensors="pt", return_dict=False
|
| 113 |
)
|
| 114 |
if hasattr(encoded, "shape"):
|
| 115 |
+
prompt_ids = encoded
|
| 116 |
else:
|
| 117 |
+
prompt_ids = encoded["input_ids"]
|
| 118 |
input_len = prompt_ids.shape[-1]
|
| 119 |
+
if self._on_zerogpu:
|
| 120 |
+
out = self._generate_on_gpu(prompt_ids, max_new_tokens)
|
| 121 |
+
else:
|
| 122 |
+
out = self._generate_core(prompt_ids.to(self.device), max_new_tokens)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 123 |
latency = time.time() - t0
|
| 124 |
new_tokens = out[0, input_len:]
|
| 125 |
decoded = self.tokenizer.decode(new_tokens, skip_special_tokens=True).strip()
|