Spaces:
No application file
No application file
Upload app.py
Browse files
app.py
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import gradio as gr
|
| 2 |
+
import onnxruntime as ort
|
| 3 |
+
import numpy as np
|
| 4 |
+
import torch
|
| 5 |
+
from transformers import CLIPTokenizer, CLIPTextModel
|
| 6 |
+
import mdm_tools
|
| 7 |
+
import viz_utils
|
| 8 |
+
import os
|
| 9 |
+
|
| 10 |
+
# --- GLOBAL STATE ---
|
| 11 |
+
CURRENT_SESSION = None
|
| 12 |
+
CURRENT_MODEL_NAME = None
|
| 13 |
+
CLIP_TOKENIZER = None
|
| 14 |
+
CLIP_MODEL = None
|
| 15 |
+
|
| 16 |
+
def init_clip():
|
| 17 |
+
global CLIP_TOKENIZER, CLIP_MODEL
|
| 18 |
+
if CLIP_TOKENIZER is None:
|
| 19 |
+
print("Loading CLIP...")
|
| 20 |
+
CLIP_TOKENIZER = CLIPTokenizer.from_pretrained("openai/clip-vit-base-patch32")
|
| 21 |
+
CLIP_MODEL = CLIPTextModel.from_pretrained("openai/clip-vit-base-patch32")
|
| 22 |
+
|
| 23 |
+
def load_engine(model_name):
|
| 24 |
+
global CURRENT_SESSION, CURRENT_MODEL_NAME
|
| 25 |
+
|
| 26 |
+
onnx_file = mdm_tools.MODELS_CONFIG[model_name]['onnx_file']
|
| 27 |
+
if not os.path.exists(onnx_file):
|
| 28 |
+
return f"Error: {onnx_file} not found. Please complete the Setup tab."
|
| 29 |
+
|
| 30 |
+
try:
|
| 31 |
+
# Load ONNX with Optimization
|
| 32 |
+
opts = ort.SessionOptions()
|
| 33 |
+
opts.intra_op_num_threads = 4
|
| 34 |
+
opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
|
| 35 |
+
CURRENT_SESSION = ort.InferenceSession(onnx_file, opts)
|
| 36 |
+
CURRENT_MODEL_NAME = model_name
|
| 37 |
+
|
| 38 |
+
init_clip()
|
| 39 |
+
return f"Engine {model_name} Loaded Successfully."
|
| 40 |
+
except Exception as e:
|
| 41 |
+
return f"ONNX Runtime Error: {e}"
|
| 42 |
+
|
| 43 |
+
def generate_animation(model_name, prompt, steps):
|
| 44 |
+
# 1. Check Engine
|
| 45 |
+
if CURRENT_SESSION is None or CURRENT_MODEL_NAME != model_name:
|
| 46 |
+
res = load_engine(model_name)
|
| 47 |
+
if "Error" in res: return None, None, res
|
| 48 |
+
|
| 49 |
+
# 2. Text Embedding
|
| 50 |
+
inputs = CLIP_TOKENIZER([prompt], padding="max_length", max_length=77, return_tensors="pt")
|
| 51 |
+
with torch.no_grad():
|
| 52 |
+
embeddings = CLIP_MODEL(**inputs).last_hidden_state
|
| 53 |
+
|
| 54 |
+
# Context [1, 1, 512] - Using first token
|
| 55 |
+
context = embeddings.numpy().astype(np.float32)[:, 0:1, :]
|
| 56 |
+
|
| 57 |
+
# 3. Diffusion Process (Simulated Loop)
|
| 58 |
+
cfg = mdm_tools.MODELS_CONFIG[model_name]
|
| 59 |
+
batch = 1
|
| 60 |
+
seq = 196
|
| 61 |
+
feats = cfg['feats']
|
| 62 |
+
|
| 63 |
+
# Initial Noise
|
| 64 |
+
x = np.random.randn(batch, seq, feats).astype(np.float32)
|
| 65 |
+
|
| 66 |
+
# Reverse Diffusion Steps (e.g., 1000 -> 0)
|
| 67 |
+
step_vals = list(reversed(range(0, 1000, int(1000/steps))))
|
| 68 |
+
|
| 69 |
+
for i, t_val in enumerate(step_vals):
|
| 70 |
+
timesteps = np.array([t_val], dtype=np.int64)
|
| 71 |
+
ort_inputs = {"x": x, "timesteps": timesteps, "context": context}
|
| 72 |
+
|
| 73 |
+
# Run ONNX Model
|
| 74 |
+
pred = CURRENT_SESSION.run(None, ort_inputs)[0]
|
| 75 |
+
|
| 76 |
+
# Visual mixing (Approximation of DDIM)
|
| 77 |
+
alpha = 1.0 / (len(step_vals) - i)
|
| 78 |
+
x = x * (1 - alpha) + pred * alpha
|
| 79 |
+
|
| 80 |
+
# 4. Post-Process (Features -> 3D XYZ)
|
| 81 |
+
xyz_data = mdm_tools.recover_motion_to_xyz(x, model_name) # [Seq, Joints, 3]
|
| 82 |
+
|
| 83 |
+
# 5. Visualization
|
| 84 |
+
fig = viz_utils.plot_skeleton(xyz_data)
|
| 85 |
+
|
| 86 |
+
# 6. Export to Blender
|
| 87 |
+
# Returns path to the generated BVH
|
| 88 |
+
filename = f"mdm_{model_name}_animation.bvh"
|
| 89 |
+
bvh_path = viz_utils.save_bvh_for_blender(xyz_data, filename)
|
| 90 |
+
|
| 91 |
+
return fig, bvh_path, f"Success: '{prompt}' generated."
|
| 92 |
+
|
| 93 |
+
# --- GRADIO INTERFACE ---
|
| 94 |
+
|
| 95 |
+
with gr.Blocks(title="MDM ONNX Studio") as app:
|
| 96 |
+
gr.Markdown("# 🏃 Motion Diffusion Model - ONNX Runtime Studio")
|
| 97 |
+
gr.Markdown("Create 3D human motion from text. Optimized for CPU.")
|
| 98 |
+
|
| 99 |
+
with gr.Tabs():
|
| 100 |
+
# --- TAB 1: SETUP ---
|
| 101 |
+
with gr.Tab("1. Setup & Install"):
|
| 102 |
+
gr.Markdown("### 1. Initialize System")
|
| 103 |
+
gr.Markdown("*Clones the MDM repository and installs dependencies.*")
|
| 104 |
+
|
| 105 |
+
setup_btn = gr.Button("Clone Repository & Setup", variant="primary")
|
| 106 |
+
setup_log = gr.Textbox(label="Setup Log")
|
| 107 |
+
|
| 108 |
+
# Wrapper to handle tuple output from setup
|
| 109 |
+
def run_setup():
|
| 110 |
+
success, log = mdm_tools.setup_environment()
|
| 111 |
+
return log
|
| 112 |
+
setup_btn.click(run_setup, outputs=setup_log)
|
| 113 |
+
|
| 114 |
+
gr.Markdown("### 2. Download & Convert Models")
|
| 115 |
+
with gr.Row():
|
| 116 |
+
model_dd = gr.Dropdown(choices=["HumanML3D", "KIT"], value="HumanML3D", label="Model")
|
| 117 |
+
dl_chk = gr.Checkbox(label="Use GDrive (Slow)", value=False)
|
| 118 |
+
|
| 119 |
+
with gr.Row():
|
| 120 |
+
dl_btn = gr.Button("Download Weights")
|
| 121 |
+
conv_btn = gr.Button("Convert to ONNX")
|
| 122 |
+
|
| 123 |
+
model_log = gr.Textbox(label="Process Log")
|
| 124 |
+
|
| 125 |
+
dl_btn.click(mdm_tools.download_model, inputs=[model_dd, dl_chk], outputs=model_log)
|
| 126 |
+
conv_btn.click(mdm_tools.export_onnx, inputs=[model_dd], outputs=model_log)
|
| 127 |
+
|
| 128 |
+
# --- TAB 2: GENERATE ---
|
| 129 |
+
with gr.Tab("2. Generate Motion"):
|
| 130 |
+
with gr.Row():
|
| 131 |
+
with gr.Column(scale=1):
|
| 132 |
+
i_model = gr.Dropdown(choices=["HumanML3D", "KIT"], value="HumanML3D", label="Active Model")
|
| 133 |
+
i_prompt = gr.Textbox(label="Prompt", value="A person walking confidentially")
|
| 134 |
+
i_steps = gr.Slider(10, 50, value=20, label="Diffusion Steps", step=1)
|
| 135 |
+
i_btn = gr.Button("Generate 3D Animation", variant="primary")
|
| 136 |
+
i_lbl = gr.Label(value="Idle")
|
| 137 |
+
|
| 138 |
+
with gr.Column(scale=2):
|
| 139 |
+
viz_out = gr.Plot(label="3D Preview")
|
| 140 |
+
# The verified download button
|
| 141 |
+
file_out = gr.File(label="Download Blender File (.bvh)")
|
| 142 |
+
|
| 143 |
+
i_btn.click(
|
| 144 |
+
generate_animation,
|
| 145 |
+
inputs=[i_model, i_prompt, i_steps],
|
| 146 |
+
outputs=[viz_out, file_out, i_lbl]
|
| 147 |
+
)
|
| 148 |
+
|
| 149 |
+
if __name__ == "__main__":
|
| 150 |
+
app.launch()
|