Pi4 / app.py
yyfz233's picture
Add arXiv paper link
72ac801
Raw History Blame Contribute Delete
24.1 kB
import gc
from functools import lru_cache, partial
from html import escape
import json
import logging
import math
import os
from pathlib import Path
import re
import shutil
import sys
import time
import uuid
from urllib.parse import quote
import spaces
import gradio as gr
import torch
from fastapi import APIRouter, HTTPException
from fastapi.responses import FileResponse
from starlette.middleware import Middleware
from starlette.middleware.gzip import GZipMiddleware
HERE = Path(__file__).resolve().parent
ROOT = HERE if (HERE / "pi4").is_dir() else HERE.parent
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
VIEWER = ROOT / "viewer"
OUTPUTS = Path(os.environ.get("PI4_OUTPUT_DIR", ROOT / "outputs" / "space")).resolve()
RETENTION_SECONDS = 6 * 60 * 60
IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".webp", ".bmp"}
RUN_ID = re.compile(r"[0-9a-f]{32}")
LOGGER = logging.getLogger(__name__)
STYLE = (HERE / "space.css").read_text()
EXAMPLE_COVERS = json.loads((HERE / "example-covers.json").read_text())
EXAMPLES = json.loads((ROOT / "examples" / "catalog.json").read_text())
HEADER = """
<header class="site-header">
<div class="brand-lockup">
<a class="wordmark" href="#pi4-header" aria-label="Pi4 home">
<svg class="paper-logo" viewBox="0 0 952 762" aria-label="Pi4" role="img">
<defs><linearGradient id="space-logo-gradient" x1="0%" y1="100%" x2="100%" y2="0%">
<stop offset="0%" stop-color="#2563eb"/><stop offset="52%" stop-color="#7c3aed"/>
<stop offset="100%" stop-color="#c026d3"/>
</linearGradient>
<symbol id="space-icon-arrow" viewBox="0 0 24 24"><path d="M5 12h14M13 6l6 6-6 6"/></symbol>
<symbol id="space-icon-external" viewBox="0 0 24 24"><path d="M7 17 17 7M7 7h10v10"/></symbol>
<symbol id="space-icon-github" viewBox="0 0 24 24"><path fill="currentColor" stroke="none" d="M12 .7C6.5.7 2 5.2 2 10.7c0 4.4 2.9 8.2 6.8 9.5.5.1.7-.2.7-.5v-1.9c-2.8.6-3.4-1.2-3.4-1.2-.5-1.2-1.1-1.5-1.1-1.5-.9-.6.1-.6.1-.6 1 .1 1.5 1 1.5 1 .9 1.6 2.4 1.1 3 .9.1-.7.4-1.1.6-1.4-2.2-.3-4.6-1.1-4.6-5 0-1.1.4-2 1-2.7-.1-.3-.4-1.3.1-2.7 0 0 .8-.3 2.8 1a9.4 9.4 0 0 1 5 0c1.9-1.3 2.8-1 2.8-1 .5 1.4.2 2.4.1 2.7.6.7 1 1.6 1 2.7 0 3.9-2.4 4.7-4.6 5 .4.3.7.9.7 1.9v2.8c0 .3.2.6.7.5 4-1.3 6.8-5.1 6.8-9.5C22 5.2 17.5.7 12 .7Z"/></symbol>
<symbol id="space-icon-report" viewBox="0 0 24 24"><path d="M7 3h7l4 4v14H7V3Z"/><path d="M14 3v5h4M10 12h5M10 16h5"/></symbol>
</defs>
<g id="space-pi4-glyph" fill="url(#space-logo-gradient)">
<path transform="translate(0 740) scale(1 -1)" d="M171 485C139 485 101 485 79 468C55 450 31 355 18 291L46 279C62 325 92 388 135 388H229C221 333 200 243 180 200C145 125 89 40 75 21L76 8L154 -4L167 4C221 131 258 254 283 388H411L418 380C410 315 404 248 404 183C404 96 399 1 507 1C566 1 615 48 654 85L644 105L634 107C620 94 598 81 582 81C511 81 509 174 509 258C509 294 513 344 518 388H596C626 388 661 461 685 511L669 523C642 493 589 485 543 485Z"/>
<path transform="translate(660 320) scale(.45 -.45)" d="M606 154V210H497V627C497 657 492 662 462 662H447C428 662 420 662 407 647L40 210V154H360V56H236V0C285 4 375 4 427 4C449 4 581 2 606 0V56H497V154ZM373 524V210H109Z"/>
</g>
</svg>
</a>
<div class="lab-lockup"><a class="lab-brand" href="https://www.shlab.org.cn/" target="_blank" rel="noopener" aria-label="Shanghai Artificial Intelligence Laboratory"><img src="/space-assets/branding/ailab-logo-transparent.webp" alt="Shanghai Artificial Intelligence Laboratory" width="1542" height="185"></a><span class="team-name">Physical Intelligence Team</span></div>
</div>
<nav aria-label="Main navigation">
<span class="nav-item"><span class="nav-note nav-note-left" aria-hidden="true"><span>open source!</span><svg viewBox="0 0 28 22"><path d="M3 2c10 0 17 5 18 14m-6-4 6 5 4-7"/></svg></span><a class="nav-button nav-external" href="https://github.com/yyfz/Pi4" target="_blank" rel="noopener"><svg aria-hidden="true"><use href="#space-icon-github"/></svg>GitHub</a></span>
<a class="nav-button nav-external" href="https://arxiv.org/abs/2611.23144" target="_blank" rel="noopener"><svg aria-hidden="true"><use href="#space-icon-report"/></svg>Technical Report</a>
<span class="nav-item"><span class="nav-note nav-note-right" aria-hidden="true"><span>see more demos!</span><svg viewBox="0 0 28 22"><path d="M3 2c10 0 17 5 18 14m-6-4 6 5 4-7"/></svg></span><a class="nav-button nav-external" href="https://yyfz.github.io/pi4" target="_blank" rel="noopener"><svg aria-hidden="true"><use href="#space-icon-external"/></svg>Project Page</a></span>
<span class="nav-item"><span class="nav-note nav-note-left" aria-hidden="true"><span>open dataset!</span><svg viewBox="0 0 28 22"><path d="M3 2c10 0 17 5 18 14m-6-4 6 5 4-7"/></svg></span><a class="nav-button nav-external" href="https://weiguangzhao.github.io/GameRealm/" target="_blank" rel="noopener"><svg aria-hidden="true" viewBox="0 0 24 24"><ellipse cx="12" cy="5" rx="8" ry="3"/><path d="M4 5v7c0 1.7 3.6 3 8 3s8-1.3 8-3V5M4 12v7c0 1.7 3.6 3 8 3s8-1.3 8-3v-7"/></svg>GameRealm</a></span>
</nav>
</header>
"""
FOOTER = """
<footer class="site-footer">
<div class="footer-main">
<div class="brand-lockup"><a class="wordmark" href="#pi4-header" aria-label="Pi4 home"><svg class="paper-logo" viewBox="0 0 952 762" aria-hidden="true"><use href="#space-pi4-glyph"/></svg></a><div class="lab-lockup"><a class="lab-brand" href="https://www.shlab.org.cn/" target="_blank" rel="noopener" aria-label="Shanghai Artificial Intelligence Laboratory"><img src="/space-assets/branding/ailab-logo-transparent.webp" alt="Shanghai Artificial Intelligence Laboratory" width="1542" height="185"></a><span class="team-name">Physical Intelligence Team</span></div></div>
</div>
<p class="footer-license">π⁴ · Research demo <span>CC BY-NC 4.0 · Noncommercial use only.</span></p>
<p class="footer-notice">Uploads and outputs are temporary, not private storage. Do not upload sensitive content. Forward timing excludes queueing, model transfer, and export. Third-party terms apply; see LICENSE, LICENSE_MODEL, and NOTICE.</p>
</footer>
"""
EMPTY_VIEWER = """
<div class="empty-scene">
<div class="empty-scene-icon" aria-hidden="true">
<svg viewBox="0 0 96 96" fill="none">
<path d="m48 16 29 17v32L48 82 19 65V33Z" stroke="#d4c4ec" stroke-width="1.2"/>
<path d="m19 33 29 17 29-17M48 50v32M48 16v34" stroke="#d4c4ec" stroke-width="1.2"/>
<path d="m35 39 13-8 13 8v16l-13 8-13-8Z" fill="#f0eafa" stroke="#9b79c6" stroke-width="1.3"/>
<path d="m35 39 13 8 13-8M48 47v16" stroke="#9b79c6" stroke-width="1.3"/>
<circle cx="19" cy="33" r="2.5" fill="#bda2df"/><circle cx="77" cy="65" r="2.5" fill="#bda2df"/>
<circle cx="48" cy="16" r="2.5" fill="#bda2df"/>
</svg>
</div>
<h1 class="scene-title">
<span class="hero-title-brand" aria-hidden="true"><svg class="paper-logo" viewBox="0 0 952 762"><use href="#space-pi4-glyph"/></svg><span>:</span></span>
<span class="hero-title-copy"><span class="sr-only">π⁴: </span><span class="hero-title-line">A Foundation Model for 4D Scene Geometry and Motion</span></span>
</h1>
<h2>Your scene, in another dimension.</h2>
<p>Upload a clip or try an example.<br>Explore geometry, cameras, and motion right here.</p>
<div class="empty-scene-steps"><span><b>01</b> Add media</span><i></i><span><b>02</b> Reconstruct</span><i></i><span><b>03</b> Explore</span></div>
</div>
"""
def create_theme():
palette = {
"body_background_fill": "#f7f7f9",
"background_fill_primary": "#ffffff",
"background_fill_secondary": "#faf9fc",
"body_text_color": "#26272e",
"body_text_color_subdued": "#737580",
"border_color_primary": "#e9e7ee",
"block_background_fill": "#ffffff",
"block_border_color": "#e9e7ee",
"block_label_background_fill": "#ffffff",
"block_label_text_color": "#737580",
"input_background_fill": "#ffffff",
"input_border_color": "#e9e7ee",
"input_border_color_focus": "#a78bce",
"button_primary_background_fill": "#26272e",
"button_primary_background_fill_hover": "#454650",
"button_primary_border_color": "#26272e",
"button_primary_text_color": "#ffffff",
"checkbox_background_color_selected": "#8060a6",
"checkbox_border_color_selected": "#8060a6",
"checkbox_label_background_fill_selected": "#f5f0fb",
"checkbox_label_border_color_selected": "#d7c8eb",
"slider_color": "#a78bce",
"loader_color": "#8060a6",
}
theme = gr.themes.Base(
primary_hue="purple",
neutral_hue="gray",
).set(**palette)
values = theme.to_dict()["theme"]
return theme.set(**{
f"{name}_dark": value for name, value in values.items()
if not name.endswith("_dark") and f"{name}_dark" in values
})
def integer(value, name, minimum, maximum):
if (
isinstance(value, bool)
or not isinstance(value, (int, float))
or not math.isfinite(value)
or value != int(value)
or not minimum <= value <= maximum
):
raise gr.Error(f"{name} must be an integer from {minimum} to {maximum}.")
return int(value)
def prune_outputs():
OUTPUTS.mkdir(parents=True, exist_ok=True)
cutoff = time.time() - RETENTION_SECONDS
for directory in OUTPUTS.iterdir():
if RUN_ID.fullmatch(directory.name) and directory.is_dir():
try:
if directory.stat().st_mtime < cutoff:
shutil.rmtree(directory)
except OSError:
LOGGER.warning("Could not expire %s", directory.name, exc_info=True)
@lru_cache(maxsize=1)
def get_model():
from pi4 import Pi4
return Pi4.from_pretrained(
os.environ.get("PI4_MODEL_ID", "yyfz233/Pi4"),
revision=os.environ.get("PI4_MODEL_REVISION") or None,
)
@spaces.GPU(duration=120)
def predict(frames, query_frame, model):
if not torch.cuda.is_available():
raise gr.Error("A CUDA GPU is required. Select ZeroGPU in the Space settings.")
images = None
prediction = None
try:
model.cuda()
images = (
torch.from_numpy(frames.copy())
.permute(0, 3, 1, 2)
.float()
.div_(255)
.unsqueeze(0)
.cuda()
)
torch.cuda.synchronize()
started = time.perf_counter()
with torch.inference_mode():
prediction = model(images, query_frame=query_frame)
torch.cuda.synchronize()
elapsed = time.perf_counter() - started
result = {
name: value.detach().cpu() if isinstance(value, torch.Tensor) else value
for name, value in prediction.items()
}
return result, elapsed
except torch.cuda.OutOfMemoryError as error:
raise gr.Error(
"GPU memory is full. Reduce the frame limit and retry. "
"The demo does not silently split the sequence or lower its resolution."
) from error
finally:
model.cpu()
del images, prediction
gc.collect()
torch.cuda.empty_cache()
def reconstruct(
input_type,
video,
images,
tracking,
query_frame,
stride,
max_frames,
progress=gr.Progress(),
):
from pi4.io import load_images, save_results
stride = integer(stride, "Sampling stride", 1, 60)
max_frames = integer(max_frames, "Frame limit", 1, 150)
query = integer(query_frame, "Query frame", 0, 149) if tracking else None
if input_type == "Video":
if not video or not Path(video).is_file():
raise gr.Error("Upload a video or select an example first.")
elif input_type == "Images":
if not images or len(images) > 300:
raise gr.Error("Upload between 1 and 300 images.")
if any(
not Path(image).is_file() or Path(image).suffix.lower() not in IMAGE_SUFFIXES
for image in images
):
raise gr.Error("Images must be JPG, PNG, WEBP, or BMP files.")
else:
raise gr.Error("Choose Video or Images as the input type.")
prune_outputs()
run = OUTPUTS / uuid.uuid4().hex
run.mkdir()
try:
progress(0.05, desc="Loading and sampling RGB frames on CPU")
if input_type == "Images":
source = run / "input"
source.mkdir()
for index, image in enumerate(sorted(images, key=lambda item: Path(item).name)):
image = Path(image)
shutil.copyfile(image, source / f"{index:06d}{image.suffix.lower()}")
else:
source = Path(video)
frames, metadata = load_images(
source, stride=stride, max_frames=max_frames, max_size=512
)
metadata["source"] = "uploaded images" if input_type == "Images" else source.name
if input_type == "Images":
shutil.rmtree(source)
if tracking and len(frames) < 2:
raise gr.Error("Tracking needs at least two sampled frames. Disable tracking for one image.")
if query is not None and query >= len(frames):
raise gr.Error(
f"Only {len(frames)} frames were sampled. Query frame must be from 0 "
f"to {len(frames) - 1}, after sampling."
)
progress(0.2, desc="Loading cached weights on CPU; first use may take longer")
model = get_model()
progress(0.35, desc="Waiting for ZeroGPU and running one joint forward pass")
prediction, elapsed = predict(frames, query, model)
progress(0.8, desc="Exporting the interactive viewer and downloads on CPU")
data = run / "data"
scene = save_results(data, frames, prediction, metadata, viewer=True)
del prediction
archive = shutil.make_archive(str(run / "pi4-result"), "zip", root_dir=data)
width, height = scene["image_size"]
status = (
f"**Done:** {len(frames)} frames · {width} × {height} · "
f"{elapsed:.1f} s model forward · {scene['preview']['points']:,} preview points.\n\n"
"One joint forward pass · Results kept for up to 6 hours."
)
url = f"/view/{run.name}/"
preview_source = "" if input_type == "Images" else escape(f"/gradio_api/file={quote(str(source.resolve()), safe='/')}", quote=True)
viewer = (
f'<div class="viewer-link"><a href="{url}" target="_blank" rel="noopener">Open in a new tab ↗</a></div>'
f'<iframe src="{url}" data-source-video="{preview_source}" title="Pi4 geometry and motion viewer" '
'style="width:100%;height:720px;border:0;display:block" '
'allow="fullscreen" allowfullscreen></iframe>'
)
gallery = [
(frame, f"Source frame {metadata['source_frame_indices'][index]}")
for index, frame in enumerate(frames[:24])
]
progress(1, desc="Ready")
return viewer, status, [str(data / scene["preview"]["file"]), archive], gallery
except gr.Error:
shutil.rmtree(run, ignore_errors=True)
raise
except Exception as error:
shutil.rmtree(run, ignore_errors=True)
LOGGER.exception("Pi4 reconstruction failed")
raise gr.Error(
"Reconstruction failed. Check that the upload is readable; try fewer frames "
"to reduce memory use. The Space logs contain the error details."
) from error
def load_example(filename, stride=1):
return "Video", str(ROOT / "examples" / filename), None, True, 0, stride, 150
def create_demo():
with gr.Blocks(title="π⁴: A Foundation Model for 4D Scene Geometry and Motion", delete_cache=(3600, RETENTION_SECONDS)) as demo:
gr.HTML(HEADER, elem_id="pi4-header")
with gr.Column(elem_id="pi4-workspace"):
with gr.Column(min_width=0, elem_id="pi4-results"):
with gr.Column(elem_id="pi4-scene-card"):
viewer = gr.HTML(EMPTY_VIEWER, elem_id="pi4-viewer")
status = gr.Markdown("Your reconstruction will appear here.", elem_id="pi4-status")
with gr.Accordion("Downloads & sampled frames", open=False, elem_id="pi4-artifacts"):
downloads = gr.File(label="Preview PLY / complete scene ZIP", file_count="multiple", interactive=False)
gallery = gr.Gallery(label="Sampled RGB frames (first 24)", columns=6, height=240)
with gr.Row(elem_id="pi4-controls", equal_height=False):
with gr.Column(scale=1, min_width=340, elem_id="pi4-input-card"):
input_type = gr.Radio(
["Video", "Images"], value="Video", label="Input type",
show_label=False, elem_id="pi4-input-type",
)
video = gr.Video(
label="Upload a video", sources=["upload"], height=206,
show_label=False, elem_id="pi4-video",
)
images = gr.File(
label="Upload images (filename order)",
file_count="multiple",
file_types=sorted(IMAGE_SUFFIXES),
type="filepath",
height=206, visible=False, elem_id="pi4-images",
)
gr.Markdown("Drop media here, or load a clip from the example cards.", elem_id="pi4-upload-hint")
tracking = gr.Checkbox(
value=True, label="Enable dense motion tracking",
elem_id="pi4-tracking",
)
with gr.Accordion("Sampling settings", open=True, elem_id="pi4-sampling"):
stride = gr.Slider(1, 60, value=1, step=1, label="Take every Nth source frame")
max_frames = gr.Slider(1, 150, value=150, step=1, label="Maximum sampled frames")
query_frame = gr.Number(value=0, precision=0, label="Tracking query frame (after sampling)")
gr.Markdown("All sampled frames run together. Lower the frame limit for less GPU memory use. Disable tracking for a single image.")
gr.HTML('<div class="inference-defaults"><span>Defaults</span><i></i><span>512 px</span><i></i><span>150 frames</span><i></i><span>Stride 1</span></div>')
submit = gr.Button("Reconstruct & Track", variant="primary", elem_id="pi4-submit")
gr.Markdown(
"Figure Skating defaults to its first 150 frames (~12 s).\n\n"
"GPU availability follows your Hugging Face quota.",
elem_id="pi4-input-note",
)
with gr.Column(scale=1, min_width=340, elem_id="pi4-examples-card"):
gr.Markdown("People, places, and motion from the project page. Pick a cover to load its RGB video and sampling preset.", elem_id="pi4-examples-hint")
with gr.Row(elem_id="pi4-examples", equal_height=True):
for example in EXAMPLES:
title = example["title"]
card = gr.HTML(
f'<button type="button" class="example-card" aria-label="Load {title} example">'
f'<span class="example-cover"><img src="{EXAMPLE_COVERS[example["key"]]}" alt="{title} video preview" width="512" height="288"/>'
'<span class="example-badge">RGB VIDEO</span><span class="example-load" aria-hidden="true">↗</span></span>'
f'<span class="example-copy"><span class="example-title">{title}</span>'
f'<span class="example-description">{example["description"]}</span>'
f'<span class="example-meta">{example["meta"]}<span>Load clip ↗</span></span></span></button>',
js_on_load="element.querySelector('button').addEventListener('click', () => trigger('click'));",
elem_id=f"pi4-example-{example['key']}", min_width=0,
)
card.click(
partial(load_example, example["file"], example["stride"]),
outputs=[input_type, video, images, tracking, query_frame, stride, max_frames],
queue=False, api_visibility="private",
)
gr.Markdown("Selecting a cover only loads the input. Click **Reconstruct & Track** to run Pi4, then explore the full-width viewer above.", elem_id="pi4-examples-note")
input_type.change(
lambda selected: [gr.update(visible=selected == "Video"), gr.update(visible=selected == "Images")],
inputs=input_type, outputs=[video, images], queue=False, api_visibility="private",
)
submit.click(
reconstruct,
inputs=[input_type, video, images, tracking, query_frame, stride, max_frames],
outputs=[viewer, status, downloads, gallery],
concurrency_limit=1,
api_name="reconstruct",
)
gr.HTML(FOOTER, elem_id="pi4-footer")
return demo.queue(max_size=10, default_concurrency_limit=1)
def result_directory(run_id):
if not RUN_ID.fullmatch(run_id):
raise HTTPException(status_code=404)
data = OUTPUTS / run_id / "data"
if not (data / "scene.json").is_file():
raise HTTPException(status_code=404)
return data
def serve_asset(root, asset, suffixes, *, media_type=None):
path = (root / asset).resolve()
if not path.is_relative_to(root.resolve()) or not path.is_file() or path.suffix not in suffixes:
raise HTTPException(status_code=404)
return FileResponse(path, media_type=media_type, headers={"Cache-Control": "private, no-cache", "X-Content-Type-Options": "nosniff"})
def create_viewer_router():
router = APIRouter()
@router.api_route("/space-assets/fonts/{filename}", methods=["GET", "HEAD"])
def font_asset(filename: str):
return serve_asset(HERE / "assets" / "fonts", filename, {".ttf"})
@router.api_route("/space-assets/branding/{filename}", methods=["GET", "HEAD"])
def branding_asset(filename: str):
media_types = {".webp": "image/webp", ".svg": "image/svg+xml"}
return serve_asset(HERE / "assets" / "branding", filename, set(media_types), media_type=media_types.get(Path(filename).suffix))
@router.api_route("/view/{run_id}/data/{asset:path}", methods=["GET", "HEAD"])
def scene_asset(run_id: str, asset: str):
return serve_asset(result_directory(run_id), asset, {".json", ".bin", ".jpg"})
@router.api_route("/view/{run_id}/", methods=["GET", "HEAD"])
def viewer_index(run_id: str):
result_directory(run_id)
return serve_asset(VIEWER, "index.html", {".html"})
@router.api_route("/view/{run_id}/{asset:path}", methods=["GET", "HEAD"])
def viewer_asset(run_id: str, asset: str):
result_directory(run_id)
return serve_asset(VIEWER, asset, {".html", ".js", ".css"})
return router
demo = create_demo()
def main():
prune_outputs()
return demo.launch(
server_name="0.0.0.0",
server_port=int(os.environ.get("GRADIO_SERVER_PORT", "7860")),
share=False,
app_kwargs={
"routes": create_viewer_router().routes,
"middleware": [Middleware(GZipMiddleware, minimum_size=1000)],
},
allowed_paths=[str(OUTPUTS), *[str(ROOT / "examples" / example["file"]) for example in EXAMPLES]],
max_file_size="256mb",
show_error=False,
ssr_mode=False,
run_history=False,
theme=create_theme(),
css=STYLE,
footer_links=[],
)
if __name__ == "__main__":
main()