Spaces:
Running on Zero
Running on Zero
Muhammad Uzair Khattak Claude Opus 5.5 commited on
Commit ·
2bbca49
1
Parent(s): aa23328
Remove dashes from visible UI text
Browse filesCo-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
app.py
CHANGED
|
@@ -144,10 +144,10 @@ MODALITY_COLORS = {
|
|
| 144 |
"vjepa": "#e879f9",
|
| 145 |
}
|
| 146 |
MODALITY_TIPS = {
|
| 147 |
-
"rgb": "Raw pixel video
|
| 148 |
"caption": "Natural-language description of the clip",
|
| 149 |
"transcription": "Per-second dense description with [SEC_n] markers",
|
| 150 |
-
"dinov2": "Self-supervised semantic feature maps
|
| 151 |
"siglip": "Vision-language feature maps aligned with text",
|
| 152 |
"depth": "Per-pixel distance from the camera",
|
| 153 |
"normal": "Per-pixel surface orientation",
|
|
@@ -296,8 +296,8 @@ def _header_html():
|
|
| 296 |
<h1><span class="brand">Any-to-Any Video Modeling</span>: Modeling the World by Traversing Time and Modalities</h1>
|
| 297 |
<p>Video-4M is an any-to-any multimodal model that enables video modeling across different modalities. By flexibly traversing across the time and modality axes, it can predict any modality, at any point in time, given any combination of input modalities.</p>
|
| 298 |
<p>We show two ways to use this demo:<br>
|
| 299 |
-
<b>i) Any-to-any generation</b>
|
| 300 |
-
<b>ii) Future prediction in any modality</b>
|
| 301 |
<nav class="header-links">
|
| 302 |
<a href="https://video-4m.epfl.ch" target="_blank" rel="noopener">Website</a>
|
| 303 |
<a href="https://github.com/EPFL-VILAB/Video-4M" target="_blank" rel="noopener">GitHub</a>
|
|
@@ -330,7 +330,7 @@ def _build_hyperparam_controls(default_preset_key=None):
|
|
| 330 |
# accordion always hashes to the precomputed configuration.
|
| 331 |
defaults = dict(zip(HYPERPARAM_SPEC, default_slider_values(default_preset_key)))
|
| 332 |
gr.Markdown(
|
| 333 |
-
"Temperature and CFG scale per modality
|
| 334 |
"current input modality; adjust to override. Sliders for modalities "
|
| 335 |
"not in your chain are ignored.",
|
| 336 |
elem_classes=["hint-md"],
|
|
@@ -1459,7 +1459,7 @@ def _tl_title_html(obs_mod, n_frames, stem):
|
|
| 1459 |
name = MODALITY_DISPLAY_NAMES[obs_mod]
|
| 1460 |
stem_html = (
|
| 1461 |
f' of <span class="mono" style="color:var(--text-3)">{stem}</span>'
|
| 1462 |
-
if stem else "
|
| 1463 |
)
|
| 1464 |
return (
|
| 1465 |
f'<div class="tl-title">Input to the model: <b style="color:{color}">{name}</b> · '
|
|
@@ -1653,7 +1653,7 @@ with gr.Blocks(title="Video-4M: Any-to-Any Video Modeling", theme=_THEME, css=_C
|
|
| 1653 |
gr.HTML(_step_label_html(
|
| 1654 |
1, "Select input to Video-4M",
|
| 1655 |
"Pick an example clip or upload your own video for RGB input, provide a caption, "
|
| 1656 |
-
"or choose another modality from a curated example
|
| 1657 |
"becomes the input Video-4M conditions on.",
|
| 1658 |
))
|
| 1659 |
with gr.Column(elem_id="src-zone"):
|
|
@@ -1670,7 +1670,7 @@ with gr.Blocks(title="Video-4M: Any-to-Any Video Modeling", theme=_THEME, css=_C
|
|
| 1670 |
selected_md, rgb_select_btns = _build_example_gallery(
|
| 1671 |
_GALLERY_STEMS, selected_stem=_A2A_PRE["stem"] if _A2A_PRE else None)
|
| 1672 |
upload_video = gr.Video(
|
| 1673 |
-
label="Upload a short clip
|
| 1674 |
sources=["upload"], elem_id="upload-dropbox", height=200,
|
| 1675 |
)
|
| 1676 |
|
|
@@ -1724,7 +1724,7 @@ with gr.Blocks(title="Video-4M: Any-to-Any Video Modeling", theme=_THEME, css=_C
|
|
| 1724 |
elem_classes=["example-preview-video", "clip-tile"],
|
| 1725 |
)
|
| 1726 |
else:
|
| 1727 |
-
gr.HTML('<div class="abs-text-tile">no preview
|
| 1728 |
b = gr.Button("Select", size="sm", elem_classes=["select-btn"])
|
| 1729 |
|
| 1730 |
def _abs_select(src_mode, upload_path, caption_text, abs_m, chain,
|
|
@@ -1743,8 +1743,8 @@ with gr.Blocks(title="Video-4M: Any-to-Any Video Modeling", theme=_THEME, css=_C
|
|
| 1743 |
|
| 1744 |
gr.HTML(_step_label_html(
|
| 1745 |
2, "Generation chain",
|
| 1746 |
-
"Add modalities one at a time, in the order you want them generated
|
| 1747 |
-
"
|
| 1748 |
))
|
| 1749 |
with gr.Column(elem_id="chain-zone"):
|
| 1750 |
preset_radio = gr.Radio(
|
|
@@ -2065,7 +2065,7 @@ with gr.Blocks(title="Video-4M: Any-to-Any Video Modeling", theme=_THEME, css=_C
|
|
| 2065 |
except Exception as e:
|
| 2066 |
traceback.print_exc()
|
| 2067 |
yield _frame({
|
| 2068 |
-
IDX_STATUS: gr.update(value=_status_html("error
|
| 2069 |
IDX_EMPTY: gr.update(value=_empty_state_html(), visible=False),
|
| 2070 |
IDX_BTN: gr.update(value="Generate", interactive=True),
|
| 2071 |
})
|
|
@@ -2154,7 +2154,7 @@ with gr.Blocks(title="Video-4M: Any-to-Any Video Modeling", theme=_THEME, css=_C
|
|
| 2154 |
gr.HTML(_step_label_html(
|
| 2155 |
1, "Select observation",
|
| 2156 |
"Pick an example clip (or upload your own video for RGB observation) and the "
|
| 2157 |
-
"modality you want to observe
|
| 2158 |
))
|
| 2159 |
_future_stems = _FUTURE_STEMS
|
| 2160 |
future_example_state = gr.State(_FP_PRE["stem"] if _FP_PRE else None)
|
|
@@ -2188,7 +2188,7 @@ with gr.Blocks(title="Video-4M: Any-to-Any Video Modeling", theme=_THEME, css=_C
|
|
| 2188 |
elem_classes=["example-preview-video", "clip-tile"],
|
| 2189 |
)
|
| 2190 |
else:
|
| 2191 |
-
gr.HTML('<div class="abs-text-tile">no preview
|
| 2192 |
b = gr.Button("Select", size="sm", elem_classes=["select-btn"])
|
| 2193 |
b.click(None, js=_sel_busy_js("fp-selected"))
|
| 2194 |
b.click(None, js=_sel_busy_js(FP_RESULTS_HOST, size=RESULTS_SPINNER_SIZE))
|
|
@@ -2210,7 +2210,7 @@ with gr.Blocks(title="Video-4M: Any-to-Any Video Modeling", theme=_THEME, css=_C
|
|
| 2210 |
# same precedence rule as tab 1's upload_video. Placed
|
| 2211 |
# after the gallery, same ordering as tab 1's body_video.
|
| 2212 |
future_upload_video = gr.Video(
|
| 2213 |
-
label="Upload a short clip
|
| 2214 |
sources=["upload"], elem_id="fp-upload-dropbox", height=160,
|
| 2215 |
visible=(DEFAULT_INPUT_MODALITY == "rgb"),
|
| 2216 |
)
|
|
@@ -2259,8 +2259,8 @@ with gr.Blocks(title="Video-4M: Any-to-Any Video Modeling", theme=_THEME, css=_C
|
|
| 2259 |
lines=TEXT_OUTPUT_LINES, max_lines=TEXT_OUTPUT_LINES, elem_id="steer-text",
|
| 2260 |
)
|
| 2261 |
future_extra_tags_note = gr.Markdown(
|
| 2262 |
-
"**
|
| 2263 |
-
"
|
| 2264 |
visible=False, elem_classes=["hint-md"],
|
| 2265 |
)
|
| 2266 |
|
|
@@ -2662,7 +2662,7 @@ with gr.Blocks(title="Video-4M: Any-to-Any Video Modeling", theme=_THEME, css=_C
|
|
| 2662 |
except Exception as e:
|
| 2663 |
traceback.print_exc()
|
| 2664 |
yield _fp_frame({
|
| 2665 |
-
FP_IDX_STATUS: gr.update(value=_status_html("error
|
| 2666 |
FP_IDX_EMPTY: gr.update(value=_empty_state_html(), visible=False),
|
| 2667 |
FP_IDX_BTN: gr.update(value="Predict the future", interactive=True),
|
| 2668 |
})
|
|
@@ -2724,7 +2724,7 @@ with gr.Blocks(title="Video-4M: Any-to-Any Video Modeling", theme=_THEME, css=_C
|
|
| 2724 |
"Video-4M · predictions are 128×128 @ 4 fps · feature maps visualized via PCA-to-RGB."
|
| 2725 |
+ (
|
| 2726 |
f' · Cards marked “{CACHED_BADGE}” were generated ahead of time with this same model '
|
| 2727 |
-
"and seed
|
| 2728 |
"Any other configuration runs live."
|
| 2729 |
if _CACHE.enabled else ""
|
| 2730 |
)
|
|
|
|
| 144 |
"vjepa": "#e879f9",
|
| 145 |
}
|
| 146 |
MODALITY_TIPS = {
|
| 147 |
+
"rgb": "Raw pixel video, i.e., what a camera sees",
|
| 148 |
"caption": "Natural-language description of the clip",
|
| 149 |
"transcription": "Per-second dense description with [SEC_n] markers",
|
| 150 |
+
"dinov2": "Self-supervised semantic feature maps (layout and object identity)",
|
| 151 |
"siglip": "Vision-language feature maps aligned with text",
|
| 152 |
"depth": "Per-pixel distance from the camera",
|
| 153 |
"normal": "Per-pixel surface orientation",
|
|
|
|
| 296 |
<h1><span class="brand">Any-to-Any Video Modeling</span>: Modeling the World by Traversing Time and Modalities</h1>
|
| 297 |
<p>Video-4M is an any-to-any multimodal model that enables video modeling across different modalities. By flexibly traversing across the time and modality axes, it can predict any modality, at any point in time, given any combination of input modalities.</p>
|
| 298 |
<p>We show two ways to use this demo:<br>
|
| 299 |
+
<b>i) Any-to-any generation:</b> for a given video, you can select any modality as input, and any modality as output. Optionally you can define the intermediate modalities as the chain.<br>
|
| 300 |
+
<b>ii) Future prediction in any modality:</b> given initial observations in any modality space, predict the future in any desired modality. You can also add further controls by additionally conditioning on other modalities, such as a caption or transcription.</p>
|
| 301 |
<nav class="header-links">
|
| 302 |
<a href="https://video-4m.epfl.ch" target="_blank" rel="noopener">Website</a>
|
| 303 |
<a href="https://github.com/EPFL-VILAB/Video-4M" target="_blank" rel="noopener">GitHub</a>
|
|
|
|
| 330 |
# accordion always hashes to the precomputed configuration.
|
| 331 |
defaults = dict(zip(HYPERPARAM_SPEC, default_slider_values(default_preset_key)))
|
| 332 |
gr.Markdown(
|
| 333 |
+
"Temperature and CFG scale per modality. Defaults are auto-tuned to match your "
|
| 334 |
"current input modality; adjust to override. Sliders for modalities "
|
| 335 |
"not in your chain are ignored.",
|
| 336 |
elem_classes=["hint-md"],
|
|
|
|
| 1459 |
name = MODALITY_DISPLAY_NAMES[obs_mod]
|
| 1460 |
stem_html = (
|
| 1461 |
f' of <span class="mono" style="color:var(--text-3)">{stem}</span>'
|
| 1462 |
+
if stem else " (pick an example above)"
|
| 1463 |
)
|
| 1464 |
return (
|
| 1465 |
f'<div class="tl-title">Input to the model: <b style="color:{color}">{name}</b> · '
|
|
|
|
| 1653 |
gr.HTML(_step_label_html(
|
| 1654 |
1, "Select input to Video-4M",
|
| 1655 |
"Pick an example clip or upload your own video for RGB input, provide a caption, "
|
| 1656 |
+
"or choose another modality from a curated example. Whichever you choose "
|
| 1657 |
"becomes the input Video-4M conditions on.",
|
| 1658 |
))
|
| 1659 |
with gr.Column(elem_id="src-zone"):
|
|
|
|
| 1670 |
selected_md, rgb_select_btns = _build_example_gallery(
|
| 1671 |
_GALLERY_STEMS, selected_stem=_A2A_PRE["stem"] if _A2A_PRE else None)
|
| 1672 |
upload_video = gr.Video(
|
| 1673 |
+
label="Upload a short clip (the first ~4 s are used, center-cropped to 128×128)",
|
| 1674 |
sources=["upload"], elem_id="upload-dropbox", height=200,
|
| 1675 |
)
|
| 1676 |
|
|
|
|
| 1724 |
elem_classes=["example-preview-video", "clip-tile"],
|
| 1725 |
)
|
| 1726 |
else:
|
| 1727 |
+
gr.HTML('<div class="abs-text-tile">no preview,<br>still selectable</div>')
|
| 1728 |
b = gr.Button("Select", size="sm", elem_classes=["select-btn"])
|
| 1729 |
|
| 1730 |
def _abs_select(src_mode, upload_path, caption_text, abs_m, chain,
|
|
|
|
| 1743 |
|
| 1744 |
gr.HTML(_step_label_html(
|
| 1745 |
2, "Generation chain",
|
| 1746 |
+
"Add modalities one at a time, in the order you want them generated. "
|
| 1747 |
+
"Each one conditions on everything added before it.",
|
| 1748 |
))
|
| 1749 |
with gr.Column(elem_id="chain-zone"):
|
| 1750 |
preset_radio = gr.Radio(
|
|
|
|
| 2065 |
except Exception as e:
|
| 2066 |
traceback.print_exc()
|
| 2067 |
yield _frame({
|
| 2068 |
+
IDX_STATUS: gr.update(value=_status_html("error (see server logs)")),
|
| 2069 |
IDX_EMPTY: gr.update(value=_empty_state_html(), visible=False),
|
| 2070 |
IDX_BTN: gr.update(value="Generate", interactive=True),
|
| 2071 |
})
|
|
|
|
| 2154 |
gr.HTML(_step_label_html(
|
| 2155 |
1, "Select observation",
|
| 2156 |
"Pick an example clip (or upload your own video for RGB observation) and the "
|
| 2157 |
+
"modality you want to observe. Its future frames are then predicted.",
|
| 2158 |
))
|
| 2159 |
_future_stems = _FUTURE_STEMS
|
| 2160 |
future_example_state = gr.State(_FP_PRE["stem"] if _FP_PRE else None)
|
|
|
|
| 2188 |
elem_classes=["example-preview-video", "clip-tile"],
|
| 2189 |
)
|
| 2190 |
else:
|
| 2191 |
+
gr.HTML('<div class="abs-text-tile">no preview,<br>still selectable</div>')
|
| 2192 |
b = gr.Button("Select", size="sm", elem_classes=["select-btn"])
|
| 2193 |
b.click(None, js=_sel_busy_js("fp-selected"))
|
| 2194 |
b.click(None, js=_sel_busy_js(FP_RESULTS_HOST, size=RESULTS_SPINNER_SIZE))
|
|
|
|
| 2210 |
# same precedence rule as tab 1's upload_video. Placed
|
| 2211 |
# after the gallery, same ordering as tab 1's body_video.
|
| 2212 |
future_upload_video = gr.Video(
|
| 2213 |
+
label="Upload a short clip (the first ~4 s are used, center-cropped to 128×128)",
|
| 2214 |
sources=["upload"], elem_id="fp-upload-dropbox", height=160,
|
| 2215 |
visible=(DEFAULT_INPUT_MODALITY == "rgb"),
|
| 2216 |
)
|
|
|
|
| 2259 |
lines=TEXT_OUTPUT_LINES, max_lines=TEXT_OUTPUT_LINES, elem_id="steer-text",
|
| 2260 |
)
|
| 2261 |
future_extra_tags_note = gr.Markdown(
|
| 2262 |
+
"**Please keep the `[SEC_1]`-`[SEC_4]`/`[EOS]` tags**, as they are required by the "
|
| 2263 |
+
"model. You can freely edit the text between and around them.",
|
| 2264 |
visible=False, elem_classes=["hint-md"],
|
| 2265 |
)
|
| 2266 |
|
|
|
|
| 2662 |
except Exception as e:
|
| 2663 |
traceback.print_exc()
|
| 2664 |
yield _fp_frame({
|
| 2665 |
+
FP_IDX_STATUS: gr.update(value=_status_html("error (see server logs)")),
|
| 2666 |
FP_IDX_EMPTY: gr.update(value=_empty_state_html(), visible=False),
|
| 2667 |
FP_IDX_BTN: gr.update(value="Predict the future", interactive=True),
|
| 2668 |
})
|
|
|
|
| 2724 |
"Video-4M · predictions are 128×128 @ 4 fps · feature maps visualized via PCA-to-RGB."
|
| 2725 |
+ (
|
| 2726 |
f' · Cards marked “{CACHED_BADGE}” were generated ahead of time with this same model '
|
| 2727 |
+
"and seed. GPU time here is limited, so the default configuration is precomputed. "
|
| 2728 |
"Any other configuration runs live."
|
| 2729 |
if _CACHE.enabled else ""
|
| 2730 |
)
|