prompt-samples

app.py

@@ -581,14 +581,19 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
     .prose .ranking-table td {
         padding: 8px 10px !important;
     }
+    .ranking-table,
+    .prose .ranking-table {
+        --rank-col-width: 3.25rem;
+    }
     .ranking-table .rank,
     .prose .ranking-table .rank,
     .ranking-table th.rank {
         position: sticky !important;
         left: 0 !important;
-        width: 2.4rem;
-        min-width: 2.4rem;
-        box-shadow: 6px 0 8px -6px rgba(0, 0, 0, 0.45);
+        width: var(--rank-col-width);
+        min-width: var(--rank-col-width);
+        max-width: var(--rank-col-width);
+        box-shadow: none;
     }
     .ranking-table .model-cell,
     .prose .ranking-table .model-cell {
@@ -598,6 +603,7 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
         min-width: 140px;
         max-width: none;
         background: transparent !important;
+        box-shadow: none !important;
     }
     .ranking-table th.model-cell,
     .prose .ranking-table th.model-cell {
@@ -608,6 +614,7 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
         min-width: 140px;
         max-width: none;
         background: var(--pruna-bg-header) !important;
+        box-shadow: 0 1px 0 var(--pruna-hairline) !important;
     }
     .ranking-table tbody tr:hover .model-cell {
         background: var(--pruna-table-hover) !important;
@@ -1107,11 +1114,11 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
     max-height: min(70vh, 720px);
     overflow-x: auto;
     overflow-y: auto;
-    -webkit-overflow-scrolling: touch;
-    overscroll-behavior-x: contain;
+    overscroll-behavior: none;
 }
 .ranking-table,
 .prose .ranking-table {
+    --rank-col-width: 4.25rem;
     width: 100%;
     margin: 0 !important;
     overflow: visible;
@@ -1187,10 +1194,14 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
 .prose .ranking-table .rank {
     position: sticky;
     left: 0;
-    z-index: 1;
+    z-index: 2;
     box-sizing: border-box;
-    width: 3.25rem;
-    min-width: 3.25rem;
+    width: var(--rank-col-width);
+    min-width: var(--rank-col-width);
+    max-width: var(--rank-col-width);
+    padding-left: 0.5rem !important;
+    padding-right: 0.5rem !important;
+    overflow: hidden;
     color: var(--pruna-lavender) !important;
     font-weight: 700 !important;
     text-align: center;
@@ -1207,18 +1218,21 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
 .ranking-table .model-cell,
 .prose .ranking-table .model-cell {
     position: sticky;
-    left: 3.25rem;
-    z-index: 1;
+    left: var(--rank-col-width);
+    z-index: 2;
+    box-sizing: border-box;
     min-width: 180px;
     max-width: 260px;
     background: var(--pruna-table-sticky) !important;
+    box-shadow: 8px 0 10px -8px rgba(0, 0, 0, 0.35) !important;
 }
 .ranking-table th.model-cell,
 .prose .ranking-table th.model-cell {
     top: 0;
-    left: 3.25rem;
+    left: var(--rank-col-width);
     z-index: 5;
     background: var(--pruna-bg-header) !important;
+    box-shadow: 0 1px 0 var(--pruna-hairline), 8px 0 10px -8px rgba(0, 0, 0, 0.35) !important;
 }
 .ranking-table tbody tr:hover .rank,
 .ranking-table tbody tr:hover .model-cell {
@@ -1769,6 +1783,9 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
     max-width: 100% !important;
 }
 .compare-cell { min-width: 0; }
+.compare-cell.compare-source .compare-model-label {
+    color: var(--pruna-text-muted);
+}
 .compare-model-label {
     margin-bottom: 6px;
     color: var(--pruna-lavender);
@@ -1776,15 +1793,23 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
     font-weight: 700;
     word-break: break-word;
 }
-.compare-cell img {
+.compare-cell img,
+.compare-cell video {
     display: block;
     width: 100%;
-    aspect-ratio: 1 / 1;
-    object-fit: cover;
     border-radius: 12px;
     border: 1px solid var(--pruna-border);
     background: var(--pruna-bg-elevated);
 }
+.compare-cell img {
+    aspect-ratio: 1 / 1;
+    object-fit: cover;
+}
+.compare-cell video {
+    aspect-ratio: 16 / 9;
+    max-height: 360px;
+    object-fit: contain;
+}
 .compare-empty,
 .pareto-note-copy {
     margin: 0;
@@ -2143,14 +2168,20 @@ def load_sample_comparison_data(folder):
         return None
 
     prompts = {}
+    source_videos = {}
     with prompts_path.open() as handle:
         for line in handle:
             if not line.strip():
                 continue
             row = json.loads(line)
-            prompts[row["prompt_id"]] = row.get("text", "")
+            prompt_id = row["prompt_id"]
+            prompts[prompt_id] = row.get("text", "")
+            source_video = row.get("source_video")
+            if source_video:
+                source_videos[prompt_id] = source_video
 
     images = defaultdict(dict)
+    kinds = set()
     with generations_path.open() as handle:
         for line in handle:
             if not line.strip():
@@ -2158,20 +2189,33 @@ def load_sample_comparison_data(folder):
             row = json.loads(line)
             model_id = row["model_id"]
             prompt_id = row["prompt_id"]
-            image_url = row.get("image")
-            if model_id and prompt_id and image_url:
-                images[model_id][prompt_id] = image_url
+            media_url = row.get("image") or row.get("video")
+            if row.get("video"):
+                kinds.add("video")
+            elif row.get("image"):
+                kinds.add("image")
+            if model_id and prompt_id and media_url:
+                images[model_id][prompt_id] = media_url
+            if prompt_id and prompt_id not in source_videos:
+                params = row.get("params") or {}
+                input_video = params.get("input_video") or row.get("input_video")
+                if input_video:
+                    source_videos[prompt_id] = input_video
 
     models = sorted(images)
     if not models or not prompts:
         return None
 
-    return {
+    loaded = {
         "prompts": prompts,
         "images": {model: dict(prompt_map) for model, prompt_map in images.items()},
         "models": models,
         "prompt_ids": sorted(prompts),
+        "kind": "video" if "video" in kinds else "image",
     }
+    if source_videos:
+        loaded["source_videos"] = source_videos
+    return loaded
 
 
 def _as_numeric(df, columns):
@@ -2301,6 +2345,60 @@ def load_qwen_combined_dataframe(path):
     return df.reset_index(drop=True)
 
 
+def load_video_editing_dataframe(path):
+    """Load the video-to-video editing leaderboard."""
+    df = pd.read_csv(path, na_values=["N/A", "n/a", ""])
+    df = df.rename(
+        columns={
+            "display_name": "Model",
+            "elo": "Datapoint Elo",
+            "min_generation_s": "Min Generation Time (s)",
+            "median_generation_s": "Median Generation Time (s)",
+            "p20_generation_s": "P20 Generation Time (s)",
+            "generation_s_per_output_video_s": "Time / Output Video Second (s)",
+            "predict_time_s_per_output_video_s": "Predict Time / Output Video Second (s)",
+            "model_execution_time_s_per_output_video_s": (
+                "Execution Time / Output Video Second (s)"
+            ),
+            "price": "Price / Second of Video (USD)",
+        }
+    )
+    df = df.drop(columns=["wandb_run_ids", "n_generations"], errors="ignore")
+    df["Model"] = df["Model"].astype(str).str.strip()
+    df = _as_numeric(
+        df,
+        [
+            "Datapoint Elo",
+            "Min Generation Time (s)",
+            "Median Generation Time (s)",
+            "P20 Generation Time (s)",
+            "Time / Output Video Second (s)",
+            "Predict Time / Output Video Second (s)",
+            "Execution Time / Output Video Second (s)",
+            "Price / Second of Video (USD)",
+        ],
+    )
+    end_to_end = df.get("Time / Output Video Second (s)")
+    execution = df.get("Execution Time / Output Video Second (s)")
+    if end_to_end is not None:
+        if "model_id" in df.columns:
+            is_ours = df["model_id"].astype(str).str.lower().str.startswith(
+                "p_video_edit"
+            )
+        else:
+            is_ours = df["Model"].astype(str).str.casefold().str.startswith(
+                "p-video-edit"
+            )
+        if execution is None:
+            pareto_time = end_to_end
+        else:
+            ours_time = execution.where(execution.notna(), end_to_end)
+            pareto_time = end_to_end.where(~is_ours, ours_time)
+        df["Pareto Time / Output Video Second (s)"] = pareto_time
+    df = df.drop(columns=["model_id"], errors="ignore")
+    return df.reset_index(drop=True)
+
+
 df = load_oneig_dataframe(oneig_path)
 
 oneig_metric_columns = [
@@ -2348,6 +2446,10 @@ qwen_combined_dir = _resolve_data_path(
     data_dir / "qwen_image_bench_combined",
     space_root.parent / "qwen_image_bench_combined",
 )
+video_combined_dir = _resolve_data_path(
+    data_dir / "video_editing_combined",
+    space_root.parent / "video_editing_combined",
+)
 qwen_path = _resolve_data_path(
     data_dir / "qwen_image_bench_model_price_and_median_generation_time_10_august.csv",
     space_root.parent / "qwen_image_bench_model_price_and_median_generation_time_10_august.csv",
@@ -2360,10 +2462,15 @@ arena_path = _resolve_data_path(
     data_dir / "arena_ai_text_to_image_leaderboard.csv",
     space_root.parent / "arena_ai_text_to_image_leaderboard.csv",
 )
+video_path = _resolve_data_path(
+    data_dir / "video-editing-leaderboard.csv",
+    space_root.parent / "video-editing-leaderboard.csv",
+)
 
 qwen_df = load_qwen_combined_dataframe(qwen_path)
 aa_df = load_artificial_analysis_dataframe(aa_path)
 arena_df = load_arena_ai_dataframe(arena_path)
+video_df = load_video_editing_dataframe(video_path)
 qwen_display_columns = [
     col
     for col in [
@@ -2395,9 +2502,22 @@ arena_display_columns = [
     ]
     if col in arena_df.columns
 ]
+video_display_columns = [
+    col
+    for col in [
+        "Model",
+        "Datapoint Elo",
+        "Time / Output Video Second (s)",
+        "Median Generation Time (s)",
+        "Min Generation Time (s)",
+        "Price / Second of Video (USD)",
+    ]
+    if col in video_df.columns
+]
 
 oneig_samples = load_sample_comparison_data(oneig_combined_dir)
 qwen_samples = load_sample_comparison_data(qwen_combined_dir)
+video_samples = load_sample_comparison_data(video_combined_dir)
 
 metrics = [
     {"id": "datapoint_elo", "column": "Datapoint Elo"},
@@ -2456,11 +2576,27 @@ arena_metric_ids = _metric_ids_for(
         "arena_text",
     ],
 )
+video_metric_ids = _metric_ids_for(video_df, ["datapoint_elo"])
 
 datasets = [
+    {
+        "id": "video_editing",
+        "name": "Pruna Internal Video-Edit Benchmark",
+        "modality": "video_to_video",
+        "data": video_df,
+        "columns": video_display_columns,
+        "metric_ids": video_metric_ids,
+        "note": (
+            "Datapoint Elo from pairwise video-edit preference. Price is USD "
+            "per second of output video. Generation time per second of video "
+            "is end-to-end wall time to produce one second of output."
+        ),
+        "samples": video_samples,
+    },
     {
         "id": "qwen",
         "name": "Qwen Image Dataset",
+        "modality": "text_to_image",
         "data": qwen_df,
         "columns": qwen_display_columns,
         "metric_ids": qwen_metric_ids,
@@ -2470,6 +2606,7 @@ datasets = [
     {
         "id": "oneig",
         "name": "OneIG Alignment Dataset",
+        "modality": "text_to_image",
         "data": oneig_df,
         "columns": oneig_display_columns,
         "metric_ids": oneig_metric_ids,
@@ -2482,6 +2619,7 @@ datasets = [
     {
         "id": "artificial_analysis",
         "name": "Artificial Analysis Dataset",
+        "modality": "text_to_image",
         "data": aa_df,
         "columns": aa_display_columns,
         "metric_ids": aa_metric_ids,
@@ -2491,6 +2629,7 @@ datasets = [
     {
         "id": "arena_ai",
         "name": "Arena AI Dataset",
+        "modality": "text_to_image",
         "data": arena_df,
         "columns": arena_display_columns,
         "metric_ids": arena_metric_ids,
@@ -2501,8 +2640,11 @@ datasets = [
 datasets = [dataset for dataset in datasets if dataset["metric_ids"]]
 
 DEFAULT_DATASET_ID = next(
-    (dataset["id"] for dataset in datasets if dataset["id"] == "qwen"),
-    datasets[0]["id"] if datasets else None,
+    (dataset["id"] for dataset in datasets if dataset["id"] == "video_editing"),
+    next(
+        (dataset["id"] for dataset in datasets if dataset["id"] == "qwen"),
+        datasets[0]["id"] if datasets else None,
+    ),
 )
 DEFAULT_METRIC_ID = None
 

data/video-editing-leaderboard.csv

@@ -0,0 +1,12 @@
+model_id,display_name,elo,wandb_run_ids,n_generations,min_generation_s,median_generation_s,p20_generation_s,generation_s_per_output_video_s,predict_time_s_per_output_video_s,model_execution_time_s_per_output_video_s,price
+gemini_omni_flash_edit__fal,Gemini Omni Flash Edit,1054,3iuctwj9,56,32.574746752001374,58.2327566820004,50.44402900500063,13.78,,,0.13
+grok_imagine_video__replicate,Grok Imagine Video,974,q0ogvekx,59,31.640928319000523,43.75649321700257,32.88782280539963,11.04,,,0.05
+happyhorse_1_0__wavespeed,HappyHorse 1.0,1048,k4zj5wqk,71,106.5824136010051,201.2431439649954,141.46595527700265,46.73,,,0.14
+ltx_2_3_quality_reference_video_to_video__fal,LTX 2.3 Video Edit,911,p19ks8ad,73,60.768441981999786,73.62109239100027,70.20137865180223,15.83,,,0.054
+lucy_edit_pro__fal,Lucy Edit Pro,879,na6bz9xx,72,118.3459930579993,136.44442766549764,124.13885582720104,27.18,,,0.15
+minimax_h3_reference_to_video__fal,MiniMax H3 Reference-to-Video,1060,639xmakz,63,180.8385945170012,308.29198316100155,248.2394383729996,58.31,,,0.06
+p_video_edit_preview__replicate_final,P-Video-Edit,1000,m8irpsa6,73,31.41135125700021,86.86665409700072,57.21095684959946,23.18,17.99,12.06,0.045
+p_video_edit_preview__replicate_final__draft,P-Video-Edit Draft,994,6f88e6mx,73,23.04809326099712,42.9797806409988,33.19193872759861,11.48,11.28,4.46,0.025
+seedance_2_5_video_edit_turbo__wavespeed,Seedance 2.5 Video Edit Turbo,1063,2nr274vp,68,142.49452170499717,293.7401115540015,223.11569286320045,59.93,,,0.24
+wan_2_7_video_edit__wavespeed,Wan 2.7 Video Edit,1055,9lq803be,66,134.3744560209998,313.957433804002,219.2491260079987,68.04,,,0.2
+

data/video_editing_combined/generations.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ec4ea0a431484e65f30990c8cc4292b00c665be96db587775426da1c84a4ab8
+size 762227

data/video_editing_combined/prompts.jsonl

@@ -0,0 +1,78 @@
+{"prompt_id": "video_edit_internal__advertising__p0000", "text": "Replace the white bottle with an orange sunscreen bottle.", "dataset": "video_edit_internal", "category": "advertising", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/advertising/4620329ad65820ce.mp4"}
+{"prompt_id": "video_edit_internal__advertising__p0001", "text": "Replace the woman with an asian woman riding on a donkey through a chinese small town.", "dataset": "video_edit_internal", "category": "advertising", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/advertising/44db5689846be901.mp4"}
+{"prompt_id": "video_edit_internal__advertising__p0002", "text": "Place the couple on a snowy mountain next to a mountain hut.", "dataset": "video_edit_internal", "category": "advertising", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/advertising/481db4a047688bbe.mp4"}
+{"prompt_id": "video_edit_internal__advertising__p0003", "text": "Turn this into a scene in the summer with birds flying in the sky and butterflies in the foreground.", "dataset": "video_edit_internal", "category": "advertising", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/advertising/32b9f56763c56482.mp4"}
+{"prompt_id": "video_edit_internal__advertising__p0004", "text": "Replace the robot arm with a dancing hamster on a small table.", "dataset": "video_edit_internal", "category": "advertising", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/advertising/22d02ae2df7f9045.mp4"}
+{"prompt_id": "video_edit_internal__anonymization__p0000", "text": "Anonymize the womans face", "dataset": "video_edit_internal", "category": "anonymization", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/anonymization/7c410085f2fd939d.mp4"}
+{"prompt_id": "video_edit_internal__anonymization__p0001", "text": "Anonymize the mans face", "dataset": "video_edit_internal", "category": "anonymization", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/anonymization/7b495805c0c270ea.mp4"}
+{"prompt_id": "video_edit_internal__anonymization__p0002", "text": "Anonymize the mans face", "dataset": "video_edit_internal", "category": "anonymization", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/anonymization/736b6fb504bd32be.mp4"}
+{"prompt_id": "video_edit_internal__anonymization__p0003", "text": "Anonymize the girls face", "dataset": "video_edit_internal", "category": "anonymization", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/anonymization/2c82413d39579cd5.mp4"}
+{"prompt_id": "video_edit_internal__artificial_analysis__p0000", "text": "Change the rainforest setting to a neon-lit urban alley at night with steam from vents and wet reflective asphalt, and move into an ariel shot of the character after the initial camera movement to focus on the character", "dataset": "video_edit_internal", "category": "artificial_analysis", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/artificial_analysis/e2f6121269a206d3.mp4"}
+{"prompt_id": "video_edit_internal__artificial_analysis__p0001", "text": "Replace the magenta hover-car with a chrome-blue car, keeping the drift and neon light trails.", "dataset": "video_edit_internal", "category": "artificial_analysis", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/artificial_analysis/55d1baa7f3f71b26.mp4"}
+{"prompt_id": "video_edit_internal__artificial_analysis__p0002", "text": "Make the footage look like it was shot on a 1970s film camera, with grainy film texture, faded warm colors, slight softness, and slightly darker corners around the edges.", "dataset": "video_edit_internal", "category": "artificial_analysis", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/artificial_analysis/fd1ab436de020152.mp4"}
+{"prompt_id": "video_edit_internal__artificial_analysis__p0003", "text": "A person appears at the top of the waterfall, leaps into the lake below, and disappears into the misty water beneath the falls.", "dataset": "video_edit_internal", "category": "artificial_analysis", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/artificial_analysis/7ee1623303a8da68.mp4"}
+{"prompt_id": "video_edit_internal__artificial_analysis__p0004", "text": "A fluffy orange cat leaps gracefully from the floor onto the couch, paws sinking into the soft cushions. It circles once in place, tail swaying gently, before curling up tightly on one of the cushions and settling in comfortably.", "dataset": "video_edit_internal", "category": "artificial_analysis", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/artificial_analysis/9a73c608b343bbbe.mp4"}
+{"prompt_id": "video_edit_internal__camera_editing__p0000", "text": "Zoom in on the man's face to show his focused expression", "dataset": "video_edit_internal", "category": "camera_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/camera_editing/0336e64b0594bd7a.mp4"}
+{"prompt_id": "video_edit_internal__camera_editing__p0001", "text": "Perform an arc shot around the tram as it arrives at the station", "dataset": "video_edit_internal", "category": "camera_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/camera_editing/204aa93703da117b.mp4"}
+{"prompt_id": "video_edit_internal__camera_editing__p0002", "text": "Change the view to a high angle.", "dataset": "video_edit_internal", "category": "camera_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/camera_editing/c2ccb5351be5c718.mp4"}
+{"prompt_id": "video_edit_internal__camera_editing__p0003", "text": "Change the view to a high angle.", "dataset": "video_edit_internal", "category": "camera_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/camera_editing/b725aaded02d65e9.mp4"}
+{"prompt_id": "video_edit_internal__camera_editing__p0004", "text": "Gradually move the camera away from the doctor", "dataset": "video_edit_internal", "category": "camera_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/camera_editing/33f24f4d083df743.mp4"}
+{"prompt_id": "video_edit_internal__design_arena__p0000", "text": "Create a transition of the video of the product whihc is the jeans on the lady model with an atitiude that goes into a zoom out camera that she is in a party .Focus on the vibes", "dataset": "video_edit_internal", "category": "design_arena", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/design_arena/b0b4642272a481b1.mp4"}
+{"prompt_id": "video_edit_internal__design_arena__p0001", "text": "add ethnic urban women dancing at the bottem and at the bar wearing DMI Tshirts", "dataset": "video_edit_internal", "category": "design_arena", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/design_arena/0f730f98f3c51356.mp4"}
+{"prompt_id": "video_edit_internal__design_arena__p0002", "text": "Generate a commercial of this dog drinking beer at an electronic music party in a world where dogs and humans are mixed together.", "dataset": "video_edit_internal", "category": "design_arena", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/design_arena/575fc4b9b6d7b3ab.mp4"}
+{"prompt_id": "video_edit_internal__design_arena__p0003", "text": "everything is the same excpet the winter season, snowing", "dataset": "video_edit_internal", "category": "design_arena", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/design_arena/8674fd022867a736.mp4"}
+{"prompt_id": "video_edit_internal__design_arena__p0004", "text": "A cinematic character introduction of Jason, framed in a medium close-up with subtle camera movement, confident body language, and expressive facial detail. Moody, high-contrast lighting with a cool-toned color palette, shallow depth of field, and a slow dramatic reveal that builds intrigue over a few seconds. I want a video in a 9:16 aspect ratio for tiktok. remove all text and have him in a mid day campus setting", "dataset": "video_edit_internal", "category": "design_arena", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/design_arena/91873bbafe4173da.mp4"}
+{"prompt_id": "video_edit_internal__e_commerce__p0000", "text": "Remove the black blazer.", "dataset": "video_edit_internal", "category": "e_commerce", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/e_commerce/b89faaa5080c31db.mp4"}
+{"prompt_id": "video_edit_internal__e_commerce__p0001", "text": "Replace her skirt with a jeans skirt.", "dataset": "video_edit_internal", "category": "e_commerce", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/e_commerce/c34a33d07e843804.mp4"}
+{"prompt_id": "video_edit_internal__e_commerce__p0002", "text": "Replace the black leggings and the black shirt with a white dress.", "dataset": "video_edit_internal", "category": "e_commerce", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/e_commerce/489d2439cbaeed79.mp4"}
+{"prompt_id": "video_edit_internal__e_commerce__p0003", "text": "Replace the white woman with a lebanese-looking woman.", "dataset": "video_edit_internal", "category": "e_commerce", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/e_commerce/0ad8fb773f303a11.mp4"}
+{"prompt_id": "video_edit_internal__e_commerce__p0004", "text": "Remove all items on the table and place an eyeshadow pallete on the table next to the woman.", "dataset": "video_edit_internal", "category": "e_commerce", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/e_commerce/f28f3aa12b1ec220.mp4"}
+{"prompt_id": "video_edit_internal__lighting__p0000", "text": "After the sun sets behind the mountains, the scene transitions into nighttime.", "dataset": "video_edit_internal", "category": "lighting", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/lighting/229c165e0ac97daf.mp4"}
+{"prompt_id": "video_edit_internal__lighting__p0001", "text": "Change the weather to a dazzling starry night.", "dataset": "video_edit_internal", "category": "lighting", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/lighting/f42c41ab2f3adc26.mp4"}
+{"prompt_id": "video_edit_internal__lighting__p0002", "text": "Change the weather to a thunderstorm with heavy rain.", "dataset": "video_edit_internal", "category": "lighting", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/lighting/daa36567634ca6be.mp4"}
+{"prompt_id": "video_edit_internal__lighting__p0003", "text": "Change the weather to a torrential downpour.", "dataset": "video_edit_internal", "category": "lighting", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/lighting/6817a4142b69d602.mp4"}
+{"prompt_id": "video_edit_internal__lighting__p0004", "text": "Change the weather to a torrential downpour.", "dataset": "video_edit_internal", "category": "lighting", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/lighting/ac01488a685a1096.mp4"}
+{"prompt_id": "video_edit_internal__long__p0000", "text": "Change the weather to a dazzling starry night.", "dataset": "video_edit_internal", "category": "long", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/long/e9af2a7039e43f16.mp4"}
+{"prompt_id": "video_edit_internal__long__p0001", "text": "Adjust the color of book to blue", "dataset": "video_edit_internal", "category": "long", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/long/eab696d1555be316.mp4"}
+{"prompt_id": "video_edit_internal__long__p0002", "text": "Make the young woman turn into sand and blow away.", "dataset": "video_edit_internal", "category": "long", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/long/d18378f1d7e4ca0e.mp4"}
+{"prompt_id": "video_edit_internal__long__p0003", "text": "Make the bird flap its wings", "dataset": "video_edit_internal", "category": "long", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/long/3835b37b43f5e5bf.mp4"}
+{"prompt_id": "video_edit_internal__long__p0004", "text": "Transform the video into a ukiyo-e style", "dataset": "video_edit_internal", "category": "long", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/long/09fe5bb4b5304b66.mp4"}
+{"prompt_id": "video_edit_internal__movie_concept_art__p0000", "text": "Add more blood and wounds to the mans face.", "dataset": "video_edit_internal", "category": "movie_concept_art", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/movie_concept_art/1d1a99167475720b.mp4"}
+{"prompt_id": "video_edit_internal__movie_concept_art__p0001", "text": "Make the scene less dark.", "dataset": "video_edit_internal", "category": "movie_concept_art", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/movie_concept_art/10cb44d8c0bf4116.mp4"}
+{"prompt_id": "video_edit_internal__movie_concept_art__p0002", "text": "Replace the female warrior with a male warrior with ginger hair.", "dataset": "video_edit_internal", "category": "movie_concept_art", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/movie_concept_art/9e45454390c20b8a.mp4"}
+{"prompt_id": "video_edit_internal__movie_concept_art__p0003", "text": "Turn the man's hand into a robotic hand.", "dataset": "video_edit_internal", "category": "movie_concept_art", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/movie_concept_art/99977e7a91a950b4.mp4"}
+{"prompt_id": "video_edit_internal__real_estate__p0000", "text": "Replace the interior with a gold-black interior heavy luxury style.", "dataset": "video_edit_internal", "category": "real_estate", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/real_estate/869064ab06bb5daa.mp4"}
+{"prompt_id": "video_edit_internal__real_estate__p0001", "text": "Replace the grey wallpaper with a beige painted wall and turn the grey curtains a dark brown.", "dataset": "video_edit_internal", "category": "real_estate", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/real_estate/1c2f358bfe9b8299.mp4"}
+{"prompt_id": "video_edit_internal__real_estate__p0002", "text": "Remove all decoration from the walls and keep only the furniture.", "dataset": "video_edit_internal", "category": "real_estate", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/real_estate/2058bcf38389f8fe.mp4"}
+{"prompt_id": "video_edit_internal__real_estate__p0003", "text": "Exchange the wooden floor for a marble floor.", "dataset": "video_edit_internal", "category": "real_estate", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/real_estate/6a4c09a23dda57d9.mp4"}
+{"prompt_id": "video_edit_internal__real_estate__p0004", "text": "Replace the bedframe with a modern wooden bed.", "dataset": "video_edit_internal", "category": "real_estate", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/real_estate/3e0f6dc5df891001.mp4"}
+{"prompt_id": "video_edit_internal__style_transfer__p0000", "text": "Transform the video into a cyberpunk style", "dataset": "video_edit_internal", "category": "style_transfer", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/style_transfer/2253f6ed3674f006.mp4"}
+{"prompt_id": "video_edit_internal__style_transfer__p0001", "text": "Convert to different shades of orange", "dataset": "video_edit_internal", "category": "style_transfer", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/style_transfer/edb6a105be03bae0.mp4"}
+{"prompt_id": "video_edit_internal__style_transfer__p0002", "text": "Convert to black and white", "dataset": "video_edit_internal", "category": "style_transfer", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/style_transfer/c3dc058b3be26cbd.mp4"}
+{"prompt_id": "video_edit_internal__style_transfer__p0003", "text": "Apply Ghibli-style editing to the video", "dataset": "video_edit_internal", "category": "style_transfer", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/style_transfer/83a638223a20911c.mp4"}
+{"prompt_id": "video_edit_internal__style_transfer__p0004", "text": "Relight the scene as if it were shot during golden hour, with warm low-angle sunlight, soft shadows, and natural highlights on faces and surfaces.", "dataset": "video_edit_internal", "category": "style_transfer", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/style_transfer/daa36567634ca6be.mp4"}
+{"prompt_id": "video_edit_internal__subject_editing__p0000", "text": "Replace the rainbow colors of the logo with different shades of purple", "dataset": "video_edit_internal", "category": "subject_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/subject_editing/6358f26b46baadf5.mp4"}
+{"prompt_id": "video_edit_internal__subject_editing__p0001", "text": "Add a small dog running beside the scooter", "dataset": "video_edit_internal", "category": "subject_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/subject_editing/052c0bfd7b68fea7.mp4"}
+{"prompt_id": "video_edit_internal__subject_editing__p0002", "text": "Replace the splashing waves with a calm water surface", "dataset": "video_edit_internal", "category": "subject_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/subject_editing/c3f79578a39f415d.mp4"}
+{"prompt_id": "video_edit_internal__subject_editing__p0003", "text": "Add a violinist in the background", "dataset": "video_edit_internal", "category": "subject_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/subject_editing/a0eebf32a076e2fc.mp4"}
+{"prompt_id": "video_edit_internal__subject_editing__p0004", "text": "Add a group of people walking on the pathway", "dataset": "video_edit_internal", "category": "subject_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/subject_editing/3c2b7e3a5b284e1d.mp4"}
+{"prompt_id": "video_edit_internal__subject_motion_editing__p0000", "text": "Make the bird hop around instead of walking and foraging", "dataset": "video_edit_internal", "category": "subject_motion_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/subject_motion_editing/ac38b2a97d1a5c3a.mp4"}
+{"prompt_id": "video_edit_internal__subject_motion_editing__p0001", "text": "The male colleague is walking around to observe.", "dataset": "video_edit_internal", "category": "subject_motion_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/subject_motion_editing/8aaab7d304c5ad99.mp4"}
+{"prompt_id": "video_edit_internal__subject_motion_editing__p0002", "text": "Make the static spider-man in the mural dynamic and make him swing faster", "dataset": "video_edit_internal", "category": "subject_motion_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/subject_motion_editing/3412b39686fd5879.mp4"}
+{"prompt_id": "video_edit_internal__subject_motion_editing__p0003", "text": "Change the woman's jogging to taking off.", "dataset": "video_edit_internal", "category": "subject_motion_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/subject_motion_editing/17c4dc0514772321.mp4"}
+{"prompt_id": "video_edit_internal__subject_motion_editing__p0004", "text": "Make the knight lunging forward and the creature swiping at the knight", "dataset": "video_edit_internal", "category": "subject_motion_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/subject_motion_editing/980354550011911b.mp4"}
+{"prompt_id": "video_edit_internal__synthetic_data__p0000", "text": "Turn the scene into nighttime.", "dataset": "video_edit_internal", "category": "synthetic_data", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/synthetic_data/e3b1e603bfaa1aec.mp4"}
+{"prompt_id": "video_edit_internal__synthetic_data__p0001", "text": "Remove the crosswalk.", "dataset": "video_edit_internal", "category": "synthetic_data", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/synthetic_data/30df5fd5fe28af65.mp4"}
+{"prompt_id": "video_edit_internal__synthetic_data__p0002", "text": "Add a bicycle riding in front of the car.", "dataset": "video_edit_internal", "category": "synthetic_data", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/synthetic_data/d093491e25682b3a.mp4"}
+{"prompt_id": "video_edit_internal__synthetic_data__p0003", "text": "Turn the scene into a snowstorm scene.", "dataset": "video_edit_internal", "category": "synthetic_data", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/synthetic_data/1da5913f13362f8d.mp4"}
+{"prompt_id": "video_edit_internal__synthetic_data__p0004", "text": "Make it rain in the scene.", "dataset": "video_edit_internal", "category": "synthetic_data", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/synthetic_data/936bdb89558f5a7f.mp4"}
+{"prompt_id": "video_edit_internal__text__p0000", "text": "Add the text \"True Love\" in the foreground in pink romantic font.", "dataset": "video_edit_internal", "category": "text", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/text/c763407793ca9b0e.mp4"}
+{"prompt_id": "video_edit_internal__text__p0001", "text": "Replace any mention of \"Fanta\" with the branding \"Cola\".", "dataset": "video_edit_internal", "category": "text", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/text/7b878ea6d99834b2.mp4"}
+{"prompt_id": "video_edit_internal__text__p0002", "text": "Remove all text from the video.", "dataset": "video_edit_internal", "category": "text", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/text/76763c31b2609109.mp4"}
+{"prompt_id": "video_edit_internal__text__p0003", "text": "Replace the branding \"Royalty\" with the phrasing \"Princess\".", "dataset": "video_edit_internal", "category": "text", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/text/d221d299c9e49064.mp4"}
+{"prompt_id": "video_edit_internal__text__p0004", "text": "Remove all text from the video.", "dataset": "video_edit_internal", "category": "text", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/text/5c169901af8e4ed6.mp4"}
+{"prompt_id": "video_edit_internal__transitions__p0000", "text": "After a black screen transition, the road transforms into lush grass.", "dataset": "video_edit_internal", "category": "transitions", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/transitions/964c35b19ee1fc1f.mp4"}
+{"prompt_id": "video_edit_internal__transitions__p0001", "text": "After a wave-foam transition, the small fishing boat is eaten by a giant whale.", "dataset": "video_edit_internal", "category": "transitions", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/transitions/b8b88f43316363f3.mp4"}
+{"prompt_id": "video_edit_internal__transitions__p0002", "text": "Add a cut transition, then show a bowl of ramen topped with parsley.", "dataset": "video_edit_internal", "category": "transitions", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/transitions/c823d6c77bdc87f4.mp4"}
+{"prompt_id": "video_edit_internal__transitions__p0003", "text": "Add a smoke transition, then show the circuit board burning.", "dataset": "video_edit_internal", "category": "transitions", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/transitions/edb9927ff06c1175.mp4"}
+{"prompt_id": "video_edit_internal__transitions__p0004", "text": "After a smoke transition, the Basilica of the Sacred Heart of Paris catches fire.", "dataset": "video_edit_internal", "category": "transitions", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/transitions/77bb900383d4f370.mp4"}

model_display.py

@@ -88,6 +88,22 @@ MODEL_DISPLAY_NAMES = {
     "p_image_2_ideogram_high_2k": "P-Image-Ideogram High 2K",
     "P-Image-Ideogram (High)": "P-Image-Ideogram High",
     "p_image_2_ideogram_very_high_1k": "P-Image-Ideogram Very High 1K",
+    # P-Video-Edit
+    "P-Video-Edit": "P-Video-Edit",
+    "P-Video-Edit Draft": "P-Video-Edit Draft",
+    "P-Video Edit Final": "P-Video-Edit",
+    "P-Video Edit Final (draft)": "P-Video-Edit Draft",
+    "p_video_edit_preview__replicate_final": "P-Video-Edit",
+    "p_video_edit_preview__replicate_final__draft": "P-Video-Edit Draft",
+    # Video-to-video leaderboard
+    "gemini_omni_flash_edit__fal": "Gemini Omni Flash Edit",
+    "grok_imagine_video__replicate": "Grok Imagine Video",
+    "happyhorse_1_0__wavespeed": "HappyHorse 1.0",
+    "ltx_2_3_quality_reference_video_to_video__fal": "LTX 2.3 Video Edit",
+    "lucy_edit_pro__fal": "Lucy Edit Pro",
+    "minimax_h3_reference_to_video__fal": "MiniMax H3 Reference-to-Video",
+    "seedance_2_5_video_edit_turbo__wavespeed": "Seedance 2.5 Video Edit Turbo",
+    "wan_2_7_video_edit__wavespeed": "Wan 2.7 Video Edit",
     # Others overlapping P-Bench
     "z_image": "Z-Image",
     "glm_image": "GLM-Image",
@@ -210,6 +226,6 @@ def display_model_name(model_id) -> str:
     if raw in MODEL_DISPLAY_NAMES:
         return MODEL_DISPLAY_NAMES[raw]
     # Already a human label (spaces / punctuation) — keep as-is.
-    if re.search(r"[\s.\[\]()]", raw):
+    if re.search(r"[\s.\[\]()-]", raw):
         return raw
     return _prettify_snake_case(raw)

ui.py

@@ -25,19 +25,41 @@ MAX_COMPARE_PROMPTS = 8
 MAX_PARETO_METRICS = 8
 _PARETO_SLOT_COUNT = 1 + MAX_PARETO_METRICS * 8
 _PARETO_PRICE_COLUMN = "Price / Image (USD)"
+_PARETO_VIDEO_PRICE_COLUMN = "Price / Second of Video (USD)"
+_PARETO_PRICE_COLUMNS = (_PARETO_PRICE_COLUMN, _PARETO_VIDEO_PRICE_COLUMN)
 _PARETO_TIME_COLUMN = "Min Generation Time (s)"
+_PARETO_VIDEO_TIME_COLUMN = "Pareto Time / Output Video Second (s)"
+_PARETO_TIME_COLUMNS = (_PARETO_VIDEO_TIME_COLUMN, _PARETO_TIME_COLUMN)
+_PARETO_PRICE_TITLES = {
+    _PARETO_PRICE_COLUMN: "Price per image (USD)",
+    _PARETO_VIDEO_PRICE_COLUMN: "Price per second of video (USD)",
+}
+_PARETO_TIME_TITLES = {
+    _PARETO_TIME_COLUMN: "Min generation time (s)",
+    _PARETO_VIDEO_TIME_COLUMN: "Generation time per second of video",
+}
 _PARETO_SCALE_CHOICES = [
     ("Log", "Logarithmic"),
     ("Linear", "Linear"),
 ]
 _PARETO_SCALE_VALUES = {value for _, value in _PARETO_SCALE_CHOICES}
 _PARETO_SCALE_DEFAULT = "Logarithmic"
+_PARETO_PRUNA_COLOR = "#c084fc"
+_PARETO_OTHER_COLOR = "#9aa3b5"
+_PARETO_FRONTIER_OUTLINE = "#3fa87e"
 
 TAB_LEADERBOARDS = "leaderboards"
 TAB_PARETO = "pareto"
 TAB_SAMPLES = "samples"
 TAB_ABOUT = "about"
 
+MODALITY_VIDEO_TO_VIDEO = "video_to_video"
+MODALITY_TEXT_TO_IMAGE = "text_to_image"
+MODALITY_CHOICES = [
+    ("Video to Video", MODALITY_VIDEO_TO_VIDEO),
+    ("Text to Image", MODALITY_TEXT_TO_IMAGE),
+]
+
 _MODEL_CHOICES_CACHE = {}
 _VIEW_EVENTS = {
     "show_progress": "hidden",
@@ -49,21 +71,24 @@ _VIEW_EVENTS = {
 ABOUT_OVERVIEW_CONTENT = """
 # About P-Bench
 
-P-Bench compares **text-to-image models**, including optimized or accelerated
-endpoints, on **quality, speed, and price**. Each view is a **dataset** scored
-with a **metric**, written as `Dataset | Metric`. There is no single score
-across P-Bench.
+P-Bench compares **text-to-image** and **video-to-video** models, including
+optimized or accelerated endpoints, on **quality, speed, and price**. Each
+view is a **dataset** scored with a **metric**, written as `Dataset | Metric`.
+There is no single score across P-Bench.
 
 ## How to read it
 
-1. Pick a **dataset** and a **metric**.
+1. Pick a **type** (Video to Video or Text to Image), then a **dataset**
+   and a **metric**.
 2. **Leaderboards**: ranked by that metric. Price and generation time sit in
    the same table when the source publishes them.
 3. **Pareto plots**: mark models that are not beaten on both higher score
    and lower price (or time). Only datasets with price or generation time
    can open this tab (not Arena AI).
 4. **Samples**: the same prompts, side by side. Only for datasets we
-   generated (Qwen Image Dataset and OneIG Alignment Dataset).
+   generated (Qwen Image Dataset, OneIG Alignment Dataset, and the
+   Pruna Internal Video-Edit Benchmark). Video samples show the source
+   clip first, then each model's edit.
 
 ## How a score is made
 
@@ -82,6 +107,17 @@ prompt suites, so samples are not shown.
 
 ## Current datasets
 
+### Pruna Internal Video-Edit Benchmark
+Pruna's internal video-to-video editing benchmark, collected by our
+research engineers. It combines prompts from public video-editing
+benchmarks with use-case examples we gathered for advertisement,
+e-commerce, real estate, concept art, and similar work. The suite also
+covers camera-angle and movement changes, lighting, and text in video
+(altering, adding, or removing it). Quality is Datapoint Elo from
+pairwise preference. Price is USD per second of output video;
+generation time is wall time per second of output video. Samples show
+the source clip beside each model's edit.
+
 ### Qwen Image Dataset
 100 prompts from the 1,000-prompt Qwen Image Bench set, sampled for coverage
 across its fine-grained (L3) categories. Metrics include Datapoint Elo,
@@ -124,12 +160,14 @@ ABOUT_DETAILS_CONTENT = """
 - **Arena Elo**: Elo published by Arena AI on their own dataset, plus
   category Elos (branding, 3D, cartoon/anime, photorealistic, art, portraits,
   text rendering).
-- **Generation time**: median and minimum generation time in seconds, as
-  reported in the evaluation table. This is not a p95, and we do not state
-  warm vs cold or concurrent load. Not available for Arena AI.
-- **Price**: USD per image in the evaluation table. We do not state list
-  price vs amount paid, or whether failed generations are included. Not
-  available for Arena AI.
+- **Generation time**: median and minimum generation time in seconds for
+  images, as reported in the evaluation table. For video, generation time
+  per second of output video is the more informative figure (end-to-end
+  wall time). This is not a p95, and we do not state warm vs cold or
+  concurrent load. Not available for Arena AI.
+- **Price**: USD per image for text-to-image, or USD per second of output
+  video for video-to-video. We do not state list price vs amount paid, or
+  whether failed generations are included. Not available for Arena AI.
 
 Scores from different datasets or metrics are **not interchangeable**. A high
 OneIG alignment score is not the same quantity as a Datapoint Elo. Compare
@@ -143,14 +181,18 @@ models *within* a Dataset | Metric view.
 - **Prompt counts:** OneIG Alignment uses 100 anime, 100 human, and 99 object
   prompts (299 total). Qwen Image Dataset uses 100 prompts sampled from the
   1,000-prompt pool for roughly even coverage of its fine-grained (L3)
-  categories. Artificial Analysis and Arena AI use their own private prompt
-  sets.
+  categories. The Pruna Internal Video-Edit Benchmark uses 78 prompts
+  across advertising, e-commerce, real estate, camera, lighting, text, and
+  related categories. Artificial Analysis and Arena AI use their own
+  private prompt sets.
 - **Generation (Qwen and OneIG):** one image per prompt per endpoint when
   the run exists. Default resolution is 1024×1024. Exceptions: FLUX 1.1 Pro
   Ultra at 2K, FLUX 2 Flex at 1008×1008, and any endpoint labeled 2K. The
   seed is derived from the prompt, so every model gets the same seed for the
   same prompt. Steps, CFG, prompt rewrite, and safety filters follow each
   endpoint's default. This does not describe Artificial Analysis or Arena AI.
+- **Generation (Video-Edit):** one edited clip per prompt per endpoint when
+  the run exists. Every model sees the same source video for a prompt.
 - **Datapoint (Qwen and OneIG):** every model pair is compared on every
   prompt, with 10 votes per battle.
 - **Rapidata (Qwen and OneIG):** prompts longer than 400 characters are
@@ -222,7 +264,7 @@ def render_header():
               </svg>
             </button>
           </div>
-          <p class="app-header-tagline">Compare text-to-image models on quality, speed, and price</p>
+          <p class="app-header-tagline">Compare models on quality, speed, and price</p>
         </header>
         """,
         padding=False,
@@ -237,10 +279,42 @@ def _item(items, item_id):
     return items[0] if items else None
 
 
-def _dataset_choices(datasets, *, require_samples=False, require_pareto=False):
+def _dataset_modality(dataset):
+    return (dataset or {}).get("modality") or MODALITY_TEXT_TO_IMAGE
+
+
+def _datasets_for_modality(datasets, modality):
+    if not modality:
+        return list(datasets)
+    scoped = [
+        dataset
+        for dataset in datasets
+        if _dataset_modality(dataset) == modality
+    ]
+    return scoped or list(datasets)
+
+
+def _modality_choices(datasets):
+    present = {_dataset_modality(dataset) for dataset in datasets}
+    return [
+        (label, value) for label, value in MODALITY_CHOICES if value in present
+    ]
+
+
+def _default_dataset_id(datasets, modality, preferred=None):
+    scoped = _datasets_for_modality(datasets, modality)
+    if preferred and any(dataset["id"] == preferred for dataset in scoped):
+        return preferred
+    return scoped[0]["id"] if scoped else None
+
+
+def _dataset_choices(
+    datasets, *, modality=None, require_samples=False, require_pareto=False
+):
+    scoped = _datasets_for_modality(datasets, modality)
     return [
         (dataset["name"], dataset["id"])
-        for dataset in datasets
+        for dataset in scoped
         if (not require_samples or dataset.get("samples"))
         and (not require_pareto or _dataset_has_pareto(datasets, dataset["id"]))
     ]
@@ -256,20 +330,72 @@ def _sample_model_ids(datasets, dataset_id):
     samples = dataset.get("samples") if dataset else None
     if not samples:
         return set()
-    return set(samples.get("models") or [])
+    models = set(samples.get("models") or [])
+    return models | {display_model_name(model) for model in models}
+
+
+def _sample_media_map(samples):
+    return (samples or {}).get("images") or {}
+
+
+def _resolve_sample_model(samples, model):
+    media = _sample_media_map(samples)
+    if model in media:
+        return model
+    wanted = {str(model or "").strip(), display_model_name(model)}
+    wanted.discard("")
+    for key in media:
+        if key in wanted or display_model_name(key) in wanted:
+            return key
+    return None
+
+
+def _default_sample_models(samples):
+    models = list((samples or {}).get("models") or [])
+    preferred = [model for model in models if _is_pruna_model(model)]
+    preferred.sort(
+        key=lambda model: (
+            "draft" in str(model).casefold()
+            or "draft" in display_model_name(model).casefold(),
+            display_model_name(model).casefold(),
+        )
+    )
+    return (preferred or models)[:2]
+
+
+def _pareto_price_column(data):
+    columns = getattr(data, "columns", []) if data is not None else []
+    for column in _PARETO_PRICE_COLUMNS:
+        if column in columns:
+            return column
+    return None
+
+
+def _pareto_time_column(data):
+    columns = getattr(data, "columns", []) if data is not None else []
+    for column in _PARETO_TIME_COLUMNS:
+        if column in columns:
+            return column
+    return None
 
 
 def _dataset_has_pareto(datasets, dataset_id):
     dataset = _item(datasets, dataset_id)
-    columns = getattr(dataset.get("data") if dataset else None, "columns", [])
-    return _PARETO_PRICE_COLUMN in columns or _PARETO_TIME_COLUMN in columns
+    data = dataset.get("data") if dataset else None
+    return (
+        _pareto_price_column(data) is not None
+        or _pareto_time_column(data) is not None
+    )
 
 
-def _dataset_dropdown_update(datasets, tab, dataset_id):
+def _dataset_dropdown_update(datasets, tab, dataset_id, modality=None):
     """Limit the dataset list to what the current tab can show."""
+    if modality is None:
+        modality = _dataset_modality(_item(datasets, dataset_id))
     return gr.update(
         choices=_dataset_choices(
             datasets,
+            modality=modality,
             require_samples=tab == TAB_SAMPLES
             and _dataset_has_samples(datasets, dataset_id),
             require_pareto=tab == TAB_PARETO
@@ -375,9 +501,11 @@ _LEADERBOARD_IDENTITY_COLUMNS = [
     "Optimized",
 ]
 _LEADERBOARD_META_COLUMNS = [
+    "Time / Output Video Second (s)",
     "Median Generation Time (s)",
     "Min Generation Time (s)",
     "Price / Image (USD)",
+    "Price / Second of Video (USD)",
     "Evaluation Date (UTC)",
     "Date",
 ]
@@ -604,7 +732,9 @@ def _display_label(column):
         "Arena Text Rendering Elo": "Text Rendering",
         "Median Generation Time (s)": "Median generation time",
         "Min Generation Time (s)": "Min generation time",
+        "Time / Output Video Second (s)": "Generation time per second of video",
         "Price / Image (USD)": "Price per image",
+        "Price / Second of Video (USD)": "Price per second of video",
         "Evaluation Date (UTC)": "Date",
         "Date": "Date",
     }
@@ -679,6 +809,23 @@ def _applied_key(view_state):
     )
 
 
+def _is_pruna_model(model_id) -> bool:
+    raw = str(model_id or "").casefold()
+    label = display_model_name(model_id).casefold()
+    return any(
+        value.startswith(prefix)
+        for value in (raw, label)
+        for prefix in ("p-image", "p_image", "p-video", "p_video")
+    )
+
+
+def _pareto_fill_colors(models):
+    return [
+        _PARETO_PRUNA_COLOR if _is_pruna_model(model) else _PARETO_OTHER_COLOR
+        for model in models
+    ]
+
+
 def _build_pareto_figure(
     data,
     score_column,
@@ -703,6 +850,8 @@ def _build_pareto_figure(
 
     dominated = scatter.loc[[not flag for flag in on_frontier]].copy()
     frontier = scatter.loc[on_frontier].sort_values(x_column).copy()
+    dominated_colors = _pareto_fill_colors(dominated["Model"]) if not dominated.empty else []
+    frontier_colors = _pareto_fill_colors(frontier["Model"]) if not frontier.empty else []
     if not dominated.empty:
         dominated["Model"] = dominated["Model"].map(display_model_name)
     if not frontier.empty:
@@ -723,10 +872,11 @@ def _build_pareto_figure(
                 name="Below frontier",
                 text=dominated["Model"],
                 hovertemplate=hover,
+                showlegend=False,
                 marker={
                     "size": 9,
-                    "color": "#d8b4fe",
-                    "opacity": 0.8,
+                    "color": dominated_colors,
+                    "opacity": 0.85,
                     "line": {"width": 0},
                 },
             )
@@ -740,14 +890,51 @@ def _build_pareto_figure(
                 name="On frontier",
                 text=frontier["Model"],
                 hovertemplate=hover,
-                line={"color": "#69a45c", "width": 2.5},
+                showlegend=False,
+                line={"color": _PARETO_FRONTIER_OUTLINE, "width": 2.5},
                 marker={
                     "size": 12,
-                    "color": "#69a45c",
-                    "line": {"width": 1.5, "color": "#86c077"},
+                    "color": frontier_colors,
+                    "line": {"width": 2.5, "color": _PARETO_FRONTIER_OUTLINE},
                 },
             )
         )
+    for name, marker in (
+        (
+            "Pruna",
+            {
+                "size": 10,
+                "color": _PARETO_PRUNA_COLOR,
+                "line": {"width": 0},
+            },
+        ),
+        (
+            "Other models",
+            {
+                "size": 10,
+                "color": _PARETO_OTHER_COLOR,
+                "line": {"width": 0},
+            },
+        ),
+        (
+            "On frontier",
+            {
+                "size": 12,
+                "color": "rgba(0,0,0,0)",
+                "line": {"width": 2.5, "color": _PARETO_FRONTIER_OUTLINE},
+            },
+        ),
+    ):
+        fig.add_trace(
+            go.Scatter(
+                x=[None],
+                y=[None],
+                mode="markers",
+                name=name,
+                marker=marker,
+                hoverinfo="skip",
+            )
+        )
 
     score_label = _display_label(score_column)
     fig.update_layout(
@@ -887,23 +1074,43 @@ def _pareto_pair(
     if data is None or not score_column or score_column not in data.columns:
         return None, score_missing, None, score_missing
 
+    price_column = _pareto_price_column(data) or _PARETO_PRICE_COLUMN
+    price_title = _PARETO_PRICE_TITLES.get(price_column, "Price (USD)")
+    price_missing = (
+        "Price per second of video isn't available for this dataset."
+        if price_column == _PARETO_VIDEO_PRICE_COLUMN
+        else "Price per image isn't available for this dataset."
+    )
     price_fig, price_message = _pareto_axis(
         data,
         score_column,
-        _PARETO_PRICE_COLUMN,
-        "Price per image (USD)",
-        "Price per image isn't available for this dataset.",
+        price_column,
+        price_title,
+        price_missing,
         "No models have both a score and a price for this metric.",
         x_hover_prefix="$",
         x_axis_type=_pareto_axis_type(price_scale),
     )
+    time_column = _pareto_time_column(data) or _PARETO_TIME_COLUMN
+    time_title = _PARETO_TIME_TITLES.get(time_column, "Generation time (s)")
+    time_missing = (
+        "Generation time per second of video isn't available for this dataset."
+        if time_column == _PARETO_VIDEO_TIME_COLUMN
+        else "Min generation time isn't available for this dataset."
+    )
+    time_empty = (
+        "No models have both a score and generation time per second of "
+        "video for this metric."
+        if time_column == _PARETO_VIDEO_TIME_COLUMN
+        else "No models have both a score and a min generation time for this metric."
+    )
     time_fig, time_message = _pareto_axis(
         data,
         score_column,
-        _PARETO_TIME_COLUMN,
-        "Min generation time (s)",
-        "Min generation time isn't available for this dataset.",
-        "No models have both a score and a min generation time for this metric.",
+        time_column,
+        time_title,
+        time_missing,
+        time_empty,
         x_hover_suffix="s",
         x_axis_type=_pareto_axis_type(latency_scale),
     )
@@ -911,29 +1118,29 @@ def _pareto_pair(
 
 
 def _pareto_dataset_message(data):
-    has_price = data is not None and _PARETO_PRICE_COLUMN in data.columns
-    has_time = data is not None and _PARETO_TIME_COLUMN in data.columns
+    has_price = _pareto_price_column(data) is not None
+    has_time = _pareto_time_column(data) is not None
     if has_price or has_time:
         return None
     return (
-        "Price per image and min generation time aren't available for "
+        "Price and generation time aren't available for "
         "this dataset, so these plots can't be drawn."
     )
 
 
 def _pareto_slot_note(price_fig, price_message, time_fig, time_message, data):
-    has_price = data is not None and _PARETO_PRICE_COLUMN in data.columns
-    has_time = data is not None and _PARETO_TIME_COLUMN in data.columns
+    has_price = _pareto_price_column(data) is not None
+    has_time = _pareto_time_column(data) is not None
     notes = []
     if has_price and not has_time:
         notes.append(
-            "Min generation time isn't available for this dataset, so only "
+            "Generation time isn't available for this dataset, so only "
             "price vs score is shown."
         )
     elif has_time and not has_price:
         notes.append(
-            "Price per image isn't available for this dataset, so only min "
-            "generation time vs score is shown."
+            "Price isn't available for this dataset, so only "
+            "time vs score is shown."
         )
     if price_fig is None and has_price:
         notes.append(price_message)
@@ -954,8 +1161,8 @@ def _pareto_slot_updates(
     score_columns = [column for column in (score_columns or []) if column]
     price_scales = _normalize_pareto_scales(price_scales)
     time_scales = _normalize_pareto_scales(time_scales)
-    has_price = data is not None and _PARETO_PRICE_COLUMN in data.columns
-    has_time = data is not None and _PARETO_TIME_COLUMN in data.columns
+    has_price = _pareto_price_column(data) is not None
+    has_time = _pareto_time_column(data) is not None
     dataset_note = _pareto_dataset_message(data)
     updates = [_pareto_note_update(dataset_note)]
     hide_all_slots = not has_price and not has_time
@@ -1034,15 +1241,48 @@ def _samples_html(samples, selected_models, num_prompts, seed=0):
         return _pareto_unavailable_html(
             "Samples aren't available for this dataset."
         )
-    images = samples.get("images", {})
-    models = [model for model in (selected_models or []) if model in images]
+    models = [
+        resolved
+        for model in (selected_models or [])
+        if (resolved := _resolve_sample_model(samples, model))
+    ]
     if not models:
-        models = (samples.get("models") or [])[:2]
+        models = _default_sample_models(samples)
     return _build_compare_samples_html(samples, models, num_prompts, seed)
 
 
+def _compare_media_html(url, label, *, kind):
+    safe_url = escape(url, quote=True)
+    safe_label = escape(label)
+    if kind == "video":
+        return (
+            f'<video src="{safe_url}" controls preload="metadata" '
+            f'playsinline></video>'
+        )
+    return (
+        f'<a href="{safe_url}" target="_blank" rel="noopener noreferrer">'
+        f'<img src="{safe_url}" alt="{safe_label} sample" loading="lazy" />'
+        f"</a>"
+    )
+
+
+def _compare_cell_html(label, url, *, kind, extra_class=""):
+    classes = "compare-cell"
+    if extra_class:
+        classes = f"{classes} {extra_class}"
+    return f"""
+                <div class="{classes}">
+                    <div class="compare-model-label">{escape(label)}</div>
+                    {_compare_media_html(url, label, kind=kind)}
+                </div>
+                """
+
+
 def _build_compare_samples_html(samples, selected_models, num_prompts, seed=0):
     selected_models = list(selected_models or [])[:MAX_COMPARE_MODELS]
+    media = _sample_media_map(samples)
+    kind = (samples or {}).get("kind") or "image"
+    source_videos = (samples or {}).get("source_videos") or {}
 
     if not selected_models:
         return (
@@ -1053,7 +1293,7 @@ def _build_compare_samples_html(samples, selected_models, num_prompts, seed=0):
 
     shared_prompt_ids = None
     for model in selected_models:
-        model_prompt_ids = set(samples["images"][model])
+        model_prompt_ids = set(media.get(model) or [])
         shared_prompt_ids = (
             model_prompt_ids
             if shared_prompt_ids is None
@@ -1073,29 +1313,31 @@ def _build_compare_samples_html(samples, selected_models, num_prompts, seed=0):
     rng.shuffle(prompt_pool)
     chosen = prompt_pool[: max(1, min(int(num_prompts), len(prompt_pool)))]
 
-    columns = len(selected_models)
     blocks = []
     for index, prompt_id in enumerate(chosen, start=1):
         prompt_text = escape(samples["prompts"].get(prompt_id, ""))
         cells = []
+        source_url = source_videos.get(prompt_id)
+        if source_url:
+            cells.append(
+                _compare_cell_html(
+                    "Source", source_url, kind="video", extra_class="compare-source"
+                )
+            )
         for model in selected_models:
-            image_url = escape(samples["images"][model][prompt_id], quote=True)
             cells.append(
-                f"""
-                <div class="compare-cell">
-                    <div class="compare-model-label">{escape(display_model_name(model))}</div>
-                    <a href="{image_url}" target="_blank" rel="noopener noreferrer">
-                        <img src="{image_url}" alt="{escape(display_model_name(model))} sample" loading="lazy" />
-                    </a>
-                </div>
-                """
+                _compare_cell_html(
+                    display_model_name(model),
+                    media[model][prompt_id],
+                    kind=kind,
+                )
             )
+        columns = len(cells)
         blocks.append(
             f"""
             <div class="compare-prompt-block">
                 <div class="compare-prompt-meta">
                     <span>Prompt {index}</span>
-                    <span>{escape(prompt_id)}</span>
                 </div>
                 <p class="compare-prompt-text">{prompt_text}</p>
                 <div class="compare-row" style="grid-template-columns: repeat({columns}, minmax(0, 1fr));">
@@ -1123,9 +1365,19 @@ def _filter_row(datasets, metrics, default_dataset_id, default_metric_id=None):
     metric_id = _coerce_metric(
         datasets, metrics, default_dataset_id, default_metric_id
     )
+    default_modality = _dataset_modality(_item(datasets, default_dataset_id))
     with gr.Row(elem_classes="view-filters"):
+        modality_dd = gr.Dropdown(
+            choices=_modality_choices(datasets),
+            value=default_modality,
+            label="Type",
+            type="value",
+            filterable=False,
+            scale=1,
+            min_width=150,
+        )
         dataset_dd = gr.Dropdown(
-            choices=_dataset_choices(datasets),
+            choices=_dataset_choices(datasets, modality=default_modality),
             value=default_dataset_id,
             label="Dataset",
             type="value",
@@ -1157,7 +1409,7 @@ def _filter_row(datasets, metrics, default_dataset_id, default_metric_id=None):
             min_width=180,
             elem_classes="filter-chips",
         )
-    return dataset_dd, metric_dd, models_dd
+    return modality_dd, dataset_dd, metric_dd, models_dd
 
 
 def render_image_workspace(datasets, metrics, default_dataset_id, default_metric_id):
@@ -1172,15 +1424,16 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
         with gr.Column(elem_classes="workspace-filters") as filters_host:
             gr.Markdown(
                 "<p class='filter-help'>"
-                "These filters apply to Leaderboards, Pareto plots, and Samples. "
-                "On Samples, only datasets and models we have generations for "
-                "are listed. On Pareto plots, only datasets with price or "
-                "generation time are listed. Search in Models, or leave it "
-                "empty to include every model."
+                "Start with Type to switch between Video to Video and Text "
+                "to Image. The rest of the filters follow you across "
+                "Leaderboards, Pareto plots, and Samples. Samples only "
+                "lists datasets and models we have generations for; Pareto "
+                "plots only lists datasets with price or generation time. "
+                "Search in Models, or leave it empty to include every model."
                 "</p>",
                 elem_classes="filter-help-host",
             )
-            dataset_dd, metric_dd, models_dd = _filter_row(
+            modality_dd, dataset_dd, metric_dd, models_dd = _filter_row(
                 datasets, metrics, default_dataset_id, None
             )
         with gr.Tabs(elem_classes="main-tabs") as main_tabs:
@@ -1306,7 +1559,7 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
                                 elem_classes="pareto-col",
                             ) as slot_time_col:
                                 slot_time_scale = _pareto_plot_heading(
-                                    "Min generation time vs score"
+                                    "Time vs score"
                                 )
                                 slot_time = gr.Plot(
                                     value=None,
@@ -1342,7 +1595,8 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
                 with gr.Column(visible=bool(initial_samples)) as samples_panel:
                     gr.Markdown(
                         f"<p class='view-help'>"
-                        f"The same prompts, side by side. Select up to "
+                        f"The same prompts, side by side. Video edits show the "
+                        f"source clip first. Select up to "
                         f"<strong>{MAX_COMPARE_MODELS}</strong> models above, or leave "
                         f"Models empty for two defaults."
                         f"</p>",
@@ -1460,8 +1714,14 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
     ):
         prev = dict(view_state or {})
         extras = extras or {}
+        modality = _dataset_modality(_item(datasets, dataset_id))
+        last_by_modality = dict(prev.get("dataset_by_modality") or {})
+        if dataset_id:
+            last_by_modality[modality] = dataset_id
         return {
             "dataset_id": dataset_id,
+            "modality": modality,
+            "dataset_by_modality": last_by_modality,
             "metric_id": metric_id,
             "models": list(models or []),
             "current_tab": tab,
@@ -1599,13 +1859,14 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
         tab = view_state.get("current_tab") or TAB_LEADERBOARDS
         selected_raw = _normalize_metric_ids(metric_id)
         incoming_models = list(models or [])
-        dataset_changed = source == "dataset" and dataset_id != view_state.get(
+        filter_changed = source in {"dataset", "modality"}
+        dataset_changed = filter_changed and dataset_id != view_state.get(
             "dataset_id"
         )
         can_pareto = _dataset_has_pareto(datasets, dataset_id)
         can_samples = _dataset_has_samples(datasets, dataset_id)
         selected_tab = tab
-        if source == "dataset":
+        if filter_changed:
             if tab == TAB_SAMPLES and not can_samples:
                 selected_tab = TAB_LEADERBOARDS
             elif tab == TAB_PARETO and not can_pareto:
@@ -1614,7 +1875,7 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
                 dataset_id,
                 metric_id,
                 models,
-                clear_metric=dataset_changed,
+                clear_metric=source == "modality" or dataset_changed,
                 require_samples=selected_tab == TAB_SAMPLES,
             )
             dataset_id, metric_id, models = synced[:3]
@@ -1659,7 +1920,7 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
             list(optimized_value or []),
         )
         extra_updates = None
-        if source == "dataset":
+        if filter_changed:
             view = resolve_view(datasets, metrics, dataset_id, metric_id)
             extra_updates = _leaderboard_extras(
                 view["data"] if view else None,
@@ -1712,6 +1973,67 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
             ),
         }
 
+    def on_modality(
+        modality,
+        dataset_id,
+        metric_id,
+        models,
+        platform_value,
+        owner_value,
+        optimized_value,
+        num_prompts,
+        seed,
+        view_state,
+    ):
+        view_state = dict(view_state or {})
+        last_by_modality = dict(view_state.get("dataset_by_modality") or {})
+        current_modality = view_state.get("modality") or _dataset_modality(
+            _item(datasets, dataset_id)
+        )
+        if dataset_id:
+            last_by_modality[current_modality] = dataset_id
+        dataset_id = _default_dataset_id(
+            datasets, modality, last_by_modality.get(modality)
+        )
+        view_state["modality"] = modality
+        view_state["dataset_by_modality"] = last_by_modality
+        result = _apply_filter_change(
+            "modality",
+            dataset_id,
+            metric_id,
+            models,
+            platform_value,
+            owner_value,
+            optimized_value,
+            num_prompts,
+            seed,
+            view_state,
+        )
+        if result is None:
+            return _skip_all(len(dataset_outputs))
+        extras = result["extra_updates"]
+        return (
+            _dataset_dropdown_update(
+                datasets,
+                result["selected_tab"],
+                result["dataset_id"],
+                modality=modality,
+            ),
+            result["metric_update"],
+            result["models_update"],
+            extras[6],
+            extras[0],
+            extras[1],
+            extras[2],
+            *result["views"],
+            gr.update(interactive=result["can_pareto"]),
+            gr.update(interactive=result["can_samples"]),
+            gr.update(selected=result["selected_tab"])
+            if result["selected_tab"] != result["tab"]
+            else gr.skip(),
+            result["state"],
+        )
+
     def on_dataset(
         dataset_id,
         metric_id,
@@ -2061,9 +2383,12 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
         handler.__name__ = f"on_tab_{tab}"
         return handler
 
+    default_modality = _dataset_modality(_item(datasets, default_dataset_id))
     view_state = gr.State(
         {
             "dataset_id": default_dataset_id,
+            "modality": default_modality,
+            "dataset_by_modality": {default_modality: default_dataset_id},
             "metric_id": None,
             "models": [],
             "current_tab": TAB_LEADERBOARDS,
@@ -2145,6 +2470,12 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
         main_tabs,
         view_state,
     ]
+    modality_dd.change(
+        on_modality,
+        inputs=[modality_dd, *filter_inputs],
+        outputs=dataset_outputs,
+        **_VIEW_EVENTS,
+    )
     dataset_dd.change(
         on_dataset,
         inputs=filter_inputs,