p-video-2

app.py

@@ -2345,6 +2345,34 @@ def load_qwen_combined_dataframe(path):
     return df.reset_index(drop=True)
 
 
+def _is_pruna_video_model(series):
+    models = series.astype(str).str.casefold()
+    return models.str.startswith("p_video") | models.str.startswith("p-video")
+
+
+def _apply_video_timings(df, *, replace_displayed_time=False):
+    """Mix Fal wall time with Pruna model execution time."""
+    fal = df.get("Time / Output Video Second (s)")
+    execution = df.get("Execution Time / Output Video Second (s)")
+    if fal is None:
+        return df
+
+    if "model_id" in df.columns:
+        is_ours = _is_pruna_video_model(df["model_id"])
+    else:
+        is_ours = _is_pruna_video_model(df["Model"])
+
+    if execution is None:
+        mixed = fal
+    else:
+        ours_time = execution.where(execution.notna(), fal)
+        mixed = fal.where(~is_ours, ours_time)
+    if replace_displayed_time:
+        df["Time / Output Video Second (s)"] = mixed
+    df["Pareto Time / Output Video Second (s)"] = mixed
+    return df
+
+
 def load_video_editing_dataframe(path):
     """Load the video-to-video editing leaderboard."""
     df = pd.read_csv(path, na_values=["N/A", "n/a", ""])
@@ -2378,27 +2406,62 @@ def load_video_editing_dataframe(path):
             "Price / Second of Video (USD)",
         ],
     )
-    end_to_end = df.get("Time / Output Video Second (s)")
-    execution = df.get("Execution Time / Output Video Second (s)")
-    if end_to_end is not None:
-        if "model_id" in df.columns:
-            is_ours = df["model_id"].astype(str).str.lower().str.startswith(
-                "p_video_edit"
-            )
-        else:
-            is_ours = df["Model"].astype(str).str.casefold().str.startswith(
-                "p-video-edit"
-            )
-        if execution is None:
-            pareto_time = end_to_end
-        else:
-            ours_time = execution.where(execution.notna(), end_to_end)
-            pareto_time = end_to_end.where(~is_ours, ours_time)
-        df["Pareto Time / Output Video Second (s)"] = pareto_time
+    df = _apply_video_timings(df)
     df = df.drop(columns=["model_id"], errors="ignore")
     return df.reset_index(drop=True)
 
 
+def load_text_to_video_dataframe(path):
+    """Load the text-to-video leaderboard (P-Video-2 and Fal models)."""
+    df = pd.read_csv(path, na_values=["N/A", "n/a", ""])
+    model_column = "model_id" if "model_id" in df.columns else "Model"
+    df = df[~df[model_column].astype(str).str.casefold().str.startswith("agnes")].copy()
+    df = df.rename(
+        columns={
+            "model_id": "Model",
+            "datapoint_elo": "Datapoint Elo",
+            "rapidata_elo": "Rapidata Elo",
+            "min_generation_s": "Min Generation Time (s)",
+            "median_generation_s": "Median Generation Time (s)",
+            "p20_generation_s": "P20 Generation Time (s)",
+            "generation_s_per_output_video_s": "Time / Output Video Second (s)",
+            "model_execution_s_per_output_video_s": (
+                "Execution Time / Output Video Second (s)"
+            ),
+            "price_usd_per_second": "Price / Second of Video (USD)",
+        }
+    )
+    df = df.drop(columns=["wandb_run_ids", "n_generations"], errors="ignore")
+    df["Model"] = df["Model"].astype(str).str.strip()
+    df = _as_numeric(
+        df,
+        [
+            "Datapoint Elo",
+            "Rapidata Elo",
+            "Min Generation Time (s)",
+            "Median Generation Time (s)",
+            "P20 Generation Time (s)",
+            "Time / Output Video Second (s)",
+            "Execution Time / Output Video Second (s)",
+            "Price / Second of Video (USD)",
+        ],
+    )
+    # Pruna rows use model execution time; Fal rows keep Fal wall time.
+    df = _apply_video_timings(df, replace_displayed_time=True)
+    df = df.drop(
+        columns=["Execution Time / Output Video Second (s)"],
+        errors="ignore",
+    )
+    elo_columns = [
+        column
+        for column in ("Datapoint Elo", "Rapidata Elo")
+        if column in df.columns
+    ]
+    if elo_columns:
+        df = df.dropna(subset=elo_columns, how="all")
+    return df.reset_index(drop=True)
+
+
 df = load_oneig_dataframe(oneig_path)
 
 oneig_metric_columns = [
@@ -2466,11 +2529,16 @@ video_path = _resolve_data_path(
     data_dir / "video-editing-leaderboard.csv",
     space_root.parent / "video-editing-leaderboard.csv",
 )
+text_to_video_path = _resolve_data_path(
+    data_dir / "p-video-2-leaderboard.csv",
+    space_root.parent / "p-video-2-leaderboard.csv",
+)
 
 qwen_df = load_qwen_combined_dataframe(qwen_path)
 aa_df = load_artificial_analysis_dataframe(aa_path)
 arena_df = load_arena_ai_dataframe(arena_path)
 video_df = load_video_editing_dataframe(video_path)
+text_to_video_df = load_text_to_video_dataframe(text_to_video_path)
 qwen_display_columns = [
     col
     for col in [
@@ -2514,6 +2582,19 @@ video_display_columns = [
     ]
     if col in video_df.columns
 ]
+text_to_video_display_columns = [
+    col
+    for col in [
+        "Model",
+        "Datapoint Elo",
+        "Rapidata Elo",
+        "Time / Output Video Second (s)",
+        "Median Generation Time (s)",
+        "Min Generation Time (s)",
+        "Price / Second of Video (USD)",
+    ]
+    if col in text_to_video_df.columns
+]
 
 oneig_samples = load_sample_comparison_data(oneig_combined_dir)
 qwen_samples = load_sample_comparison_data(qwen_combined_dir)
@@ -2577,8 +2658,26 @@ arena_metric_ids = _metric_ids_for(
     ],
 )
 video_metric_ids = _metric_ids_for(video_df, ["datapoint_elo"])
+text_to_video_metric_ids = _metric_ids_for(
+    text_to_video_df, ["datapoint_elo", "rapidata_elo"]
+)
 
 datasets = [
+    {
+        "id": "text_to_video",
+        "name": "VBench-2.0 Dataset",
+        "modality": "text_to_video",
+        "data": text_to_video_df,
+        "columns": text_to_video_display_columns,
+        "metric_ids": text_to_video_metric_ids,
+        "note": (
+            "Datapoint Elo and Rapidata Elo from pairwise text-to-video "
+            "preference. Price is USD per second of output video. Time per "
+            "second of video is Fal wall time, except Pruna models which use "
+            "model execution time."
+        ),
+        "samples": None,
+    },
     {
         "id": "video_editing",
         "name": "Pruna Internal Video-Edit Benchmark",
@@ -2640,10 +2739,13 @@ datasets = [
 datasets = [dataset for dataset in datasets if dataset["metric_ids"]]
 
 DEFAULT_DATASET_ID = next(
-    (dataset["id"] for dataset in datasets if dataset["id"] == "video_editing"),
+    (dataset["id"] for dataset in datasets if dataset["id"] == "text_to_video"),
     next(
-        (dataset["id"] for dataset in datasets if dataset["id"] == "qwen"),
-        datasets[0]["id"] if datasets else None,
+        (dataset["id"] for dataset in datasets if dataset["id"] == "video_editing"),
+        next(
+            (dataset["id"] for dataset in datasets if dataset["id"] == "qwen"),
+            datasets[0]["id"] if datasets else None,
+        ),
     ),
 )
 DEFAULT_METRIC_ID = None

data/p-video-2-leaderboard.csv

@@ -0,0 +1,19 @@
+model_id,wandb_run_ids,n_generations,min_generation_s,median_generation_s,p20_generation_s,generation_s_per_output_video_s,model_execution_s_per_output_video_s,datapoint_elo,rapidata_elo,price_usd_per_second
+gemini_omni_1_1_flash,mfajagum,89,26.276599962002365,31.884095050001633,29.0771403791965,6.399738597206617,,1037,1197,0.100
+grok_imagine_video,q39r8v9e,90,68.9776645039965,99.17614604350092,78.69151808420138,16.13514411181308,,958,994,0.050
+grok_imagine_video_v1_5,z2fp7hd7,90,38.42492011199647,52.268091876499966,46.47850653079659,11.452002471104521,,959,1016,0.140
+ltx_2_5_fast,h6se1hfi,90,21.695961473000352,26.684429597007693,24.118977216200438,5.270079915707363,,1002,1035,0.090
+ltx_2_5_pro,kjbgxbab,90,23.997383733993047,27.897931821993552,25.263995286799037,4.936325968288616,,1007,978,0.120
+minimax_h3,xpcv2m58,89,97.61419671699696,117.51345164199665,106.57570995900024,24.968780374361877,,1026,1108,0.100
+minimax_h3_max,0d2lkmp6,90,4.173863318999793,4.49869444649994,4.4671880990012145,1.3807388451000264,0.644,1031,1197,0.080
+minimax_h3_max_turbo__prompt_expansion_mode_balanced,kxuwq31c,90,2.891819470001792,3.487746603501364,3.134695611400821,1.35231252479333,0.61,1040,1214,0.040
+p_video_2__draft_false__prompt_upsampling_false__resolution_1080p,9hgkhujw,90,13.33159556199098,16.702878594005597,14.793676785795833,4.033411242884629,2.151682222222222,,,0.050
+p_video_2__draft_false__prompt_upsampling_false__resolution_720p,m2jf4z7h,89,5.859760620005545,8.162274362999597,7.147269867203431,2.1525601170473116,0.9053123595505614,,,0.025
+p_video_2__draft_false__prompt_upsampling_true__resolution_1080p,uoqpv10b,90,14.615217412007041,18.316011337505188,16.441080821005745,4.5638632221758275,2.152566666666667,979,991,0.050
+p_video_2__draft_false__prompt_upsampling_true__resolution_720p,2q5jydr8,90,7.227749306999613,9.880705762494472,9.154919161600992,2.4789854674801206,0.9079755555555558,985,1039,0.025
+p_video_2__draft_true__prompt_upsampling_false__resolution_1080p,qbt0atz4,89,6.250065040003392,9.528199804000906,7.504525098402519,2.431133616615736,0.7517640449438197,,,0.030
+p_video_2__draft_true__prompt_upsampling_false__resolution_720p,9ur57h54,88,3.379928643000312,5.454214365498046,4.208131522199255,1.4960965825225272,0.40954545454545455,,,0.015
+p_video_2__draft_true__prompt_upsampling_true__resolution_1080p,gcbyy4wn,90,8.047415203996934,11.128662197996164,9.22702455239487,2.8557857865532665,0.7514066666666668,,,0.030
+p_video_2__draft_true__prompt_upsampling_true__resolution_720p,6hi994vk,90,4.985804016003385,7.4922584965024726,6.472871531004785,2.4024102996157146,0.40678444444444434,982,1048,0.015
+seedance_2_5_turbo,l9zjnv7n,90,168.997338743,252.07832621250054,211.93920676639829,67.60899374696221,,1037,1080,0.200
+veo_3_1_lite,jjbobes4,90,33.894167409991496,37.83729844300251,37.009501291197374,6.745116482181308,,968,986,0.050

model_display.py

@@ -95,6 +95,17 @@ MODEL_DISPLAY_NAMES = {
     "P-Video Edit Final (draft)": "P-Video-Edit Draft",
     "p_video_edit_preview__replicate_final": "P-Video-Edit",
     "p_video_edit_preview__replicate_final__draft": "P-Video-Edit Draft",
+    # Text-to-video leaderboard
+    "gemini_omni_1_1_flash": "Gemini Omni 1.1 Flash",
+    "grok_imagine_video": "Grok Imagine Video",
+    "grok_imagine_video_v1_5": "Grok Imagine Video 1.5",
+    "ltx_2_5_fast": "LTX 2.5 Fast",
+    "ltx_2_5_pro": "LTX 2.5 Pro",
+    "minimax_h3": "MiniMax H3",
+    "minimax_h3_max": "MiniMax H3 Max",
+    "minimax_h3_max_turbo__prompt_expansion_mode_balanced": "MiniMax H3 Max Turbo",
+    "seedance_2_5_turbo": "Seedance 2.5 Turbo",
+    "veo_3_1_lite": "Veo 3.1 Lite",
     # Video-to-video leaderboard
     "gemini_omni_flash_edit__fal": "Gemini Omni Flash Edit",
     "grok_imagine_video__replicate": "Grok Imagine Video",
@@ -201,6 +212,35 @@ MODEL_DISPLAY_NAMES = {
 }
 
 
+def _p_video_2_display_name(model_id: str):
+    """Turn p_video_2 variant ids into P-Video-2 Draft 720p labels."""
+    raw = str(model_id).strip()
+    if raw != "p_video_2" and not raw.startswith("p_video_2__"):
+        return None
+    if raw == "p_video_2":
+        return "P-Video-2"
+
+    draft = False
+    upsample = None
+    resolution = None
+    for part in raw.split("__")[1:]:
+        if part.startswith("draft_"):
+            draft = part.endswith("true")
+        elif part.startswith("prompt_upsampling_"):
+            upsample = part.endswith("true")
+        elif part.startswith("resolution_"):
+            resolution = part[len("resolution_") :]
+
+    label = "P-Video-2"
+    if draft:
+        label += " Draft"
+    if resolution:
+        label += f" {resolution}"
+    if upsample is False:
+        label += " (no prompt upsampling)"
+    return label
+
+
 def _prettify_snake_case(model_id: str) -> str:
     parts = [part for part in str(model_id).split("_") if part]
     pretty = []
@@ -225,6 +265,9 @@ def display_model_name(model_id) -> str:
         return ""
     if raw in MODEL_DISPLAY_NAMES:
         return MODEL_DISPLAY_NAMES[raw]
+    p_video_2 = _p_video_2_display_name(raw)
+    if p_video_2:
+        return p_video_2
     # Already a human label (spaces / punctuation) — keep as-is.
     if re.search(r"[\s.\[\]()-]", raw):
         return raw

ui.py

@@ -53,9 +53,11 @@ TAB_PARETO = "pareto"
 TAB_SAMPLES = "samples"
 TAB_ABOUT = "about"
 
+MODALITY_TEXT_TO_VIDEO = "text_to_video"
 MODALITY_VIDEO_TO_VIDEO = "video_to_video"
 MODALITY_TEXT_TO_IMAGE = "text_to_image"
 MODALITY_CHOICES = [
+    ("Text to Video", MODALITY_TEXT_TO_VIDEO),
     ("Video to Video", MODALITY_VIDEO_TO_VIDEO),
     ("Text to Image", MODALITY_TEXT_TO_IMAGE),
 ]
@@ -71,15 +73,15 @@ _VIEW_EVENTS = {
 ABOUT_OVERVIEW_CONTENT = """
 # About P-Bench
 
-P-Bench compares **text-to-image** and **video-to-video** models, including
-optimized or accelerated endpoints, on **quality, speed, and price**. Each
-view is a **dataset** scored with a **metric**, written as `Dataset | Metric`.
-There is no single score across P-Bench.
+P-Bench compares **text-to-video**, **video-to-video**, and **text-to-image**
+models, including optimized or accelerated endpoints, on **quality, speed,
+and price**. Each view is a **dataset** scored with a **metric**, written as
+`Dataset | Metric`. There is no single score across P-Bench.
 
 ## How to read it
 
-1. Pick a **type** (Video to Video or Text to Image), then a **dataset**
-   and a **metric**.
+1. Pick a **model type** (Text to Video, Video to Video, or Text to Image),
+   then a **dataset** and a **metric**.
 2. **Leaderboards**: ranked by that metric. Price and generation time sit in
    the same table when the source publishes them.
 3. **Pareto plots**: mark models that are not beaten on both higher score
@@ -87,7 +89,7 @@ There is no single score across P-Bench.
    can open this tab (not Arena AI).
 4. **Samples**: the same prompts, side by side. Only for datasets we
    generated (Qwen Image Dataset, OneIG Alignment Dataset, and the
-   Pruna Internal Video-Edit Benchmark). Video samples show the source
+   Pruna Internal Video-Edit Benchmark). Video-edit samples show the source
    clip first, then each model's edit.
 
 ## How a score is made
@@ -107,6 +109,13 @@ prompt suites, so samples are not shown.
 
 ## Current datasets
 
+### VBench-2.0 Dataset
+VBench-2.0 prompts, comparing P-Video-2 variants with Fal-hosted models.
+Quality is Datapoint Elo and Rapidata Elo from pairwise preference. Price
+is USD per second of output video. Time per second of video is Fal wall
+time, except Pruna models which use model execution time. Samples are not
+shown for this dataset.
+
 ### Pruna Internal Video-Edit Benchmark
 Pruna's internal video-to-video editing benchmark, collected by our
 research engineers. It combines prompts from public video-editing
@@ -162,12 +171,15 @@ ABOUT_DETAILS_CONTENT = """
   text rendering).
 - **Generation time**: median and minimum generation time in seconds for
   images, as reported in the evaluation table. For video, generation time
-  per second of output video is the more informative figure (end-to-end
-  wall time). This is not a p95, and we do not state warm vs cold or
-  concurrent load. Not available for Arena AI.
+  per second of output video is the more informative figure. On the
+  text-to-video benchmark this is Fal wall time, except Pruna models which
+  use model execution time. On video-edit it is end-to-end wall time. This
+  is not a p95, and we do not state warm vs cold or concurrent load. Not
+  available for Arena AI.
 - **Price**: USD per image for text-to-image, or USD per second of output
-  video for video-to-video. We do not state list price vs amount paid, or
-  whether failed generations are included. Not available for Arena AI.
+  video for text-to-video and video-to-video. We do not state list price vs
+  amount paid, or whether failed generations are included. Not available
+  for Arena AI.
 
 Scores from different datasets or metrics are **not interchangeable**. A high
 OneIG alignment score is not the same quantity as a Datapoint Elo. Compare
@@ -181,16 +193,19 @@ models *within* a Dataset | Metric view.
 - **Prompt counts:** OneIG Alignment uses 100 anime, 100 human, and 99 object
   prompts (299 total). Qwen Image Dataset uses 100 prompts sampled from the
   1,000-prompt pool for roughly even coverage of its fine-grained (L3)
-  categories. The Pruna Internal Video-Edit Benchmark uses 78 prompts
-  across advertising, e-commerce, real estate, camera, lighting, text, and
-  related categories. Artificial Analysis and Arena AI use their own
-  private prompt sets.
+  categories. The VBench-2.0 Dataset uses about 90
+  generations per model. The Pruna Internal Video-Edit Benchmark uses 78
+  prompts across advertising, e-commerce, real estate, camera, lighting,
+  text, and related categories. Artificial Analysis and Arena AI use their
+  own private prompt sets.
 - **Generation (Qwen and OneIG):** one image per prompt per endpoint when
   the run exists. Default resolution is 1024×1024. Exceptions: FLUX 1.1 Pro
   Ultra at 2K, FLUX 2 Flex at 1008×1008, and any endpoint labeled 2K. The
   seed is derived from the prompt, so every model gets the same seed for the
   same prompt. Steps, CFG, prompt rewrite, and safety filters follow each
   endpoint's default. This does not describe Artificial Analysis or Arena AI.
+- **Generation (Text-to-Video):** one clip per prompt per endpoint when the
+  run exists. About 90 generations per model.
 - **Generation (Video-Edit):** one edited clip per prompt per endpoint when
   the run exists. Every model sees the same source video for a prompt.
 - **Datapoint (Qwen and OneIG):** every model pair is compared on every
@@ -1370,11 +1385,11 @@ def _filter_row(datasets, metrics, default_dataset_id, default_metric_id=None):
         modality_dd = gr.Dropdown(
             choices=_modality_choices(datasets),
             value=default_modality,
-            label="Type",
+            label="Model Type",
             type="value",
             filterable=False,
             scale=1,
-            min_width=150,
+            min_width=170,
         )
         dataset_dd = gr.Dropdown(
             choices=_dataset_choices(datasets, modality=default_modality),
@@ -1424,12 +1439,13 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
         with gr.Column(elem_classes="workspace-filters") as filters_host:
             gr.Markdown(
                 "<p class='filter-help'>"
-                "Start with Type to switch between Video to Video and Text "
-                "to Image. The rest of the filters follow you across "
-                "Leaderboards, Pareto plots, and Samples. Samples only "
-                "lists datasets and models we have generations for; Pareto "
-                "plots only lists datasets with price or generation time. "
-                "Search in Models, or leave it empty to include every model."
+                "Start with Model Type to switch modalities. The rest of "
+                "the filters follow you across Leaderboards, Pareto plots, "
+                "and Samples. "
+                "Samples only lists datasets and models we have generations "
+                "for; Pareto plots only lists datasets with price or "
+                "generation time. Search in Models, or leave it empty to "
+                "include every model."
                 "</p>",
                 elem_classes="filter-help-host",
             )