Fix wan2.2 VBench scorer cold-start failures: .mp4 staging, torch>=2.6 checkpoints, missing wget

examples/09_Wan22_VideoGen_Example/accuracy/vbench_runner.py

@@ -28,6 +28,8 @@
 
 import argparse
 import json
+import os
+import shutil
 import sys
 import traceback
 from importlib.resources import files as _pkg_files
@@ -36,6 +38,12 @@
 import vbench as _vbench_pkg
 from vbench import VBench
 
+# VBench's reference checkpoints are full pickles and fail under torch>=2.6's
+# weights_only=True default. This escape hatch is read at torch.load call time,
+# only applies when the callsite does not set weights_only explicitly, and is
+# scoped to this subprocess.
+os.environ.setdefault("TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD", "1")
+
 
 def _emit_error(exc: BaseException) -> None:
     """Print a structured JSON error line on stderr for the parent to surface."""
@@ -85,6 +93,29 @@ def main() -> int:
     )
     args = parser.parse_args()
 
+    # vbench downloads weights via wget/unzip subprocesses on a cold cache;
+    # fail fast instead of a FileNotFoundError mid-evaluation.
+    missing_tools = [t for t in ("wget", "unzip") if shutil.which(t) is None]
+    if missing_tools:
+        print(
+            json.dumps(
+                {
+                    "status": "error",
+                    "type": "MissingSystemDependency",
+                    "message": (
+                        f"Required system tool(s) not found: {', '.join(missing_tools)}. "
+                        "VBench downloads its per-dimension model weights via "
+                        "wget/unzip on first use. Install them in the client "
+                        "environment (e.g. `apt-get install wget unzip`) or "
+                        "pre-populate the VBench cache directory."
+                    ),
+                }
+            ),
+            file=sys.stderr,
+            flush=True,
+        )
+        return 2
+
     if not torch.cuda.is_available() and not args.allow_cpu:
         print(
             json.dumps(

src/inference_endpoint/evaluation/scoring.py

@@ -1495,7 +1495,9 @@ def _stage_videos(
             # strict=True surfaces missing/unmounted sources here, not as an
             # opaque decord read failure inside VBench 30 minutes later.
             resolved_src = src.resolve(strict=True)
-            dst = staged_dir / f"{safe_prompt}-{idx}{src.suffix or '.mp4'}"
+            # Always .mp4: VBench dispatches on extension (non-mp4 raises
+            # NotImplementedError); decord detects the container by content.
+            dst = staged_dir / f"{safe_prompt}-{idx}.mp4"
             dst.symlink_to(resolved_src)
 
     def _run_vbench_subprocess(

tests/unit/evaluation/test_scoring.py

@@ -774,6 +774,26 @@ def test_stage_clears_stale_files_from_prior_run(
         assert names == ["a cat-0.mp4", "a dog-0.mp4", "a tree-0.mp4"]
         assert not zombie.exists()
 
+    def test_stage_videos_renames_non_mp4_sources_to_mp4(
+        self, dataset, staged, vbench_project, tmp_path
+    ):
+        """Non-mp4 sources (e.g. .avi) stage under .mp4 names for VBench."""
+        report_dir, _ = staged
+        avi = tmp_path / "video_x.avi"
+        avi.write_bytes(b"")
+        scorer = VBenchScorer(
+            dataset_name="vid_acc",
+            dataset=dataset,
+            report_dir=report_dir,
+            ground_truth_column="prompt",
+            vbench_project_path=vbench_project,
+        )
+        staged_dir = report_dir / "vbench_videos"
+        scorer._stage_videos(staged_dir, [str(avi)], ["a cat"])
+        (staged_file,) = staged_dir.iterdir()
+        assert staged_file.name == "a cat-0.mp4"
+        assert staged_file.resolve() == avi.resolve()
+
     def test_subprocess_failure_includes_stderr_tail(
         self, dataset, staged, vbench_project, monkeypatch, tmp_path
     ):