From d932b87b7c255d97c94e35546bca2963f79b7eb6 Mon Sep 17 00:00:00 2001 From: Tin-Yin Lai Date: Wed, 8 Jul 2026 14:05:00 -0700 Subject: [PATCH 1/5] Stage VBench videos as .mp4 regardless of source suffix VBench's load_video dispatches purely on the file extension and raises NotImplementedError for anything but .mp4/.gif/frame dirs. trtllm-serve emits MJPEG .avi when ffmpeg is unavailable server-side, so a full accuracy run generated 248 videos and then failed at scoring. decord (libav) detects the real container by content, so an .avi symlinked under an .mp4 name decodes fine (verified on GB300: 81-frame 720x1280 MJPEG-AVI reads correctly). Always name the staged symlinks .mp4. --- src/inference_endpoint/evaluation/scoring.py | 4 +++- tests/unit/evaluation/test_scoring.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/inference_endpoint/evaluation/scoring.py b/src/inference_endpoint/evaluation/scoring.py index 8c4a49998..975b7f47c 100644 --- a/src/inference_endpoint/evaluation/scoring.py +++ b/src/inference_endpoint/evaluation/scoring.py @@ -1495,7 +1495,9 @@ def _stage_videos( # strict=True surfaces missing/unmounted sources here, not as an # opaque decord read failure inside VBench 30 minutes later. resolved_src = src.resolve(strict=True) - dst = staged_dir / f"{safe_prompt}-{idx}{src.suffix or '.mp4'}" + # Always .mp4: VBench dispatches on extension (non-mp4 raises + # NotImplementedError); decord detects the container by content. + dst = staged_dir / f"{safe_prompt}-{idx}.mp4" dst.symlink_to(resolved_src) def _run_vbench_subprocess( diff --git a/tests/unit/evaluation/test_scoring.py b/tests/unit/evaluation/test_scoring.py index 92f14a93d..4ec515e85 100644 --- a/tests/unit/evaluation/test_scoring.py +++ b/tests/unit/evaluation/test_scoring.py @@ -774,6 +774,26 @@ def test_stage_clears_stale_files_from_prior_run( assert names == ["a cat-0.mp4", "a dog-0.mp4", "a tree-0.mp4"] assert not zombie.exists() + def test_stage_videos_renames_non_mp4_sources_to_mp4( + self, dataset, staged, vbench_project, tmp_path + ): + """Non-mp4 sources (e.g. .avi) stage under .mp4 names for VBench.""" + report_dir, _ = staged + avi = tmp_path / "video_x.avi" + avi.write_bytes(b"") + scorer = VBenchScorer( + dataset_name="vid_acc", + dataset=dataset, + report_dir=report_dir, + ground_truth_column="prompt", + vbench_project_path=vbench_project, + ) + staged_dir = report_dir / "vbench_videos" + scorer._stage_videos(staged_dir, [str(avi)], ["a cat"]) + (staged_file,) = staged_dir.iterdir() + assert staged_file.name == "a cat-0.mp4" + assert staged_file.resolve() == avi.resolve() + def test_subprocess_failure_includes_stderr_tail( self, dataset, staged, vbench_project, monkeypatch, tmp_path ): From 0539457f261c63d44db47d8330a479768a6634b0 Mon Sep 17 00:00:00 2001 From: Tin-Yin Lai Date: Wed, 8 Jul 2026 14:05:00 -0700 Subject: [PATCH 2/5] vbench_runner: survive torch>=2.6 checkpoints and missing wget Two cold-start failures observed on aarch64 GB300 runs: - The accuracy venv resolves torch 2.12, where torch.load defaults to weights_only=True (since 2.6). VBench's reference checkpoints (motion_smoothness AMT, RAFT) are full pickles and fail with UnpicklingError. Default torch.load to weights_only=False in this subprocess only; explicit weights_only callers are unaffected. - vbench.utils.init_submodules downloads per-dimension weights via literal wget/unzip subprocesses; without them the run dies with an opaque FileNotFoundError after videos were already generated. Fail fast preflight with a structured error naming the missing tools. --- .../accuracy/vbench_runner.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/examples/09_Wan22_VideoGen_Example/accuracy/vbench_runner.py b/examples/09_Wan22_VideoGen_Example/accuracy/vbench_runner.py index db3c41229..bc461051b 100644 --- a/examples/09_Wan22_VideoGen_Example/accuracy/vbench_runner.py +++ b/examples/09_Wan22_VideoGen_Example/accuracy/vbench_runner.py @@ -27,7 +27,9 @@ """ import argparse +import functools import json +import shutil import sys import traceback from importlib.resources import files as _pkg_files @@ -37,6 +39,25 @@ from vbench import VBench +def _default_torch_load_to_full_pickles() -> None: + """Default torch.load to weights_only=False in this subprocess. + + VBench's reference checkpoints are full pickles and fail under torch>=2.6's + weights_only=True default. Explicit weights_only callers are unaffected. + """ + orig_load = torch.load + + @functools.wraps(orig_load) + def _load(*args, **kwargs): + kwargs.setdefault("weights_only", False) + return orig_load(*args, **kwargs) + + torch.load = _load + + +_default_torch_load_to_full_pickles() + + def _emit_error(exc: BaseException) -> None: """Print a structured JSON error line on stderr for the parent to surface.""" payload = { @@ -85,6 +106,29 @@ def main() -> int: ) args = parser.parse_args() + # vbench downloads weights via wget/unzip subprocesses on a cold cache; + # fail fast instead of a FileNotFoundError mid-evaluation. + missing_tools = [t for t in ("wget", "unzip") if shutil.which(t) is None] + if missing_tools: + print( + json.dumps( + { + "status": "error", + "type": "MissingSystemDependency", + "message": ( + f"Required system tool(s) not found: {', '.join(missing_tools)}. " + "VBench downloads its per-dimension model weights via " + "wget/unzip on first use. Install them in the client " + "environment (e.g. `apt-get install wget unzip`) or " + "pre-populate the VBench cache directory." + ), + } + ), + file=sys.stderr, + flush=True, + ) + return 2 + if not torch.cuda.is_available() and not args.allow_cpu: print( json.dumps( From c0d006d0aee7bc09dfa35cb0888d8c6e74caa169 Mon Sep 17 00:00:00 2001 From: Tin-Yin Lai Date: Thu, 9 Jul 2026 15:35:47 -0700 Subject: [PATCH 3/5] Use TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD instead of wrapping torch.load Same semantics (explicit weights_only callers unaffected, subprocess scoped), no library-function monkeypatch, and torch warns when the escape hatch applies. Addresses review feedback. --- .../accuracy/vbench_runner.py | 25 +++++-------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/examples/09_Wan22_VideoGen_Example/accuracy/vbench_runner.py b/examples/09_Wan22_VideoGen_Example/accuracy/vbench_runner.py index bc461051b..d0212f71a 100644 --- a/examples/09_Wan22_VideoGen_Example/accuracy/vbench_runner.py +++ b/examples/09_Wan22_VideoGen_Example/accuracy/vbench_runner.py @@ -27,8 +27,8 @@ """ import argparse -import functools import json +import os import shutil import sys import traceback @@ -38,24 +38,11 @@ import vbench as _vbench_pkg from vbench import VBench - -def _default_torch_load_to_full_pickles() -> None: - """Default torch.load to weights_only=False in this subprocess. - - VBench's reference checkpoints are full pickles and fail under torch>=2.6's - weights_only=True default. Explicit weights_only callers are unaffected. - """ - orig_load = torch.load - - @functools.wraps(orig_load) - def _load(*args, **kwargs): - kwargs.setdefault("weights_only", False) - return orig_load(*args, **kwargs) - - torch.load = _load - - -_default_torch_load_to_full_pickles() +# VBench's reference checkpoints are full pickles and fail under torch>=2.6's +# weights_only=True default. This escape hatch is read at torch.load call time, +# only applies when the callsite does not set weights_only explicitly, and is +# scoped to this subprocess. +os.environ.setdefault("TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD", "1") def _emit_error(exc: BaseException) -> None: From ce4eb7a783d33b0ab2fae2d2ee472743caed7676 Mon Sep 17 00:00:00 2001 From: Tin-Yin Lai Date: Tue, 14 Jul 2026 15:32:49 -0700 Subject: [PATCH 4/5] feat: bake VBench scorer into image (Dockerfile.dev PROVISION_VBENCH, default on) Folds in the packaging change from #414 so the scorer this PR fixes actually ships in the image. --- scripts/Dockerfile.dev | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/scripts/Dockerfile.dev b/scripts/Dockerfile.dev index c9f62bf75..5b1334d27 100644 --- a/scripts/Dockerfile.dev +++ b/scripts/Dockerfile.dev @@ -2,6 +2,11 @@ # From project root: # docker build -f scripts/Dockerfile.dev --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) -t inference-endpoint-dev . # docker run -v $(pwd):/mnt/inference-endpoint -it --shm-size=512m inference-endpoint-dev bash +# +# VBench (WAN 2.2) accuracy scorer is ON by default (consistent with the DeepSeek-R1 +# evaluator, PROVISION_DSR1=1). It lives under examples/, not src/, so it is COPYed in +# explicitly. To skip it (leaner image, no wan22 accuracy), build with PROVISION_VBENCH=0: +# docker build -f scripts/Dockerfile.dev --build-arg PROVISION_VBENCH=0 --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) -t inference-endpoint-dev . FROM python:3.12.11-slim @@ -19,8 +24,11 @@ ENV PYTHONUNBUFFERED=1 \ # git + curl + ca-certificates support the legacy_mlperf_deepseek_r1 evaluator's setup_eval.sh # (it git-clones prm800k / LiveCodeBench at pinned commits) in the provisioning RUN below; # without them that step fails and mlperf_eval/ is never baked. (eval_accuracy.py is vendored.) +# libgl1 + libglib2.0-0: opencv (cv2), pulled in by the VBench accuracy scorer's video +# reader, needs libGL.so.1 / libgthread at import time (only used when PROVISION_VBENCH=1). RUN apt-get update && \ apt-get install -y --no-install-recommends build-essential procps git curl ca-certificates \ + libgl1 libglib2.0-0 \ && rm -rf /var/lib/apt/lists/* RUN mkdir /mnt/inference-endpoint @@ -38,7 +46,8 @@ RUN if ! getent group ${GROUP_ID}; then \ fi && \ useradd -u ${USER_ID} -g ${GROUP_ID} --create-home --shell /bin/bash appuser --no-log-init && \ chown -R ${USER_ID}:${GROUP_ID} /mnt/inference-endpoint && \ - mkdir -p /opt/venv && chown ${USER_ID}:${GROUP_ID} /opt/venv + mkdir -p /opt/venv /opt/uv-python /opt/vbench_accuracy && \ + chown ${USER_ID}:${GROUP_ID} /opt/venv /opt/uv-python /opt/vbench_accuracy USER appuser ENV PATH="/opt/venv/bin:/home/appuser/.local/bin:$PATH" @@ -77,3 +86,33 @@ RUN if [ "${PROVISION_DSR1}" = "1" ]; then \ else \ echo "PROVISION_DSR1=${PROVISION_DSR1}: skipping DeepSeek-R1 evaluator provisioning" ; \ fi + +# Provision the isolated VBench (WAN 2.2) accuracy scorer. Gated by PROVISION_VBENCH +# (default 1, matching the DeepSeek-R1 evaluator above). VBench lives under examples/ +# (NOT src/), so it is COPYed in explicitly; provisioning materializes the accuracy uv +# project at /opt/vbench_accuracy with its OWN .venv, so wan22 `benchmark --mode acc|both` +# reuses it via UV_NO_SYNC=1 (accuracy configs' vbench_project_path=/opt/vbench_accuracy) +# — no host checkout, no runtime network. Build with --build-arg PROVISION_VBENCH=0 to +# skip it (leaner image that never runs wan22 accuracy). +# +# Same isolation rule as DSR1 above: the vbench project pins Python 3.11 + heavy/old deps +# (torch, decord, vbench 0.1.5 -> transformers 4.33.2 -> tokenizers 0.13.3) that must stay +# OUT of the main /opt/venv, so every uv call overrides UV_PROJECT_ENVIRONMENT to the +# subproject's own .venv. Built on a uv-managed Python 3.11 (persisted at /opt/uv-python) +# because tokenizers 0.13.3 has no cp312 wheel. The pyproject patches relax requires-python +# and swap decord->decord2 so the venv installs from wheels only — required on aarch64 +# (no decord 0.6.0 / tokenizers cp312 wheels), harmless on x86_64. +ARG PROVISION_VBENCH=1 +ENV UV_PYTHON_INSTALL_DIR=/opt/uv-python +COPY --chown=${USER_ID}:${GROUP_ID} examples/09_Wan22_VideoGen_Example/accuracy /opt/vbench_accuracy +RUN if [ "${PROVISION_VBENCH}" = "1" ]; then \ + cd /opt/vbench_accuracy && \ + sed -i 's#requires-python = ">=3.12"#requires-python = ">=3.10"#' pyproject.toml && \ + sed -i 's# "vbench==0.1.5",# "vbench==0.1.5",\n "decord2>=3.4.0",#' pyproject.toml && \ + sed -i "s#^package = false#package = false\noverride-dependencies = [\"decord; sys_platform == 'never'\"]#" pyproject.toml && \ + UV_PROJECT_ENVIRONMENT="$(pwd)/.venv" uv lock --python 3.11 && \ + UV_PROJECT_ENVIRONMENT="$(pwd)/.venv" uv sync --python 3.11 && \ + UV_PROJECT_ENVIRONMENT="$(pwd)/.venv" uv run python -c "import torch, decord, tokenizers, vbench; print('vbench', vbench.__file__, '| decord', decord.__version__, '| tokenizers', tokenizers.__version__, '| torch', torch.__version__)" ; \ + else \ + echo "PROVISION_VBENCH=${PROVISION_VBENCH}: skipping VBench provisioning" ; \ + fi From a33e02b152295e00739209086dca858c8aefae8f Mon Sep 17 00:00:00 2001 From: Tin-Yin Lai Date: Tue, 14 Jul 2026 15:41:05 -0700 Subject: [PATCH 5/5] docs: condense VBench provisioning comment in Dockerfile.dev --- scripts/Dockerfile.dev | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/scripts/Dockerfile.dev b/scripts/Dockerfile.dev index 5b1334d27..7c8910887 100644 --- a/scripts/Dockerfile.dev +++ b/scripts/Dockerfile.dev @@ -87,21 +87,12 @@ RUN if [ "${PROVISION_DSR1}" = "1" ]; then \ echo "PROVISION_DSR1=${PROVISION_DSR1}: skipping DeepSeek-R1 evaluator provisioning" ; \ fi -# Provision the isolated VBench (WAN 2.2) accuracy scorer. Gated by PROVISION_VBENCH -# (default 1, matching the DeepSeek-R1 evaluator above). VBench lives under examples/ -# (NOT src/), so it is COPYed in explicitly; provisioning materializes the accuracy uv -# project at /opt/vbench_accuracy with its OWN .venv, so wan22 `benchmark --mode acc|both` -# reuses it via UV_NO_SYNC=1 (accuracy configs' vbench_project_path=/opt/vbench_accuracy) -# — no host checkout, no runtime network. Build with --build-arg PROVISION_VBENCH=0 to -# skip it (leaner image that never runs wan22 accuracy). -# -# Same isolation rule as DSR1 above: the vbench project pins Python 3.11 + heavy/old deps -# (torch, decord, vbench 0.1.5 -> transformers 4.33.2 -> tokenizers 0.13.3) that must stay -# OUT of the main /opt/venv, so every uv call overrides UV_PROJECT_ENVIRONMENT to the -# subproject's own .venv. Built on a uv-managed Python 3.11 (persisted at /opt/uv-python) -# because tokenizers 0.13.3 has no cp312 wheel. The pyproject patches relax requires-python -# and swap decord->decord2 so the venv installs from wheels only — required on aarch64 -# (no decord 0.6.0 / tokenizers cp312 wheels), harmless on x86_64. +# VBench (WAN 2.2) accuracy scorer, gated by PROVISION_VBENCH (default 1, like PROVISION_DSR1). +# It lives under examples/ (not src/), so COPY it in and build its own .venv at +# /opt/vbench_accuracy; accuracy runs reuse it via UV_NO_SYNC=1. Isolated like DSR1: pinned to +# Python 3.11 (tokenizers 0.13.3 has no cp312 wheel) with UV_PROJECT_ENVIRONMENT set per call so +# its old deps stay out of /opt/venv. pyproject patches (requires-python, decord->decord2) keep +# the install wheel-only on aarch64. PROVISION_VBENCH=0 skips it. ARG PROVISION_VBENCH=1 ENV UV_PYTHON_INSTALL_DIR=/opt/uv-python COPY --chown=${USER_ID}:${GROUP_ID} examples/09_Wan22_VideoGen_Example/accuracy /opt/vbench_accuracy