diff --git a/examples/09_Wan22_VideoGen_Example/accuracy/vbench_runner.py b/examples/09_Wan22_VideoGen_Example/accuracy/vbench_runner.py index db3c41229..d0212f71a 100644 --- a/examples/09_Wan22_VideoGen_Example/accuracy/vbench_runner.py +++ b/examples/09_Wan22_VideoGen_Example/accuracy/vbench_runner.py @@ -28,6 +28,8 @@ import argparse import json +import os +import shutil import sys import traceback from importlib.resources import files as _pkg_files @@ -36,6 +38,12 @@ import vbench as _vbench_pkg from vbench import VBench +# VBench's reference checkpoints are full pickles and fail under torch>=2.6's +# weights_only=True default. This escape hatch is read at torch.load call time, +# only applies when the callsite does not set weights_only explicitly, and is +# scoped to this subprocess. +os.environ.setdefault("TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD", "1") + def _emit_error(exc: BaseException) -> None: """Print a structured JSON error line on stderr for the parent to surface.""" @@ -85,6 +93,29 @@ def main() -> int: ) args = parser.parse_args() + # vbench downloads weights via wget/unzip subprocesses on a cold cache; + # fail fast instead of a FileNotFoundError mid-evaluation. + missing_tools = [t for t in ("wget", "unzip") if shutil.which(t) is None] + if missing_tools: + print( + json.dumps( + { + "status": "error", + "type": "MissingSystemDependency", + "message": ( + f"Required system tool(s) not found: {', '.join(missing_tools)}. " + "VBench downloads its per-dimension model weights via " + "wget/unzip on first use. Install them in the client " + "environment (e.g. `apt-get install wget unzip`) or " + "pre-populate the VBench cache directory." + ), + } + ), + file=sys.stderr, + flush=True, + ) + return 2 + if not torch.cuda.is_available() and not args.allow_cpu: print( json.dumps( diff --git a/scripts/Dockerfile.dev b/scripts/Dockerfile.dev index c9f62bf75..7c8910887 100644 --- a/scripts/Dockerfile.dev +++ b/scripts/Dockerfile.dev @@ -2,6 +2,11 @@ # From project root: # docker build -f scripts/Dockerfile.dev --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) -t inference-endpoint-dev . # docker run -v $(pwd):/mnt/inference-endpoint -it --shm-size=512m inference-endpoint-dev bash +# +# VBench (WAN 2.2) accuracy scorer is ON by default (consistent with the DeepSeek-R1 +# evaluator, PROVISION_DSR1=1). It lives under examples/, not src/, so it is COPYed in +# explicitly. To skip it (leaner image, no wan22 accuracy), build with PROVISION_VBENCH=0: +# docker build -f scripts/Dockerfile.dev --build-arg PROVISION_VBENCH=0 --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) -t inference-endpoint-dev . FROM python:3.12.11-slim @@ -19,8 +24,11 @@ ENV PYTHONUNBUFFERED=1 \ # git + curl + ca-certificates support the legacy_mlperf_deepseek_r1 evaluator's setup_eval.sh # (it git-clones prm800k / LiveCodeBench at pinned commits) in the provisioning RUN below; # without them that step fails and mlperf_eval/ is never baked. (eval_accuracy.py is vendored.) +# libgl1 + libglib2.0-0: opencv (cv2), pulled in by the VBench accuracy scorer's video +# reader, needs libGL.so.1 / libgthread at import time (only used when PROVISION_VBENCH=1). RUN apt-get update && \ apt-get install -y --no-install-recommends build-essential procps git curl ca-certificates \ + libgl1 libglib2.0-0 \ && rm -rf /var/lib/apt/lists/* RUN mkdir /mnt/inference-endpoint @@ -38,7 +46,8 @@ RUN if ! getent group ${GROUP_ID}; then \ fi && \ useradd -u ${USER_ID} -g ${GROUP_ID} --create-home --shell /bin/bash appuser --no-log-init && \ chown -R ${USER_ID}:${GROUP_ID} /mnt/inference-endpoint && \ - mkdir -p /opt/venv && chown ${USER_ID}:${GROUP_ID} /opt/venv + mkdir -p /opt/venv /opt/uv-python /opt/vbench_accuracy && \ + chown ${USER_ID}:${GROUP_ID} /opt/venv /opt/uv-python /opt/vbench_accuracy USER appuser ENV PATH="/opt/venv/bin:/home/appuser/.local/bin:$PATH" @@ -77,3 +86,24 @@ RUN if [ "${PROVISION_DSR1}" = "1" ]; then \ else \ echo "PROVISION_DSR1=${PROVISION_DSR1}: skipping DeepSeek-R1 evaluator provisioning" ; \ fi + +# VBench (WAN 2.2) accuracy scorer, gated by PROVISION_VBENCH (default 1, like PROVISION_DSR1). +# It lives under examples/ (not src/), so COPY it in and build its own .venv at +# /opt/vbench_accuracy; accuracy runs reuse it via UV_NO_SYNC=1. Isolated like DSR1: pinned to +# Python 3.11 (tokenizers 0.13.3 has no cp312 wheel) with UV_PROJECT_ENVIRONMENT set per call so +# its old deps stay out of /opt/venv. pyproject patches (requires-python, decord->decord2) keep +# the install wheel-only on aarch64. PROVISION_VBENCH=0 skips it. +ARG PROVISION_VBENCH=1 +ENV UV_PYTHON_INSTALL_DIR=/opt/uv-python +COPY --chown=${USER_ID}:${GROUP_ID} examples/09_Wan22_VideoGen_Example/accuracy /opt/vbench_accuracy +RUN if [ "${PROVISION_VBENCH}" = "1" ]; then \ + cd /opt/vbench_accuracy && \ + sed -i 's#requires-python = ">=3.12"#requires-python = ">=3.10"#' pyproject.toml && \ + sed -i 's# "vbench==0.1.5",# "vbench==0.1.5",\n "decord2>=3.4.0",#' pyproject.toml && \ + sed -i "s#^package = false#package = false\noverride-dependencies = [\"decord; sys_platform == 'never'\"]#" pyproject.toml && \ + UV_PROJECT_ENVIRONMENT="$(pwd)/.venv" uv lock --python 3.11 && \ + UV_PROJECT_ENVIRONMENT="$(pwd)/.venv" uv sync --python 3.11 && \ + UV_PROJECT_ENVIRONMENT="$(pwd)/.venv" uv run python -c "import torch, decord, tokenizers, vbench; print('vbench', vbench.__file__, '| decord', decord.__version__, '| tokenizers', tokenizers.__version__, '| torch', torch.__version__)" ; \ + else \ + echo "PROVISION_VBENCH=${PROVISION_VBENCH}: skipping VBench provisioning" ; \ + fi diff --git a/src/inference_endpoint/evaluation/scoring.py b/src/inference_endpoint/evaluation/scoring.py index 8c4a49998..975b7f47c 100644 --- a/src/inference_endpoint/evaluation/scoring.py +++ b/src/inference_endpoint/evaluation/scoring.py @@ -1495,7 +1495,9 @@ def _stage_videos( # strict=True surfaces missing/unmounted sources here, not as an # opaque decord read failure inside VBench 30 minutes later. resolved_src = src.resolve(strict=True) - dst = staged_dir / f"{safe_prompt}-{idx}{src.suffix or '.mp4'}" + # Always .mp4: VBench dispatches on extension (non-mp4 raises + # NotImplementedError); decord detects the container by content. + dst = staged_dir / f"{safe_prompt}-{idx}.mp4" dst.symlink_to(resolved_src) def _run_vbench_subprocess( diff --git a/tests/unit/evaluation/test_scoring.py b/tests/unit/evaluation/test_scoring.py index 92f14a93d..4ec515e85 100644 --- a/tests/unit/evaluation/test_scoring.py +++ b/tests/unit/evaluation/test_scoring.py @@ -774,6 +774,26 @@ def test_stage_clears_stale_files_from_prior_run( assert names == ["a cat-0.mp4", "a dog-0.mp4", "a tree-0.mp4"] assert not zombie.exists() + def test_stage_videos_renames_non_mp4_sources_to_mp4( + self, dataset, staged, vbench_project, tmp_path + ): + """Non-mp4 sources (e.g. .avi) stage under .mp4 names for VBench.""" + report_dir, _ = staged + avi = tmp_path / "video_x.avi" + avi.write_bytes(b"") + scorer = VBenchScorer( + dataset_name="vid_acc", + dataset=dataset, + report_dir=report_dir, + ground_truth_column="prompt", + vbench_project_path=vbench_project, + ) + staged_dir = report_dir / "vbench_videos" + scorer._stage_videos(staged_dir, [str(avi)], ["a cat"]) + (staged_file,) = staged_dir.iterdir() + assert staged_file.name == "a cat-0.mp4" + assert staged_file.resolve() == avi.resolve() + def test_subprocess_failure_includes_stderr_tail( self, dataset, staged, vbench_project, monkeypatch, tmp_path ):