fix: validate exact D3 preview bundle members

.github/workflows/qar_d3_daily_preview_artifact.yml

@@ -0,0 +1,103 @@
+name: QAR D3 daily preview artifact
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - ".github/workflows/qar_d3_daily_preview_artifact.yml"
+      - "pyproject.toml"
+      - "uv.lock"
+      - "scripts/d3_*.py"
+      - "src/quant_advisor_research/**"
+      - "tests/test_d3_exact_bundle.py"
+      - "examples/political_events.example.csv"
+      - "examples/political_watchlist.example.csv"
+  workflow_dispatch:
+    inputs:
+      as_of:
+        description: "Daily representative fixture as_of"
+        required: false
+        default: "2026-06-20"
+      frozen_generated_at:
+        description: "Harness-only deterministic generated_at"
+        required: false
+        default: "2026-07-15T00:00:00Z"
+
+permissions:
+  contents: read
+
+jobs:
+  build-and-verify:
+    runs-on: ubuntu-latest
+    env:
+      INPUT_AS_OF: ${{ inputs.as_of || '2026-06-20' }}
+      FROZEN_GENERATED_AT: ${{ inputs.frozen_generated_at || '2026-07-15T00:00:00Z' }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
+        with:
+          python-version: "3.11"
+      - name: Set up uv
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        with:
+          version: "0.11.19"
+      - name: Sync locked test environment
+        run: uv sync --locked --extra test
+      - name: Build representative previews
+        id: build
+        env:
+          BASE_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.sha }}
+          HEAD_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+          BUILD_ROOT: ${{ runner.temp }}/qar-d3
+        run: |
+          mkdir -p "$BUILD_ROOT"
+          uv run --no-sync python scripts/d3_build_daily_preview.py \
+            --as-of "$INPUT_AS_OF" --political-events examples/political_events.example.csv \
+            --political-watchlist examples/political_watchlist.example.csv \
+            --frozen-generated-at "$FROZEN_GENERATED_AT" --base-sha "$BASE_SHA" --head-sha "$HEAD_SHA" \
+            --uv-version "$(uv --version)" --lock-path uv.lock --repo-root "$GITHUB_WORKSPACE" \
+            --temp-root "$BUILD_ROOT" --evidence-path "$BUILD_ROOT/build-evidence.json" \
+            --workspace-path-file "$BUILD_ROOT/workspace-path.txt"
+          echo "workspace=$(cat "$BUILD_ROOT/workspace-path.txt")" >> "$GITHUB_OUTPUT"
+          echo "evidence=$BUILD_ROOT/build-evidence.json" >> "$GITHUB_OUTPUT"
+      - name: Upload exact preview members
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: qar-d3-preview-bundle
+          path: |
+            ${{ steps.build.outputs.workspace }}/report.json
+            ${{ steps.build.outputs.workspace }}/report.html
+            ${{ steps.build.outputs.workspace }}/manifest.json
+          include-hidden-files: true
+          if-no-files-found: error
+      - name: Upload build evidence
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: qar-d3-build-evidence
+          path: ${{ steps.build.outputs.evidence }}
+          if-no-files-found: error
+      - name: Download preview members
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        with:
+          name: qar-d3-preview-bundle
+          path: ${{ runner.temp }}/qar-d3-download/bundle
+      - name: Download build evidence
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        with:
+          name: qar-d3-build-evidence
+          path: ${{ runner.temp }}/qar-d3-download/evidence
+      - name: Verify downloaded artifact
+        run: |
+          uv run --no-sync python scripts/d3_verify_daily_preview.py \
+            --workspace "${{ runner.temp }}/qar-d3-download/bundle" \
+            --evidence-path "${{ runner.temp }}/qar-d3-download/evidence/build-evidence.json" \
+            --base-sha "${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.sha }}" \
+            --head-sha "${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}" \
+            --uv-version "$(uv --version)" --lock-path uv.lock --repo-root "$GITHUB_WORKSPACE" \
+            --as-of "$INPUT_AS_OF" --political-events examples/political_events.example.csv \
+            --political-watchlist examples/political_watchlist.example.csv --frozen-generated-at "$FROZEN_GENERATED_AT"
+      - name: Cleanup private preview workspaces
+        if: always()
+        run: rm -rf "${{ runner.temp }}/qar-d3"

scripts/d3_build_daily_preview.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+"""Build two frozen-clock representative daily previews and bind evidence."""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import importlib.metadata
+import json
+import platform
+import re
+import tempfile
+from pathlib import Path
+from unittest.mock import patch
+
+from d3_evidence import DEPENDENCY_INVENTORY, EVIDENCE_VERSION, BundleSnapshot, EvidenceContractError, locked_environment_evidence, repository_file_hashes, validate_exact_bundle, validate_preview_snapshot
+from quant_advisor_research.advisory_report import build_advisory_report
+from quant_advisor_research.preview_workspace import build_preview_workspace
+
+
+def _distributions() -> list[str]:
+    return sorted({f"{name}=={dist.version}" for dist in importlib.metadata.distributions() if (name := dist.metadata.get("Name"))})
+
+
+def _build_report(args: argparse.Namespace) -> dict[str, object]:
+    with patch("quant_advisor_research.advisory_report.utc_now_iso", return_value=args.frozen_generated_at):
+        return build_advisory_report(as_of=args.as_of, cadence="daily", political_events_path=Path(args.political_events), political_watchlist_path=Path(args.political_watchlist))
+
+
+def _bundle_hashes(snapshot: BundleSnapshot) -> dict[str, dict[str, object]]:
+    return {item.name: {"sha256": item.sha256, "size": len(item.content)} for item in snapshot.members}
+
+
+def build_evidence(args: argparse.Namespace) -> None:
+    try:
+        dependency_files = repository_file_hashes(args.repo_root, DEPENDENCY_INVENTORY)
+        lock_sha = hashlib.sha256(Path(args.lock_path).read_bytes()).hexdigest()
+        environment = locked_environment_evidence(lock_sha256=lock_sha, uv_version=args.uv_version, python_version=platform.python_version(), distributions=_distributions())
+        if not re.fullmatch(r"[0-9a-f]{40}", args.base_sha) or not re.fullmatch(r"[0-9a-f]{40}", args.head_sha):
+            raise EvidenceContractError("provenance_sha_invalid")
+        first_parent = Path(tempfile.mkdtemp(prefix="qar-d3-parent-a-", dir=args.temp_root))
+        second_parent = Path(tempfile.mkdtemp(prefix="qar-d3-parent-b-", dir=args.temp_root))
+        first = build_preview_workspace(_build_report(args), first_parent)
+        second = build_preview_workspace(_build_report(args), second_parent)
+        if first == second or first.stat().st_ino == second.stat().st_ino:
+            raise EvidenceContractError("repeat_workspace_not_distinct")
+        first_snapshot = validate_exact_bundle(first)
+        second_snapshot = validate_exact_bundle(second)
+        validate_preview_snapshot(first_snapshot); validate_preview_snapshot(second_snapshot)
+        first_files, second_files = _bundle_hashes(first_snapshot), _bundle_hashes(second_snapshot)
+        if first_files != second_files or any(first_snapshot.member(name).content != second_snapshot.member(name).content for name in ("manifest.json", "report.html", "report.json")):
+            raise EvidenceContractError("repeat_build_not_equal")
+        manifest = json.loads(first_snapshot.member("manifest.json").content.decode("utf-8"))
+        source = {"schema_version": "5", "contract_version": "model_recommendations.v5", "cadence": "daily", "as_of": args.as_of, "generated_at": args.frozen_generated_at}
+        if manifest.get("bundle_contract") != "qar.preview_bundle.v1" or manifest.get("source") != source:
+            raise EvidenceContractError("source_contract_mismatch")
+        evidence = {
+            "evidence_version": EVIDENCE_VERSION, "base_sha": args.base_sha, "head_sha": args.head_sha,
+            "source": {"fixture_paths": [Path(args.political_events).as_posix(), Path(args.political_watchlist).as_posix()], "provenance": "repository_representative_fixture"},
+            "deterministic_clock": {"frozen_generated_at": args.frozen_generated_at, "producer_invocations": 2},
+            "workflow_dependency_inventory": DEPENDENCY_INVENTORY, "dependency_files": dependency_files,
+            "locked_environment": environment,
+            "bundle": {"contract": "qar.preview_bundle.v1", "source": source, "files": first_files},
+            "repeat_build": {"independent_invocations": 2, "bytes_equal": True, "files": second_files},
+        }
+        evidence_path = Path(args.evidence_path)
+        evidence_path.parent.mkdir(parents=True, exist_ok=True)
+        evidence_path.write_text(json.dumps(evidence, sort_keys=True, separators=(",", ":")) + "\n", encoding="utf-8")
+        Path(args.workspace_path_file).write_text(str(first) + "\n", encoding="utf-8")
+        print(json.dumps({"workspace": str(first), "evidence": str(evidence_path), "base_sha": args.base_sha, "head_sha": args.head_sha}, sort_keys=True))
+    except EvidenceContractError as exc:
+        raise RuntimeError(exc.code) from None
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser()
+    for name in ("as-of", "political-events", "political-watchlist", "frozen-generated-at", "base-sha", "head-sha", "uv-version", "lock-path", "repo-root", "temp-root", "evidence-path", "workspace-path-file"):
+        p.add_argument(f"--{name}", required=True)
+    return p.parse_args()
+
+
+if __name__ == "__main__":
+    build_evidence(parse_args())

scripts/d3_evidence.py

@@ -0,0 +1,206 @@
+"""Pure D3 evidence helpers, including the shared exact-bundle validator."""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import stat
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from pathlib import Path
+
+FIXED_FILES = ("manifest.json", "report.html", "report.json")
+EVIDENCE_VERSION = "qar.d3.build_evidence.v4"
+DIST_RE = re.compile(r"^([A-Za-z0-9_.-]+)==([^\s=]+)$")
+DEPENDENCY_INVENTORY = [
+    ".github/workflows/qar_d3_daily_preview_artifact.yml", "pyproject.toml", "uv.lock",
+    "scripts/d3_build_daily_preview.py", "scripts/d3_verify_daily_preview.py", "scripts/d3_evidence.py",
+    "src/quant_advisor_research/advisory_report.py", "src/quant_advisor_research/artifact_integrity.py",
+    "src/quant_advisor_research/artifacts.py", "src/quant_advisor_research/contracts.py",
+    "src/quant_advisor_research/csv_utils.py", "src/quant_advisor_research/period_contract.py",
+    "src/quant_advisor_research/preview_bundle.py", "src/quant_advisor_research/preview_workspace.py",
+    "src/quant_advisor_research/time_contract.py", "tests/test_d3_exact_bundle.py",
+    "examples/political_events.example.csv", "examples/political_watchlist.example.csv",
+]
+
+
+class EvidenceContractError(ValueError):
+    def __init__(self, code: str) -> None:
+        self.code = code
+        super().__init__(code)
+
+
+@dataclass(frozen=True, slots=True)
+class BundleMemberSnapshot:
+    name: str
+    content: bytes
+    sha256: str
+
+
+@dataclass(frozen=True, slots=True)
+class BundleSnapshot:
+    members: tuple[BundleMemberSnapshot, ...]
+
+    def member(self, name: str) -> BundleMemberSnapshot:
+        for item in self.members:
+            if item.name == name:
+                return item
+        raise EvidenceContractError("bundle_member_set_invalid")
+
+
+def _directory_flags() -> int:
+    required = ("O_DIRECTORY", "O_CLOEXEC", "O_NOFOLLOW")
+    if any(not hasattr(os, name) for name in required):
+        raise EvidenceContractError("filesystem_unsupported")
+    return os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC | os.O_NOFOLLOW
+
+
+def _member_flags() -> int:
+    if any(not hasattr(os, name) for name in ("O_CLOEXEC", "O_NOFOLLOW")):
+        raise EvidenceContractError("filesystem_unsupported")
+    return os.O_RDONLY | os.O_NONBLOCK | os.O_CLOEXEC | os.O_NOFOLLOW
+
+
+def validate_exact_bundle(workspace: str | Path) -> BundleSnapshot:
+    """Return an immutable FD-bound snapshot after validating all members."""
+    root_fd = -1
+    try:
+        root_fd = os.open(workspace, _directory_flags())
+        root_info = os.fstat(root_fd)
+        if not stat.S_ISDIR(root_info.st_mode) or stat.S_ISLNK(root_info.st_mode):
+            raise EvidenceContractError("bundle_directory_invalid")
+        names = os.listdir(root_fd)
+        if set(names) != set(FIXED_FILES) or len(names) != len(FIXED_FILES):
+            raise EvidenceContractError("bundle_member_set_invalid")
+        result: list[BundleMemberSnapshot] = []
+        for name in FIXED_FILES:
+            fd = -1
+            try:
+                fd = os.open(name, _member_flags(), dir_fd=root_fd)
+                info = os.fstat(fd)
+                if stat.S_ISLNK(info.st_mode) or not stat.S_ISREG(info.st_mode) or info.st_nlink != 1:
+                    raise EvidenceContractError("bundle_member_invalid")
+                chunks: list[bytes] = []
+                while chunk := os.read(fd, 1024 * 1024):
+                    chunks.append(chunk)
+                content = b"".join(chunks)
+                result.append(BundleMemberSnapshot(name, content, hashlib.sha256(content).hexdigest()))
+            finally:
+                if fd >= 0:
+                    os.close(fd)
+        if set(os.listdir(root_fd)) != set(FIXED_FILES):
+            raise EvidenceContractError("bundle_member_set_invalid")
+        return BundleSnapshot(tuple(result))
+    except EvidenceContractError:
+        raise
+    except (OSError, TypeError, ValueError):
+        raise EvidenceContractError("bundle_member_invalid") from None
+    finally:
+        if root_fd >= 0:
+            os.close(root_fd)
+
+
+def validate_preview_snapshot(snapshot: BundleSnapshot) -> tuple[Mapping[str, object], Mapping[str, object]]:
+    """Readback validation using only the immutable member bytes."""
+    try:
+        from quant_advisor_research.preview_bundle import _canonical_json, _manifest, _render_html, _validated_source
+        report_bytes = snapshot.member("report.json").content
+        html_bytes = snapshot.member("report.html").content
+        manifest_bytes = snapshot.member("manifest.json").content
+        report = json.loads(report_bytes.decode("utf-8"))
+        if not isinstance(report, Mapping):
+            raise EvidenceContractError("readback_invalid")
+        validated = _validated_source(report)
+        if _canonical_json(validated) != report_bytes:
+            raise EvidenceContractError("report_bytes_noncanonical")
+        manifest = json.loads(manifest_bytes.decode("utf-8"))
+        if not isinstance(manifest, Mapping) or manifest != _manifest(validated, report_bytes, html_bytes):
+            raise EvidenceContractError("manifest_mismatch")
+        if html_bytes != _render_html(validated, report_bytes) or html_bytes.count(b'href="report.json"') != 1 or html_bytes.count(b'href="manifest.json"') != 1:
+            raise EvidenceContractError("html_links_invalid")
+        return validated, manifest
+    except EvidenceContractError:
+        raise
+    except (UnicodeError, json.JSONDecodeError, TypeError, ValueError, OverflowError, RecursionError):
+        raise EvidenceContractError("readback_invalid") from None
+
+
+def canonical_distribution_snapshot(values: Iterable[str]) -> tuple[list[str], str]:
+    try:
+        raw = list(values)
+        normalized: dict[str, str] = {}
+        for value in raw:
+            if type(value) is not str or not (match := DIST_RE.fullmatch(value)):
+                raise EvidenceContractError("distribution_snapshot_invalid")
+            name = re.sub(r"[-_.]+", "-", match.group(1)).lower()
+            item = f"{name}=={match.group(2)}"
+            if name in normalized and normalized[name] != item:
+                raise EvidenceContractError("distribution_snapshot_conflict")
+            normalized[name] = item
+        snapshot = sorted(normalized.values())
+        digest = hashlib.sha256("\n".join(snapshot).encode()).hexdigest()
+        return snapshot, digest
+    except EvidenceContractError:
+        raise
+    except (TypeError, ValueError, UnicodeError):
+        raise EvidenceContractError("distribution_snapshot_invalid") from None
+
+
+def locked_environment_evidence(*, lock_sha256: str, uv_version: str, python_version: str, distributions: Iterable[str]) -> dict[str, object]:
+    if type(lock_sha256) is not str or not re.fullmatch(r"[0-9a-f]{64}", lock_sha256):
+        raise EvidenceContractError("lock_digest_invalid")
+    if type(uv_version) is not str or not uv_version.startswith("uv "):
+        raise EvidenceContractError("uv_version_invalid")
+    if type(python_version) is not str or not re.fullmatch(r"3\.11(?:\.\d+)?", python_version):
+        raise EvidenceContractError("python_version_invalid")
+    snapshot, digest = canonical_distribution_snapshot(distributions)
+    return {"lock_sha256": lock_sha256, "uv_version": uv_version, "python_version": python_version, "installed_distributions": snapshot, "installed_distributions_sha256": digest}
+
+
+def repository_file_hashes(repo_root: str | Path, paths: Iterable[str]) -> dict[str, str]:
+    root_fd = -1
+    opened: list[int] = []
+    try:
+        root_fd = os.open(repo_root, _directory_flags())
+        if not stat.S_ISDIR(os.fstat(root_fd).st_mode):
+            raise EvidenceContractError("dependency_root_invalid")
+        result: dict[str, str] = {}
+        for raw in paths:
+            relative = Path(raw)
+            if type(raw) is not str or not raw or relative.is_absolute() or any(part in ("", ".", "..") for part in relative.parts) or raw in result:
+                raise EvidenceContractError("dependency_path_invalid")
+            parent_fd = root_fd
+            traversed: list[int] = []
+            try:
+                for part in relative.parts[:-1]:
+                    next_fd = os.open(part, _directory_flags(), dir_fd=parent_fd)
+                    traversed.append(next_fd)
+                    parent_fd = next_fd
+                file_fd = os.open(relative.parts[-1], _member_flags(), dir_fd=parent_fd)
+                opened.append(file_fd)
+                info = os.fstat(file_fd)
+                if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1:
+                    raise EvidenceContractError("dependency_file_invalid")
+                digest = hashlib.sha256()
+                while chunk := os.read(file_fd, 1024 * 1024):
+                    digest.update(chunk)
+                result[raw] = digest.hexdigest()
+            finally:
+                for fd in reversed(traversed):
+                    os.close(fd)
+        return dict(sorted(result.items()))
+    except EvidenceContractError:
+        raise
+    except (OSError, TypeError, ValueError, UnicodeError):
+        raise EvidenceContractError("dependency_file_invalid") from None
+    finally:
+        for fd in opened:
+            os.close(fd)
+        if root_fd >= 0:
+            os.close(root_fd)
+
+
+def require_exact_locked_environment(value: object, expected: Mapping[str, object]) -> None:
+    if not isinstance(value, Mapping) or dict(value) != dict(expected):
+        raise EvidenceContractError("locked_environment_mismatch")