Surface skipped conformance scenarios as baselined known failures #3099
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Conformance Tests | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| workflow_dispatch: | |
| concurrency: | |
| group: conformance-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| env: | |
| # Pinned conformance harness package spec (passed verbatim to `npx --yes`). | |
| # Use a published version, e.g. @modelcontextprotocol/conformance@0.2.0-alpha.7. | |
| # Bump deliberately and reconcile both | |
| # .github/actions/conformance/expected-failures*.yml files in the same change. | |
| # | |
| # Temporarily pinned to the pkg.pr.new build of conformance main@4944b268 | |
| # (0.2.0-alpha.8, which includes #372: fail checks whose prerequisite is | |
| # missing instead of skipping them) — alpha.8 is not published to npm yet. | |
| # Pinned by commit SHA so the tarball cannot move under us; | |
| # CONFORMANCE_PKG_SHA256 pins the bytes and the fetch-and-verify step below | |
| # downloads, checks the digest, and repoints CONFORMANCE_PKG at the | |
| # verified local copy. Repin to the next published @modelcontextprotocol/ | |
| # conformance release (>=0.2.0-alpha.8) once it ships, then drop | |
| # CONFORMANCE_PKG_SHA256 and the fetch-and-verify steps. | |
| CONFORMANCE_PKG: "https://pkg.pr.new/@modelcontextprotocol/conformance@4944b268" | |
| CONFORMANCE_PKG_SHA256: "0f70c035782d319d72ab427653c5275db5c50429d59fae0241a645b33aeda1a7" | |
| jobs: | |
| server-conformance: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 | |
| with: | |
| persist-credentials: false | |
| - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 | |
| with: | |
| enable-cache: true | |
| version: 0.9.5 | |
| - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 | |
| with: | |
| node-version: 24 | |
| - name: Fetch and verify conformance harness | |
| # Only when CONFORMANCE_PKG is a URL: download, check the recorded | |
| # sha256, and re-point CONFORMANCE_PKG at the verified local tarball. | |
| # When CONFORMANCE_PKG is a registry spec, this step is a no-op (npm's | |
| # own integrity check applies). | |
| run: | | |
| case "$CONFORMANCE_PKG" in | |
| https://*) | |
| curl -fsSL "$CONFORMANCE_PKG" -o /tmp/conformance.tgz | |
| echo "$CONFORMANCE_PKG_SHA256 /tmp/conformance.tgz" | sha256sum -c - | |
| echo "CONFORMANCE_PKG=file:/tmp/conformance.tgz" >> "$GITHUB_ENV" | |
| ;; | |
| esac | |
| - run: uv sync --frozen --all-extras --package mcp-everything-server | |
| - name: Run server conformance (active suite) | |
| run: >- | |
| ./.github/actions/conformance/run-server.sh | |
| --suite active | |
| --expected-failures ./.github/actions/conformance/expected-failures.yml | |
| - name: Run server conformance (draft suite) | |
| run: >- | |
| ./.github/actions/conformance/run-server.sh | |
| --suite draft | |
| --expected-failures ./.github/actions/conformance/expected-failures.yml | |
| - name: Run server conformance (2026-07-28 wire, all suite) | |
| run: >- | |
| ./.github/actions/conformance/run-server.sh | |
| --suite all | |
| --spec-version 2026-07-28 | |
| --expected-failures ./.github/actions/conformance/expected-failures.2026-07-28.yml | |
| - name: Run server conformance (all suite, extension scenarios) | |
| # A bare `--suite all` (no --spec-version) selects every scenario | |
| # shipped with the pinned harness — including the extension-tagged | |
| # tasks-* scenarios and pending-listed ones like server-sse-polling, | |
| # which no other leg reaches (extension scenarios never match a | |
| # --spec-version filter, and the pending list keeps them out of the | |
| # active suite). Running the full set keeps unimplemented surfaces | |
| # visible as baselined known failures in expected-failures.yml instead | |
| # of silent exclusions, and stays robust to scenarios moving between | |
| # harness suite lists across pin bumps. `--suite pending` would cover | |
| # the same union slightly faster; the full set is preferred for the | |
| # self-contained run and for parity with typescript-sdk's CI. | |
| run: >- | |
| ./.github/actions/conformance/run-server.sh | |
| --suite all | |
| --expected-failures ./.github/actions/conformance/expected-failures.yml | |
| client-conformance: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 | |
| with: | |
| persist-credentials: false | |
| - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 | |
| with: | |
| enable-cache: true | |
| version: 0.9.5 | |
| - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 | |
| with: | |
| node-version: 24 | |
| - name: Fetch and verify conformance harness | |
| # Only when CONFORMANCE_PKG is a URL: download, check the recorded | |
| # sha256, and re-point CONFORMANCE_PKG at the verified local tarball. | |
| # When CONFORMANCE_PKG is a registry spec, this step is a no-op (npm's | |
| # own integrity check applies). | |
| run: | | |
| case "$CONFORMANCE_PKG" in | |
| https://*) | |
| curl -fsSL "$CONFORMANCE_PKG" -o /tmp/conformance.tgz | |
| echo "$CONFORMANCE_PKG_SHA256 /tmp/conformance.tgz" | sha256sum -c - | |
| echo "CONFORMANCE_PKG=file:/tmp/conformance.tgz" >> "$GITHUB_ENV" | |
| ;; | |
| esac | |
| - run: uv sync --frozen --all-extras --package mcp | |
| - name: Run client conformance (all suite) | |
| # The harness runs all scenarios via unbounded Promise.all; with 40 | |
| # scenarios on a 2-core runner the slowest one (sse-retry, which has a | |
| # real-time SSE reconnect wait) needs more than the 30s default budget. | |
| run: >- | |
| npx --yes "$CONFORMANCE_PKG" client | |
| --command 'uv run --frozen python .github/actions/conformance/client.py' | |
| --suite all | |
| --timeout 60000 | |
| --expected-failures ./.github/actions/conformance/expected-failures.yml | |
| - name: Run client conformance (2026-07-28 wire, all suite) | |
| run: >- | |
| npx --yes "$CONFORMANCE_PKG" client | |
| --command 'uv run --frozen python .github/actions/conformance/client.py' | |
| --suite all | |
| --timeout 60000 | |
| --spec-version 2026-07-28 | |
| --expected-failures ./.github/actions/conformance/expected-failures.2026-07-28.yml |