Surface skipped conformance scenarios as baselined known failures (#3030)

Author: maxisbey

.github/actions/conformance/expected-failures.2026-07-28.yml

@@ -22,4 +22,10 @@
 
 client: []
 
-server: []
+server:
+  # SEP-2575 subscriptions/listen is not implemented yet; see the matching
+  # entry in expected-failures.yml for the full rationale.
+  - server-stateless
+  # SEP-2243 Mcp-Param-* server-side validation is not implemented yet; see
+  # the matching entry in expected-failures.yml for the full rationale.
+  - http-custom-header-server-validation

.github/actions/conformance/expected-failures.yml

@@ -12,4 +12,40 @@
 
 client: []
 
-server: []
+server:
+  # SEP-2575 subscriptions/listen is not implemented yet. The everything-
+  # server's legacy resources/subscribe handlers make it advertise
+  # `resources.subscribe` in server/discover, and as of conformance #372 a
+  # server that advertises a subscription capability but answers
+  # subscriptions/listen with -32601 fails the three listen MUST checks
+  # ("Not testable") instead of skipping them. Remove this entry when the
+  # listen runtime lands. NOTE: while listed, this entry also masks new
+  # failures in the scenario's other 25 (currently passing) checks — the
+  # baseline is per-scenario, not per-check.
+  - server-stateless
+  # SEP-2243 Mcp-Param-* server-side validation is not implemented yet. The
+  # everything-server's `test_x_mcp_header` tool arms these checks (without an
+  # x-mcp-header-annotated tool the harness skips all of them silently); the
+  # accept-path checks pass, the reject-path checks fail until the server
+  # validates Mcp-Param headers against body params. Read by the draft leg and
+  # the bare `--suite all` leg; the 2026-07-28 leg carries its own entry.
+  - http-custom-header-server-validation
+  # SEP-2663 (io.modelcontextprotocol/tasks): the SDK does not implement the
+  # tasks extension yet. These extension-tagged scenarios are selected only by
+  # the bare `--suite all` leg — extension scenarios never match a
+  # --spec-version filter and the active/draft suites exclude them — so these
+  # entries are inert for the other legs that read this file.
+  #
+  # `tasks-status-notifications` is intentionally NOT listed: the harness
+  # skips it unconditionally (pending its rewrite against subscriptions/
+  # listen), and a baseline entry for a scenario with no failing checks is
+  # flagged stale.
+  - tasks-lifecycle
+  - tasks-capability-negotiation
+  - tasks-wire-fields
+  - tasks-request-state-removal
+  - tasks-mrtr-input
+  - tasks-request-headers
+  - tasks-dispatch-and-envelope
+  - tasks-required-task-error
+  - tasks-mrtr-composition

.github/workflows/conformance.yml

@@ -19,17 +19,17 @@ env:
   # Bump deliberately and reconcile both
   # .github/actions/conformance/expected-failures*.yml files in the same change.
   #
-  # Temporarily pinned to the pkg.pr.new build of conformance main@b18aa918
-  # (the merge of #371, which fixes the http-custom-headers fixture's
-  # spec-forbidden `number`-typed x-mcp-header annotations) — no published
-  # release includes it yet. Pinned by commit SHA so the tarball cannot move
-  # under us; CONFORMANCE_PKG_SHA256 pins the bytes and the fetch-and-verify
-  # step below downloads, checks the digest, and repoints CONFORMANCE_PKG at the
+  # Temporarily pinned to the pkg.pr.new build of conformance main@4944b268
+  # (0.2.0-alpha.8, which includes #372: fail checks whose prerequisite is
+  # missing instead of skipping them) — alpha.8 is not published to npm yet.
+  # Pinned by commit SHA so the tarball cannot move under us;
+  # CONFORMANCE_PKG_SHA256 pins the bytes and the fetch-and-verify step below
+  # downloads, checks the digest, and repoints CONFORMANCE_PKG at the
   # verified local copy. Repin to the next published @modelcontextprotocol/
-  # conformance release (>0.2.0-alpha.7) once it ships, then drop
+  # conformance release (>=0.2.0-alpha.8) once it ships, then drop
   # CONFORMANCE_PKG_SHA256 and the fetch-and-verify steps.
-  CONFORMANCE_PKG: "https://pkg.pr.new/@modelcontextprotocol/conformance@b18aa918"
-  CONFORMANCE_PKG_SHA256: "e9f6bc25085b4692e988cbdbd024a4203d54a52a6aaa065376cf8ecaa09bb680"
+  CONFORMANCE_PKG: "https://pkg.pr.new/@modelcontextprotocol/conformance@4944b268"
+  CONFORMANCE_PKG_SHA256: "0f70c035782d319d72ab427653c5275db5c50429d59fae0241a645b33aeda1a7"
 
 jobs:
   server-conformance:
@@ -75,6 +75,22 @@ jobs:
           --suite all
           --spec-version 2026-07-28
           --expected-failures ./.github/actions/conformance/expected-failures.2026-07-28.yml
+      - name: Run server conformance (all suite, extension scenarios)
+        # A bare `--suite all` (no --spec-version) selects every scenario
+        # shipped with the pinned harness — including the extension-tagged
+        # tasks-* scenarios and pending-listed ones like server-sse-polling,
+        # which no other leg reaches (extension scenarios never match a
+        # --spec-version filter, and the pending list keeps them out of the
+        # active suite). Running the full set keeps unimplemented surfaces
+        # visible as baselined known failures in expected-failures.yml instead
+        # of silent exclusions, and stays robust to scenarios moving between
+        # harness suite lists across pin bumps. `--suite pending` would cover
+        # the same union slightly faster; the full set is preferred for the
+        # self-contained run and for parity with typescript-sdk's CI.
+        run: >-
+          ./.github/actions/conformance/run-server.sh
+          --suite all
+          --expected-failures ./.github/actions/conformance/expected-failures.yml
 
   client-conformance:
     runs-on: ubuntu-latest

examples/servers/everything-server/mcp_everything_server/server.py

@@ -11,7 +11,7 @@
 import hmac
 import json
 import logging
-from typing import Any
+from typing import Annotated, Any
 
 import click
 from mcp.server import ServerRequestContext
@@ -327,6 +327,24 @@ def test_error_handling() -> str:
     raise RuntimeError("This tool intentionally returns an error for testing")
 
 
+@mcp.tool()
+def test_x_mcp_header(
+    region: Annotated[
+        str,
+        Field(
+            description="Mirrored into the Mcp-Param-Region header",
+            json_schema_extra={"x-mcp-header": "Region"},
+        ),
+    ] = "<none>",
+) -> str:
+    """Tests SEP-2243 Mcp-Param-* server-side validation.
+
+    Arms the http-custom-header-server-validation conformance scenario, which
+    skips when no tool with an `x-mcp-header` annotation is found.
+    """
+    return f"region={region}"
+
+
 @mcp.tool()
 async def test_missing_capability(ctx: Context) -> str:
     """Tests that a handler-raised MISSING_REQUIRED_CLIENT_CAPABILITY surfaces as a top-level JSON-RPC error.