diff --git a/src/index.ts b/src/index.ts index 0e8b14f4..b82093c3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -594,7 +594,8 @@ program validated, validated.scenario, details, - outputDir + outputDir, + specVersionFilter ); const { failed } = printAuthorizationServerResults( @@ -628,7 +629,8 @@ program validated, scenarioName, details, - outputDir + outputDir, + specVersionFilter ); if ( result.checks[0].status === 'SUCCESS' && diff --git a/src/runner/authorization-server.ts b/src/runner/authorization-server.ts index 6d10c9e3..15d0ad58 100644 --- a/src/runner/authorization-server.ts +++ b/src/runner/authorization-server.ts @@ -1,7 +1,10 @@ import { promises as fs } from 'fs'; import path from 'path'; -import { ConformanceCheck } from '../types'; -import { getClientScenarioForAuthorizationServer } from '../scenarios'; +import { ConformanceCheck, SpecVersion } from '../types'; +import { + getClientScenarioForAuthorizationServer, + matchesSpecVersion +} from '../scenarios'; import { createResultDir, formatPrettyChecks } from './utils'; import { AuthorizationServerOptions } from '../schemas'; @@ -9,7 +12,8 @@ export async function runAuthorizationServerConformanceTest( options: AuthorizationServerOptions, scenarioName: string, details: Record, - outputDir?: string + outputDir?: string, + specVersion?: SpecVersion ): Promise<{ checks: ConformanceCheck[]; resultDir?: string; @@ -34,18 +38,23 @@ export async function runAuthorizationServerConformanceTest( ); const checks = await scenario.run(options, details); + const filtered = specVersion + ? checks.filter( + (c) => !c.source || matchesSpecVersion(c.source, specVersion) + ) + : checks; if (resultDir) { await fs.writeFile( path.join(resultDir, 'checks.json'), - JSON.stringify(checks, null, 2) + JSON.stringify(filtered, null, 2) ); console.log(`Results saved to ${resultDir}`); } return { - checks, + checks: filtered, resultDir, scenarioDescription: scenario.description }; diff --git a/src/scenarios/authorization-server/authorization-server-metadata.test.ts b/src/scenarios/authorization-server/authorization-server-metadata.test.ts index fef6fa58..a1a5d669 100644 --- a/src/scenarios/authorization-server/authorization-server-metadata.test.ts +++ b/src/scenarios/authorization-server/authorization-server-metadata.test.ts @@ -42,7 +42,7 @@ describe('AuthorizationServerMetadataEndpointScenario', () => { const checks = await scenario.run(OPTIONS, details); - expect(checks).toHaveLength(1); + expect(checks).toHaveLength(2); const check = checks[0]; expect(check.status).toBe('SUCCESS'); @@ -73,7 +73,7 @@ describe('AuthorizationServerMetadataEndpointScenario', () => { const checks = await scenario.run(OPTIONS, details); - expect(checks).toHaveLength(1); + expect(checks).toHaveLength(2); const check = checks[0]; expect(check.status).toBe('FAILURE'); @@ -89,12 +89,79 @@ describe('AuthorizationServerMetadataEndpointScenario', () => { const checks = await scenario.run(OPTIONS, details); - expect(checks).toHaveLength(1); + expect(checks).toHaveLength(2); const check = checks[0]; expect(check.status).toBe('FAILURE'); expect(check.errorMessage).toContain('code_challenge_methods_supported'); }); + + it('returns SUCCESS for CIMD check when server metadata includes client_id_metadata_document_supported=true', async () => { + const scenario = new AuthorizationServerMetadataEndpointScenario(); + mockMetadataResponse({ + ...validMetadata, + client_id_metadata_document_supported: true + }); + + const checks = await scenario.run(OPTIONS, details); + + expect(checks).toHaveLength(2); + + const metadataCheck = checks[0]; + expect(metadataCheck.status).toBe('SUCCESS'); + + const cimdCheck = checks[1]; + expect(cimdCheck.id).toBe('authorization-server-metadata-cimd'); + expect(cimdCheck.status).toBe('SUCCESS'); + expect(cimdCheck.errorMessage).toBeUndefined(); + expect(cimdCheck.details).toEqual({ + client_id_metadata_document_supported: true + }); + }); + + it('returns WARNING for CIMD check when server metadata lacks client_id_metadata_document_supported', async () => { + const scenario = new AuthorizationServerMetadataEndpointScenario(); + mockMetadataResponse(validMetadata); + + const checks = await scenario.run(OPTIONS, details); + + expect(checks).toHaveLength(2); + + const metadataCheck = checks[0]; + expect(metadataCheck.status).toBe('SUCCESS'); + + const cimdCheck = checks[1]; + expect(cimdCheck.id).toBe('authorization-server-metadata-cimd'); + expect(cimdCheck.status).toBe('WARNING'); + expect(cimdCheck.source).toEqual({ introducedIn: '2025-11-25' }); + expect(cimdCheck.errorMessage).toContain( + 'client_id_metadata_document_supported' + ); + }); + + it('returns WARNING for CIMD check when client_id_metadata_document_supported is false', async () => { + const scenario = new AuthorizationServerMetadataEndpointScenario(); + mockMetadataResponse({ + ...validMetadata, + client_id_metadata_document_supported: false + }); + + const checks = await scenario.run(OPTIONS, details); + + expect(checks).toHaveLength(2); + + const metadataCheck = checks[0]; + expect(metadataCheck.status).toBe('SUCCESS'); + + const cimdCheck = checks[1]; + expect(cimdCheck.id).toBe('authorization-server-metadata-cimd'); + expect(cimdCheck.status).toBe('WARNING'); + expect(cimdCheck.source).toEqual({ introducedIn: '2025-11-25' }); + expect(cimdCheck.errorMessage).toContain( + 'client_id_metadata_document_supported' + ); + expect(cimdCheck.errorMessage).toContain('false'); + }); }); describe('AuthorizationServerOptionsSchema', () => { diff --git a/src/scenarios/authorization-server/authorization-server-metadata.ts b/src/scenarios/authorization-server/authorization-server-metadata.ts index cd656031..ec5fd3a3 100644 --- a/src/scenarios/authorization-server/authorization-server-metadata.ts +++ b/src/scenarios/authorization-server/authorization-server-metadata.ts @@ -3,11 +3,13 @@ */ import { AuthorizationServerOptions } from '../../schemas'; import { + CheckStatus, ClientScenarioForAuthorizationServer, ConformanceCheck } from '../../types'; import { request } from 'undici'; import { SpecReferences } from '../authorization-server/auth/spec-references'; +import { SpecReferences as ClientSpecReferences } from '../client/auth/spec-references'; type Status = 'SUCCESS' | 'FAILURE'; @@ -24,7 +26,8 @@ export class AuthorizationServerMetadataEndpointScenario implements ClientScenar - HTTP response status code MUST be 200 OK - Content-Type header MUST be application/json - Return a JSON response including issuer, authorization_endpoint, token_endpoint and response_types_supported -- The issuer value MUST match the URI obtained by removing the well-known URI string from the authorization server metadata URI.`; +- The issuer value MUST match the URI obtained by removing the well-known URI string from the authorization server metadata URI. +- (2025-11-25+) SHOULD include client_id_metadata_document_supported=true (Client ID Metadata Document support)`; async run( options: AuthorizationServerOptions, @@ -34,6 +37,7 @@ export class AuthorizationServerMetadataEndpointScenario implements ClientScenar let errorMessage: string | undefined; let details: any; let response: any | null = null; + let body: Record | undefined; try { const wellKnownUrls = this.createWellKnownUrl(options.url); @@ -57,7 +61,7 @@ export class AuthorizationServerMetadataEndpointScenario implements ClientScenar this.validateContentType(response.headers['content-type']); - const body = await this.parseJson(response); + body = await this.parseJson(response); const errors: string[] = []; this.validateMetadataBody(body, options.url, errors); @@ -75,7 +79,7 @@ export class AuthorizationServerMetadataEndpointScenario implements ClientScenar errorMessage = error instanceof Error ? error.message : String(error); } - return [ + const checks: ConformanceCheck[] = [ { id: 'authorization-server-metadata', name: 'AuthorizationServerMetadata', @@ -87,6 +91,38 @@ export class AuthorizationServerMetadataEndpointScenario implements ClientScenar ...(details ? { details } : {}) } ]; + + if (body) { + const cimdSupported = body.client_id_metadata_document_supported; + const cimdStatus: CheckStatus = + cimdSupported === true ? 'SUCCESS' : 'WARNING'; + const cimdErrorMessage = + cimdSupported === true + ? undefined + : cimdSupported === undefined + ? 'Authorization server metadata does not include "client_id_metadata_document_supported"' + : `Expected "client_id_metadata_document_supported" to be true, got ${JSON.stringify(cimdSupported)}`; + + checks.push({ + id: 'authorization-server-metadata-cimd', + name: 'AuthorizationServerMetadataCIMD', + description: + 'Authorization server metadata includes client_id_metadata_document_supported=true (Client ID Metadata Document support)', + status: cimdStatus, + source: { introducedIn: '2025-11-25' } as const, + timestamp: new Date().toISOString(), + errorMessage: cimdErrorMessage, + specReferences: [ + ClientSpecReferences.MCP_CLIENT_ID_METADATA_DOCUMENTS, + ClientSpecReferences.IETF_CIMD + ], + details: { + client_id_metadata_document_supported: cimdSupported + } + }); + } + + return checks; } private createWellKnownUrl(serverUrl: string): string[] { diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index 9db6351f..db1584ea 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -428,7 +428,7 @@ function versionIndex( } // Off-timeline sources (extensions etc.) are never selected by --spec-version. -function matchesSpecVersion( +export function matchesSpecVersion( source: ScenarioSource, version: SpecVersion ): boolean { diff --git a/src/scenarios/spec-version.test.ts b/src/scenarios/spec-version.test.ts index dc84b0e9..7e5f4517 100644 --- a/src/scenarios/spec-version.test.ts +++ b/src/scenarios/spec-version.test.ts @@ -8,6 +8,7 @@ import { listExtensionScenarios, getScenario, getScenarioSpecVersions, + matchesSpecVersion, resolveSpecVersion, ALL_SPEC_VERSIONS, scenarios, @@ -92,6 +93,17 @@ describe('specVersions helpers', () => { expect(resolveSpecVersion(LATEST_SPEC_VERSION)).toBe(LATEST_SPEC_VERSION); }); + describe('matchesSpecVersion (per-check gating)', () => { + const src = { introducedIn: '2025-11-25' } as const; + it.each(['2025-11-25', DRAFT_PROTOCOL_VERSION] as const)( + 'includes %s', + (v) => expect(matchesSpecVersion(src, v)).toBe(true) + ); + it.each(['2025-03-26', '2025-06-18'] as const)('excludes %s', (v) => + expect(matchesSpecVersion(src, v)).toBe(false) + ); + }); + it('extension-tagged scenarios are not selected by any --spec-version', () => { for (const version of ALL_SPEC_VERSIONS) { const selected = new Set(listScenariosForSpec(version)); diff --git a/src/types.ts b/src/types.ts index 4ebdd60f..eaa364b3 100644 --- a/src/types.ts +++ b/src/types.ts @@ -21,6 +21,11 @@ export interface ConformanceCheck { status: CheckStatus; timestamp: string; specReferences?: SpecReference[]; + /** + * Optional spec-version range for this individual check. When set, runners + * drop the check for `--spec-version` values outside the range. + */ + source?: ScenarioSource; details?: Record; metadata?: Record; errorMessage?: string;