diff --git a/src/winml/modelkit/analyze/core/rules_prefilter.py b/src/winml/modelkit/analyze/core/rules_prefilter.py new file mode 100644 index 000000000..d4a4950a2 --- /dev/null +++ b/src/winml/modelkit/analyze/core/rules_prefilter.py @@ -0,0 +1,94 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- +"""Rules-based prefilter utilities for runtime-check skip decisions.""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from .runtime_checker_query import RuntimeCheckerQuery + + +if TYPE_CHECKING: + import onnx + + from ...utils.constants import EPName + + +logger = logging.getLogger(__name__) + + +class RuntimeCheckerRulesPrefilter: + """Fixed rules-prefilter service bound to one EP+device pair.""" + + def __init__(self, ep_name: EPName, device_type: str) -> None: + self._ep_name = ep_name + self._device_type = device_type + + def build_skip_check_result_for_rules_all_nodes_compile_run_pass( + self, + onnx_model: onnx.ModelProto, + ) -> dict[str, Any] | None: + """Build synthetic check_result when rules indicate all nodes pass.""" + return build_skip_check_result_for_rules_all_nodes_compile_run_pass( + onnx_model, + self._ep_name, + self._device_type, + ) + + +def build_skip_check_result_for_rules_all_nodes_compile_run_pass( + onnx_model: onnx.ModelProto, + ep_name: EPName, + device_type: str, +) -> dict[str, Any] | None: + """Return synthetic check_result when all nodes have rules compile/run pass. + + Returns None when rules data is missing, any node is unsupported, + or prefilter evaluation fails. Caller should then run real compile/run. + """ + try: + query = RuntimeCheckerQuery( + model_proto=onnx_model, + ep_name=ep_name, + device_type=device_type, + ) + node_results = [ + query.run_for_node(node, run_unknown_op=False) for node in query.model_proto.graph.node + ] + + if not node_results: + return None + + all_supported = all( + result.result.compile and result.result.run and not result.result.no_data + for result in node_results + ) + if not all_supported: + return None + + reason = f"rules_all_nodes_compile_run_pass_{len(node_results)}_nodes" + return { + "compile": { + "result": {"success": True, "reason": reason}, + "stdout": "skipped_by_rules_all_nodes_compile_run_pass", + "stderr": "", + }, + "run": { + "result": {"success": True, "reason": reason}, + "stdout": "skipped_by_rules_all_nodes_compile_run_pass", + "stderr": "", + }, + } + except Exception as exc: + logger.warning("Rules prefilter failed for one case (%s); fallback to ep_checker.", exc) + return None + + +__all__ = [ + "RuntimeCheckerRulesPrefilter", + "build_skip_check_result_for_rules_all_nodes_compile_run_pass", +] diff --git a/src/winml/modelkit/analyze/pattern/check_patterns.py b/src/winml/modelkit/analyze/pattern/check_patterns.py index e5594f6fc..ebcb64bf9 100644 --- a/src/winml/modelkit/analyze/pattern/check_patterns.py +++ b/src/winml/modelkit/analyze/pattern/check_patterns.py @@ -27,7 +27,6 @@ get_pattern_input_generator, get_registered_pattern_input_generators, ) -from ...sysinfo import SysInfo if TYPE_CHECKING: @@ -38,6 +37,7 @@ from ...utils.constants import EPName from ...utils import constants +from ..core.rules_prefilter import RuntimeCheckerRulesPrefilter from ..runtime_checker.ep_checker import EPChecker from ..utils import CheckResultWriter, load_case_indices_from_conflict_file @@ -84,8 +84,8 @@ def check_patterns( Returns: Dictionary mapping pattern names to their test results: { - "Gelu": {"check_results": [...], "sys_info": {...}, "output_path": "..."}, - "MatMulAdd": {"check_results": [...], "sys_info": {...}, "output_path": "..."}, + "Gelu": {"check_results": [...], "output_path": "..."}, + "MatMulAdd": {"check_results": [...], "output_path": "..."}, ... } """ @@ -95,8 +95,6 @@ def check_patterns( case_index = load_case_indices_from_conflict_file(conflict_file) print(f"Loaded {len(case_index)} case_index values from conflict file: {conflict_file}") - sys_info = SysInfo().to_dict() - # Create output directory if it doesn't exist output_dir = Path(output_dir) output_dir.mkdir(parents=True, exist_ok=True) @@ -112,6 +110,13 @@ def check_patterns( ONNXDomain.from_str(domain): version for domain, version in opset_mapping.items() } + ep_checker.set_rules_prefilter( + RuntimeCheckerRulesPrefilter( + ep_name=ep_checker.ep_name, + device_type=ep_checker.device_type.name, + ) + ) + # Test each pattern for pattern_name in patterns: print(f"\n{'=' * 80}") @@ -151,7 +156,6 @@ def check_patterns( # Use writer as context manager (auto-flushes on exit) with CheckResultWriter( output_path, - sys_info, save_per_cases=None if dry_run else 20, rerun_failed=rerun_failed, delta_only=delta_only, @@ -162,6 +166,11 @@ def check_patterns( print(f"Running {pattern_name} tests on {ep_checker.ep_name}...") if n_cases is not None: print(f"Limiting to first {n_cases} test cases") + if not dry_run: + print( + "Rules-first mode: check node support via parquet rules before ep_checker; " + "only fallback to real compile/run when any node is unsupported/no_data." + ) check_results_iter = gen.check_on_ep( ep_checker, @@ -170,7 +179,6 @@ def check_patterns( skip_cases=0, save_failed_model=save_failed_model, skip_signature_fn=writer.should_skip_case, - yield_skipped=True, dry_run=dry_run, ) @@ -209,7 +217,6 @@ def check_patterns( # Store results for return all_results[pattern_name] = { "check_results": check_results, - "sys_info": sys_info, "output_path": str(output_path), } @@ -234,6 +241,19 @@ def __init__(self, device_type: ort.OrtHardwareDeviceType) -> None: super().__init__(ep_name="QNNExecutionProvider", device_type=device_type) +class RTXChecker(EPChecker): + """NVIDIA TensorRT RTX execution provider checker wrapper for pytest compatibility.""" + + def __init__(self, device_type: ort.OrtHardwareDeviceType) -> None: + """Initialize RTX checker.""" + if device_type != constants.DEVICE_TO_DEVICE_TYPE["GPU"]: + raise ValueError("NvTensorRTRTXExecutionProvider only supports GPU device type") + super().__init__( + ep_name="NvTensorRTRTXExecutionProvider", + device_type=constants.DEVICE_TO_DEVICE_TYPE["GPU"], + ) + + def get_ep_checker(ep_name: EPName, device: str) -> EPChecker: """Get EPChecker for given execution provider name. @@ -251,13 +271,15 @@ def get_ep_checker(ep_name: EPName, device: str) -> EPChecker: ep_name_to_checker: dict[str, Callable[..., EPChecker]] = { "QNNExecutionProvider": QNNNPUChecker, "OpenVINOExecutionProvider": OpenVINONPUChecker, + "NvTensorRTRTXExecutionProvider": RTXChecker, # Add other EPChecker subclasses here as needed } if ep_name not in ep_name_to_checker: raise ValueError( f"Unsupported execution provider: {ep_name}. " f"Available: QNNExecutionProvider, " - f"OpenVINOExecutionProvider" + f"OpenVINOExecutionProvider, " + f"NvTensorRTRTXExecutionProvider" ) return ep_name_to_checker[ep_name](device_type=device_type) @@ -292,10 +314,15 @@ def build_parser() -> argparse.ArgumentParser: "--ep", type=str, required=True, - choices=["QNNExecutionProvider", "OpenVINOExecutionProvider"], + choices=[ + "QNNExecutionProvider", + "OpenVINOExecutionProvider", + "NvTensorRTRTXExecutionProvider", + ], help=( "Execution Provider names to test. " - "Available: QNNExecutionProvider, OpenVINOExecutionProvider" + "Available: QNNExecutionProvider, OpenVINOExecutionProvider, " + "NvTensorRTRTXExecutionProvider" ), ) parser.add_argument( diff --git a/src/winml/modelkit/analyze/runtime_checker/check_ops.py b/src/winml/modelkit/analyze/runtime_checker/check_ops.py index b4fbcc022..ea66a1d7e 100644 --- a/src/winml/modelkit/analyze/runtime_checker/check_ops.py +++ b/src/winml/modelkit/analyze/runtime_checker/check_ops.py @@ -37,7 +37,6 @@ from ...utils.constants import EPName from ...pattern.op_input_gen.qdq_gen import QDQGenerator -from ...sysinfo import SysInfo from ...utils import constants from ..utils import CheckResultWriter, load_case_indices_from_conflict_file from ..utils.model_utils import get_op_since_version @@ -98,7 +97,6 @@ def check_ops( case_index = load_case_indices_from_conflict_file(conflict_file) print(f"Loaded {len(case_index)} case_index values from conflict file: {conflict_file}") - sys_info = SysInfo().to_dict() domain = ONNXDomain.from_str(opset_domain) qdq_gen = QDQGenerator(1, ONNXDomain.COM_MICROSOFT) if use_qdq else None @@ -160,7 +158,6 @@ def check_ops( # Use writer as context manager (auto-flushes on exit) with CheckResultWriter( output_path, - sys_info, save_per_cases=None if dry_run else 100, rerun_failed=rerun_failed, delta_only=delta_only, @@ -181,9 +178,6 @@ def check_ops( save_model=save_model, model_output_dir=model_output_dir, skip_signature_fn=writer.should_skip_case, - # Also yield skipped cases (with skip - # marker) to maintain order - yield_skipped=True, dry_run=dry_run, ) @@ -257,9 +251,9 @@ class VitisAIChecker(EPChecker): """VitisAI execution provider checker wrapper for pytest compatibility.""" def __init__(self, device_type: ort.OrtHardwareDeviceType) -> None: + """Initialize VitisAI checker.""" if device_type != ort.OrtHardwareDeviceType.NPU: raise ValueError("VitisAIExecutionProvider only supports NPU device type") - """Initialize VitisAI checker.""" super().__init__(ep_name="VitisAIExecutionProvider", device_type=device_type) @@ -267,9 +261,9 @@ class MIGraphXChecker(EPChecker): """MIGraphX execution provider checker wrapper for pytest compatibility.""" def __init__(self, device_type: ort.OrtHardwareDeviceType) -> None: + """Initialize MIGraphX checker.""" if device_type != ort.OrtHardwareDeviceType.GPU: raise ValueError("MIGraphXExecutionProvider only supports GPU device type") - """Initialize MIGraphX checker.""" super().__init__(ep_name="MIGraphXExecutionProvider", device_type=device_type) @@ -277,9 +271,9 @@ class RTXChecker(EPChecker): """NVIDIA TensorRT RTX execution provider checker wrapper for pytest compatibility.""" def __init__(self, device_type: ort.OrtHardwareDeviceType) -> None: + """Initialize RTX checker.""" if device_type != ort.OrtHardwareDeviceType.GPU: raise ValueError("NvTensorRTRTXExecutionProvider only supports GPU device type") - """Initialize RTX checker.""" super().__init__( ep_name="NvTensorRTRTXExecutionProvider", device_type=ort.OrtHardwareDeviceType.GPU ) diff --git a/src/winml/modelkit/analyze/runtime_checker/ep_checker.py b/src/winml/modelkit/analyze/runtime_checker/ep_checker.py index 9edd0ed11..774ea3754 100644 --- a/src/winml/modelkit/analyze/runtime_checker/ep_checker.py +++ b/src/winml/modelkit/analyze/runtime_checker/ep_checker.py @@ -6,7 +6,7 @@ import tempfile from pathlib import Path -from typing import TYPE_CHECKING, Any, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar, Protocol import onnx import onnxruntime as ort @@ -29,6 +29,16 @@ # as sess.get_session_options() may return a modified version of session options +class _RulesPrefilterProtocol(Protocol): + """Protocol for rules-prefilter service used by EPChecker.""" + + def build_skip_check_result_for_rules_all_nodes_compile_run_pass( + self, + onnx_model: onnx.ModelProto, + ) -> dict[str, Any] | None: + pass + + class EPChecker: """Test execution provider compilation and runtime behavior.""" @@ -51,10 +61,12 @@ def __init__( ep_name: EPName, device_type: ort.OrtHardwareDeviceType, provider_options: Sequence[dict[Any, Any]] | None = None, + rules_prefilter: _RulesPrefilterProtocol | None = None, ) -> None: self.device_type = device_type - self.ep_name = ep_name + self.ep_name: EPName = ep_name self._provider_options = provider_options + self._rules_prefilter = rules_prefilter def _get_sess_options(self) -> ort.SessionOptions: winml.register_execution_providers(ort=True) @@ -74,6 +86,21 @@ def needs_case_isolation(self) -> bool: return False return self.device_type in required_device_types + def set_rules_prefilter(self, rules_prefilter: _RulesPrefilterProtocol | None) -> None: + """Set or clear rules-prefilter service for this checker.""" + self._rules_prefilter = rules_prefilter + + def build_skip_check_result_for_rules_all_nodes_compile_run_pass( + self, + onnx_model: onnx.ModelProto, + ) -> dict[str, Any] | None: + """Build synthetic check_result from fixed rules-prefilter service.""" + if self._rules_prefilter is None: + return None + return self._rules_prefilter.build_skip_check_result_for_rules_all_nodes_compile_run_pass( + onnx_model + ) + def check_compile( self, path_or_bytes: str | bytes | PathLike[Any], diff --git a/src/winml/modelkit/analyze/runtime_checker/result_sanitizer.py b/src/winml/modelkit/analyze/runtime_checker/result_sanitizer.py new file mode 100644 index 000000000..dca104382 --- /dev/null +++ b/src/winml/modelkit/analyze/runtime_checker/result_sanitizer.py @@ -0,0 +1,10 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- +"""Backward-compatible re-export for runtime checker result sanitizers.""" + +from ...utils.result_sanitizer import sanitize_check_result_payload, sanitize_result_text + + +__all__ = ["sanitize_check_result_payload", "sanitize_result_text"] diff --git a/src/winml/modelkit/analyze/utils/avalizble_ep_device_ops/README.md b/src/winml/modelkit/analyze/utils/avalizble_ep_device_ops/README.md new file mode 100644 index 000000000..c6f135776 --- /dev/null +++ b/src/winml/modelkit/analyze/utils/avalizble_ep_device_ops/README.md @@ -0,0 +1,104 @@ +# GPT Helper Guide: Adding Patterns to available_ops_all.json + +This file is a compact operational guide for GPT-style assistants. + +Goal: + +- Add or update entries in `patterns` inside `available_ops_all.json` correctly. +- Keep `case_index` encoding/decoding stable. + +## Scope and Source of Truth + +- `available_ops_all.json` + - Version sets and 2-char `version_key_map` for ops/patterns. +- `avaliable_providers.json` + - 1-char key for `(EP, device)`. +- `case_index_key_codec.py` + - Key validation and file name <-> 4-char prefix codec. +- `../op_utils.py` + - Case signature and final 36-char `case_index` generation. + +## Required Pattern Version Rule + +For a pattern with ops `op_1 ... op_n`, let `V(op_i)` be the version set from op maps. + +1. `start_version = max(min(V(op_1)), min(V(op_2)), ..., min(V(op_n)))` +2. `pattern_versions = sorted(unique(union(v in V(op_i) where v >= start_version)))` +3. Pattern `version_key_map` keys must exactly match `pattern_versions`. + +Example (`MatMulAddPattern`): + +- `V(MatMul) = {9, 13}` +- `V(Add) = {7, 13, 14}` +- `start_version = max(9, 7) = 9` +- `pattern_versions = {9, 13, 14}` + +## Domain Rule for Ops + +When resolving op versions for a pattern: + +- Prefer unambiguous lookup from one domain. +- If an op name exists in both domains (`ops_ai_onnx` and `ops_com_microsoft`), select domain intentionally. +- Do not mix domains for the same op without explicit intent. + +## Key Allocation Constraints + +When adding new `version_key_map` entries: + +- Version key must be exactly 2 chars. +- Allowed alphabet: `123456789abcdefghijklmnopqrstuvwxyz`. +- Character `0` is not allowed. +- 2-char version keys must be globally unique across: + - `ops_ai_onnx` + - `ops_com_microsoft` + - `patterns` + - `ops` (if present) + +## case_index Contract + +Final format: + +- `case_index = <4-char namespace key> + <32-char md5(signature)>` +- Total length is always 36. + +4-char namespace key format: + +- 1 char: EP/device key (from `avaliable_providers.json`) +- 2 chars: version key (from `available_ops_all.json`) +- 1 char: qdq flag (`0` non-QDQ, `1` QDQ) + +## File Name Contract for Codec + +The codec expects this stem shape: + +`____opset[_qdq]` + +Supported device tokens: `CPU`, `GPU`, `NPU`. + +## Rules-First Behavior in check_patterns + +`check_patterns` first queries node-level parquet rules. + +- If all nodes are supported and have data, it writes synthetic success and skips real EP run. +- If any node is unsupported/no_data/error, it falls back to real compile/run. + +`check_ops` does not use this prefilter path. + +## Minimal Update Workflow (for GPT) + +1. Identify target pattern and op list. +2. Resolve each op version set from correct domain. +3. Compute `start_version` and `pattern_versions` using the required rule. +4. Allocate new globally unique 2-char keys for those versions. +5. Update `patterns.` in `available_ops_all.json`. +6. Validate JSON parse. +7. Validate no duplicate 2-char keys. +8. Verify codec round-trip for at least one generated file name. + +## Quick Validation Checklist + +- JSON is valid. +- Pattern version keys follow rule exactly. +- New 2-char keys are unique and alphabet-compliant. +- No accidental edits to unrelated patterns. +- `encode_file_name_to_4char_key` and `decode_4char_key_to_folder_and_file_name` work for the new entry. diff --git a/src/winml/modelkit/analyze/utils/avalizble_ep_device_ops/available_ops_all.json b/src/winml/modelkit/analyze/utils/avalizble_ep_device_ops/available_ops_all.json index 415b03744..4c0a9b154 100644 --- a/src/winml/modelkit/analyze/utils/avalizble_ep_device_ops/available_ops_all.json +++ b/src/winml/modelkit/analyze/utils/avalizble_ep_device_ops/available_ops_all.json @@ -1007,7 +1007,15 @@ "13": "8s", "14": "8t", "21": "8u" - } + }, + "check_for_eps": [ + "ov_cpu", + "ov_gpu", + "ov_npu", + "qnn_gpu", + "qnn_npu", + "nv_gpu" + ] }, "TransposeAttentionPattern": { "ops": [ @@ -1031,7 +1039,15 @@ "19": "8z", "21": "91", "22": "92" - } + }, + "check_for_eps": [ + "ov_cpu", + "ov_gpu", + "ov_npu", + "qnn_gpu", + "qnn_npu", + "nv_gpu" + ] }, "Conv2DInplaceLinear3DPattern": { "ops": [ @@ -1048,7 +1064,15 @@ "19": "96", "21": "97", "22": "98" - } + }, + "check_for_eps": [ + "ov_cpu", + "ov_gpu", + "ov_npu", + "qnn_gpu", + "qnn_npu", + "nv_gpu" + ] }, "Conv2DInplaceLinear2DPattern": { "ops": [ @@ -1063,7 +1087,15 @@ "19": "9c", "21": "9d", "22": "9e" - } + }, + "check_for_eps": [ + "ov_cpu", + "ov_gpu", + "ov_npu", + "qnn_gpu", + "qnn_npu", + "nv_gpu" + ] }, "Gelu1Pattern": { "ops": [ @@ -1078,7 +1110,12 @@ "14": "9h" }, "check_for_eps": [ - "qnn_npu" + "ov_cpu", + "ov_gpu", + "ov_npu", + "qnn_gpu", + "qnn_npu", + "nv_gpu" ] }, "Gelu2Pattern": { @@ -1094,7 +1131,12 @@ "14": "9k" }, "check_for_eps": [ - "qnn_npu" + "ov_cpu", + "ov_gpu", + "ov_npu", + "qnn_gpu", + "qnn_npu", + "nv_gpu" ] }, "Gelu3Pattern": { @@ -1110,7 +1152,12 @@ "14": "9n" }, "check_for_eps": [ - "qnn_npu" + "ov_cpu", + "ov_gpu", + "ov_npu", + "qnn_gpu", + "qnn_npu", + "nv_gpu" ] }, "Gelu4Pattern": { @@ -1125,7 +1172,12 @@ "14": "9q" }, "check_for_eps": [ - "qnn_npu" + "ov_cpu", + "ov_gpu", + "ov_npu", + "qnn_gpu", + "qnn_npu", + "nv_gpu" ] }, "MatMulAddPattern": { @@ -1139,7 +1191,12 @@ "14": "9t" }, "check_for_eps": [ - "qnn_npu" + "ov_cpu", + "ov_gpu", + "ov_npu", + "qnn_gpu", + "qnn_npu", + "nv_gpu" ] }, "ReshapeGemmReshapePattern": { @@ -1155,7 +1212,12 @@ "21": "9y" }, "check_for_eps": [ - "qnn_npu" + "ov_cpu", + "ov_gpu", + "ov_npu", + "qnn_gpu", + "qnn_npu", + "nv_gpu" ] }, "LayerNormalizationPowPattern": { @@ -1174,7 +1236,15 @@ "14": "a2", "15": "a3", "18": "a4" - } + }, + "check_for_eps": [ + "ov_cpu", + "ov_gpu", + "ov_npu", + "qnn_gpu", + "qnn_npu", + "nv_gpu" + ] }, "LayerNormalizationMulPattern": { "ops": [ @@ -1190,7 +1260,15 @@ "13": "a6", "14": "a7", "18": "a8" - } + }, + "check_for_eps": [ + "ov_cpu", + "ov_gpu", + "ov_npu", + "qnn_gpu", + "qnn_npu", + "nv_gpu" + ] }, "TransposedSingleLayerNormalizationPattern": { "ops": [ @@ -1202,7 +1280,15 @@ "17": "a9", "19": "aa", "21": "ab" - } + }, + "check_for_eps": [ + "ov_cpu", + "ov_gpu", + "ov_npu", + "qnn_gpu", + "qnn_npu", + "nv_gpu" + ] }, "RMSNormalizationPowPattern": { "ops": [ @@ -1259,7 +1345,12 @@ "21": "aq" }, "check_for_eps": [ - "ov_npu" + "ov_cpu", + "ov_gpu", + "ov_npu", + "qnn_gpu", + "qnn_npu", + "nv_gpu" ] }, "ReshapeTransposeReshapeLowDimPattern": { @@ -1275,8 +1366,15 @@ "21": "av" }, "check_for_eps": [ - "ov_npu" + "ov_cpu", + "ov_gpu", + "ov_npu", + "qnn_gpu", + "qnn_npu", + "nv_gpu" ] } } } + + diff --git a/src/winml/modelkit/analyze/utils/op_utils.py b/src/winml/modelkit/analyze/utils/op_utils.py index 6ce2e28c0..4b9d4cf03 100644 --- a/src/winml/modelkit/analyze/utils/op_utils.py +++ b/src/winml/modelkit/analyze/utils/op_utils.py @@ -158,7 +158,6 @@ class CheckResultWriter: def __init__( self, file_path: str | Path, - sys_info: dict[str, Any], save_per_cases: int | None = 100, rerun_failed: bool = False, delta_only: bool = False, @@ -169,14 +168,12 @@ def __init__( Args: file_path: Path to the output JSON file - sys_info: System information dictionary (constant during run) save_per_cases: Number of results to accumulate before saving to file. If None, disable periodic saves and save only on flush/finalization. rerun_failed: If True, rerun failed cases (compile or run failed). delta_only: If True, only run new test cases not in existing results. """ self.file_path = Path(file_path) - self.sys_info = sys_info self.save_per_cases = save_per_cases self.results: list[dict[str, Any]] = [] self.pending_count = 0 @@ -441,11 +438,6 @@ def _save(self) -> None: output_data = { "check_results": results_to_save, - # NOTE: Intentionally do not persist sys_info. - # This file may be updated across multiple runs, and different cases in - # the same output can come from different run environments/sys_info. - # Persisting one top-level sys_info would be misleading. - # "sys_info": self.sys_info, } for item in output_data["check_results"]: diff --git a/src/winml/modelkit/pattern/op_input_gen/op_input_gen.py b/src/winml/modelkit/pattern/op_input_gen/op_input_gen.py index 8e5a7cb80..d3d975463 100644 --- a/src/winml/modelkit/pattern/op_input_gen/op_input_gen.py +++ b/src/winml/modelkit/pattern/op_input_gen/op_input_gen.py @@ -21,6 +21,7 @@ from onnx.defs import OpSchema from ...onnx import ONNXDomain, SupportedONNXType +from ...utils.result_sanitizer import sanitize_check_result_payload from ..utils import get_op_input_properties from .qdq_gen import QDQGenerator @@ -1498,7 +1499,6 @@ def check_on_ep( save_model: bool = False, model_output_dir: str | Path | None = None, skip_signature_fn: Callable[[dict], bool] | None = None, - yield_skipped: bool = False, dry_run: bool = False, ) -> Any: """Test the given OpInputGenerator by generating ONNX models. @@ -1511,9 +1511,14 @@ def check_on_ep( skip_cases: number of test cases to skip before starting to run tests. skip_signature_fn: if not None, a function that takes a result dict and returns True if the case should - be skipped (for delta or rerun mode). - yield_skipped: if True, also yield skipped cases with a "_skipped" marker. - This allows the caller to reuse existing results in order. + be skipped (for delta or rerun mode). Skipped cases are yielded + with a "_skipped" marker so callers can reuse existing results. + For non-dry-run cases, rules prefilter runs only for multi-node models + (pattern-like graphs). For single-node op models, compile/run proceeds + directly on ep_checker. + When all nodes in a multi-node generated model are supported by rules, + compile/run on ep_checker is skipped and a synthetic successful + check_result is written for that case. Yields: Test result dictionaries one at a time. @@ -1573,13 +1578,12 @@ def _run_ep_check( # the model is deferred until we know the case is not discarded. if skip_signature_fn is not None: if skip_signature_fn(final_tags): - if yield_skipped: - # Reused cases still need the model payload for replay. - final_tags["model_bytes_b64"] = model_bytes_to_b64( - onnx_model.SerializeToString() - ) - final_tags["_skipped"] = True - yield final_tags + # Reused cases still need the model payload for replay. + final_tags["model_bytes_b64"] = model_bytes_to_b64( + onnx_model.SerializeToString() + ) + final_tags["_skipped"] = True + yield final_tags continue # Check if we need to skip this case elif cases_skipped < skip_cases: @@ -1590,6 +1594,34 @@ def _run_ep_check( # Keep model payload by default so every persisted case can be replayed. final_tags["model_bytes_b64"] = model_bytes_to_b64(model_bytes) + rules_all_nodes_pass_skip_check_result: dict[str, Any] | None = None + # Only run rules-based skip check for pattern-like multi-node graphs. + # Single-node models are operator checks and should run real compile/run. + if not dry_run and len(onnx_model.graph.node) > 1: + rules_all_nodes_pass_skip_check_result = ( + ep_checker.build_skip_check_result_for_rules_all_nodes_compile_run_pass( + onnx_model + ) + ) + + if rules_all_nodes_pass_skip_check_result is not None: + compile_count = _increment_test_phase_global_count("compile", True) + run_count = _increment_test_phase_global_count("run", True) + final_tags["check_result"] = rules_all_nodes_pass_skip_check_result + print( + f"{Fore.GREEN}Rules all-nodes compile/run pass; " + f"skip EP compile/run. " + f"compile_count: success<{compile_count['success']}> " + f"failed<{compile_count['failed']}> " + f"all<{compile_count['all']}>, " + f"run_count: success<{run_count['success']}> " + f"failed<{run_count['failed']}> " + f"all<{run_count['all']}>" + f"{Style.RESET_ALL}" + ) + yield final_tags + continue + qdq_types = final_tags.get(self.qdq_types_key, None) ep_checker_inputs = self.create_input_dict(kwargs, qdq_types=qdq_types) @@ -1609,6 +1641,7 @@ def _dry_run_result() -> dict[str, Any]: else _dry_run_result() ) _check_qnn_ep_setup_failure(compile_result) + sanitize_check_result_payload(compile_result) # TODO: if compilation succeeded, maybe skip run test? compile_success = compile_result["result"]["success"] @@ -1702,6 +1735,7 @@ def _json_default(o: Any) -> Any: else _dry_run_result() ) _check_qnn_ep_setup_failure(run_result) + sanitize_check_result_payload(run_result) run_success = run_result["result"]["success"] run_count = _increment_test_phase_global_count("run", run_success) diff --git a/src/winml/modelkit/utils/result_sanitizer.py b/src/winml/modelkit/utils/result_sanitizer.py new file mode 100644 index 000000000..ed3eae1e5 --- /dev/null +++ b/src/winml/modelkit/utils/result_sanitizer.py @@ -0,0 +1,63 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- +from __future__ import annotations + +import re +from typing import Any + + +# Example: "(123 us)" +_DURATION_US_RE = re.compile(r"\(\s*\d+\s*us\)") + +# Date + time styles we commonly see in ORT/EP logs. +_DATETIME_PATTERNS = ( + # 2026-07-08 13:23:06.596 / 2026/07/08 13:23:06 / 2026-07-08T13:23:06Z + re.compile( + r"\b\d{4}[-/]\d{2}[-/]\d{2}[T\s]\d{2}:\d{2}:\d{2}(?:[.,]\d{1,9})?(?:Z|[+-]\d{2}:?\d{2})?\b" + ), + # 7/8/2026 1:23:06.596 PM / 07-08-2026 13:23:06 + re.compile( + r"\b\d{1,2}[/-]\d{1,2}[/-]\d{2,4}[T\s]\d{1,2}:\d{2}:\d{2}(?:[.,]\d{1,9})?(?:\s?(?:AM|PM))?\b", + re.IGNORECASE, + ), + # Time only: 13:23:06.596 + re.compile(r"(? str: + """Normalize volatile timing fragments in result/log text for stable diffs.""" + cleaned = _DURATION_US_RE.sub("", text) + for pattern in _DATETIME_PATTERNS: + cleaned = pattern.sub("", cleaned) + return cleaned + + +def sanitize_check_result_payload(result: Any) -> None: + """In-place sanitize of one runtime-check stage payload. + + Expected input shape: + { + "result": {"success": bool | None, "reason": str | None, ...}, + "stdout": str | None, + "stderr": str | None, + } + """ + if not isinstance(result, dict): + return + + for key in ("stdout", "stderr"): + value = result.get(key) + if isinstance(value, str): + result[key] = sanitize_result_text(value) + + result_payload = result.get("result") + if isinstance(result_payload, dict): + reason = result_payload.get("reason") + if isinstance(reason, str): + result_payload["reason"] = sanitize_result_text(reason) + + +__all__ = ["sanitize_check_result_payload", "sanitize_result_text"] diff --git a/tests/unit/analyze/test_check_ops.py b/tests/unit/analyze/test_check_ops.py index 5a0b16634..2b5372513 100644 --- a/tests/unit/analyze/test_check_ops.py +++ b/tests/unit/analyze/test_check_ops.py @@ -213,7 +213,7 @@ def _make_writer(existing_cases: list[dict], tmp_path): """Create a CheckResultWriter with pre-loaded existing_signatures from cases.""" output_file = tmp_path / "Abs_QNNExecutionProvider_NPU_ai.onnx_opset13.json" output_file.write_text(json.dumps({"check_results": existing_cases}), encoding="utf-8") - return CheckResultWriter(output_file, sys_info={}, delta_only=True) + return CheckResultWriter(output_file, delta_only=True) def test_reuse_upgrades_value_array_to_same_value(self, tmp_path) -> None: """When an existing case has the old value-array format, reuse_existing_result @@ -341,10 +341,10 @@ def test_case_index_is_36_chars_and_ep_device_differs_by_first_char(self, tmp_pa qnn_output = tmp_path / "Abs_QNNExecutionProvider_NPU_ai.onnx_opset13.json" ov_output = tmp_path / "Abs_OpenVINOExecutionProvider_CPU_ai.onnx_opset13.json" - with CheckResultWriter(qnn_output, sys_info={}) as writer_qnn: + with CheckResultWriter(qnn_output) as writer_qnn: writer_qnn.append_result(dict(case_template)) - with CheckResultWriter(ov_output, sys_info={}) as writer_ov: + with CheckResultWriter(ov_output) as writer_ov: writer_ov.append_result(dict(case_template)) qnn_case = writer_qnn.results[0]