From a5019c328d0a6723013c507e7d0bc59317199328 Mon Sep 17 00:00:00 2001 From: Frank Scholter Peres Date: Wed, 20 May 2026 11:10:22 +0000 Subject: [PATCH 1/8] score_metamodel: autodiscover requirement types for metrics --- docs/conf.py | 4 + src/extensions/score_metamodel/__init__.py | 96 ++++-- ...st_traceability_metrics_json_generation.py | 300 +++++++++++++++++- 3 files changed, 379 insertions(+), 21 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 0255915c2..488b0c86b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -18,3 +18,7 @@ extensions = [ "score_sphinx_bundle", ] + +# Configure traceability metrics explicitly for this repository. +score_metamodel_requirement_types = "tool_req" +score_metamodel_include_external_needs = False diff --git a/src/extensions/score_metamodel/__init__.py b/src/extensions/score_metamodel/__init__.py index 557e7b974..f9fea3a9d 100644 --- a/src/extensions/score_metamodel/__init__.py +++ b/src/extensions/score_metamodel/__init__.py @@ -111,29 +111,36 @@ def _write_metrics_json(app: Sphinx, exception: Exception | None) -> None: return all_needs: list[Any] = list(SphinxNeedsData(app.env).get_needs_view().values()) - - raw = str(getattr(app.config, "score_metamodel_requirement_types", "tool_req")) - requirement_types = {t.strip() for t in raw.split(",") if t.strip()} or {"tool_req"} - include_not_implemented = True include_external: bool = bool( getattr(app.config, "score_metamodel_include_external_needs", False) ) + raw = str(getattr(app.config, "score_metamodel_requirement_types", "")).strip() + requirement_types = {t.strip() for t in raw.split(",") if t.strip()} + if not requirement_types: + requirement_types = _discover_requirement_types(app, all_needs, include_external) + include_not_implemented = True + metrics_by_type: dict[str, Any] = {} - for req_type in sorted(requirement_types): - type_summary = compute_traceability_summary( - all_needs=all_needs, - requirement_types={req_type}, - include_not_implemented=include_not_implemented, - filtered_test_types=set(), - include_external=include_external, + if not requirement_types: + logger.info( + "No requirement types configured or discovered; writing empty metrics.json." ) - metrics_by_type[req_type] = { - "include_not_implemented": type_summary["include_not_implemented"], - "include_external": type_summary["include_external"], - "requirements": type_summary["requirements"], - "tests": type_summary["tests"], - } + else: + for req_type in sorted(requirement_types): + type_summary = compute_traceability_summary( + all_needs=all_needs, + requirement_types={req_type}, + include_not_implemented=include_not_implemented, + filtered_test_types=set(), + include_external=include_external, + ) + metrics_by_type[req_type] = { + "include_not_implemented": type_summary["include_not_implemented"], + "include_external": type_summary["include_external"], + "requirements": type_summary["requirements"], + "tests": type_summary["tests"], + } output: dict[str, Any] = { "schema_version": "1", @@ -147,6 +154,56 @@ def _write_metrics_json(app: Sphinx, exception: Exception | None) -> None: logger.info(f"Traceability metrics written to: {out_path}") +def _get_need_value(need: Any, key: str, default: Any = None) -> Any: + getter = getattr(need, "get", None) + if callable(getter): + return getter(key, default) + try: + return need[key] + except Exception: + return default + + +def _discover_requirement_types( + app: Sphinx, all_needs: list[Any], include_external: bool +) -> set[str]: + """Discover requirement directives that are both tagged and present.""" + tagged_requirements: set[str] = set() + needs_types = getattr(app.config, "needs_types", []) + for need_type in needs_types or []: + if not isinstance(need_type, dict): + continue + directive = need_type.get("directive") + tags = need_type.get("tags", []) + if not isinstance(directive, str): + continue + if not isinstance(tags, list): + continue + normalized = {str(tag).strip() for tag in tags} + if "requirement_excl_process" in normalized or "requirement" in normalized: + tagged_requirements.add(directive) + + present_types: set[str] = set() + for need in all_needs: + is_external = bool(_get_need_value(need, "is_external", False)) + if not include_external and is_external: + continue + need_type: Any = _get_need_value(need, "type", None) + if isinstance(need_type, str): + present_types.add(need_type) + discovered = tagged_requirements.intersection(present_types) + if not discovered: + # Fallback for repositories that use *_req directives but do not tag + # requirement types in needs_types. + discovered = {t for t in present_types if t.endswith("_req")} + if discovered: + logger.info( + "score_metamodel_requirement_types is not configured; " + f"using discovered requirement types: {', '.join(sorted(discovered))}" + ) + return discovered + + def _run_checks(app: Sphinx, exception: Exception | None) -> None: # Do not run checks if an exception occurred during build if exception: @@ -334,11 +391,12 @@ def setup(app: Sphinx) -> dict[str, str | bool]: app.add_config_value( "score_metamodel_requirement_types", - "tool_req", + "", rebuild="env", description=( "Comma-separated list of need types treated as requirements for " - "traceability metrics (default: tool_req)." + "traceability metrics. If empty, requirement types are autodiscovered " + "from needs_types tags (requirement, requirement_excl_process)." ), ) diff --git a/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py b/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py index 764659874..7d5e98885 100644 --- a/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py +++ b/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py @@ -38,6 +38,14 @@ def get_needs_view(self) -> dict[str, dict[str, object]]: "testlink": "", "is_external": False, }, + "LOCAL_FEAT": { + "id": "LOCAL_FEAT", + "type": "feat_req", + "implemented": "YES", + "source_code_link": "", + "testlink": "", + "is_external": False, + }, "EXT_REQ": { "id": "EXT_REQ", "type": "tool_req", @@ -46,16 +54,105 @@ def get_needs_view(self) -> dict[str, dict[str, object]]: "testlink": "", "is_external": True, }, + "EXT_FEAT": { + "id": "EXT_FEAT", + "type": "feat_req", + "implemented": "NO", + "source_code_link": "src/ext_feat.py:1", + "testlink": "", + "is_external": True, + }, + "EXT_GD": { + "id": "EXT_GD", + "type": "gd_req", + "implemented": "NO", + "source_code_link": "src/ext_gd.py:1", + "testlink": "", + "is_external": True, + }, } -def _app(tmp_path: Path, include_external: bool) -> SimpleNamespace: +class _FakeNonReqNeedsData: + def __init__(self, env: object): + self._env = env + + def get_needs_view(self) -> dict[str, dict[str, object]]: + return { + "LOCAL_COMP": { + "id": "LOCAL_COMP", + "type": "comp", + "implemented": "YES", + "source_code_link": "", + "testlink": "", + "is_external": False, + }, + "LOCAL_DOC": { + "id": "LOCAL_DOC", + "type": "document", + "implemented": "YES", + "source_code_link": "", + "testlink": "", + "is_external": False, + }, + } + + +class _NeedObj: + def __init__(self, payload: dict[str, object]): + self._payload = payload + + def get(self, key: str, default: object | None = None) -> object | None: + return self._payload.get(key, default) + + +class _FakeObjectNeedsData: + def __init__(self, env: object): + self._env = env + + def get_needs_view(self) -> dict[str, _NeedObj]: + return { + "LOCAL_REQ": _NeedObj( + { + "id": "LOCAL_REQ", + "type": "tool_req", + "implemented": "YES", + "source_code_link": "", + "testlink": "", + "is_external": False, + } + ), + "LOCAL_FEAT": _NeedObj( + { + "id": "LOCAL_FEAT", + "type": "feat_req", + "implemented": "YES", + "source_code_link": "", + "testlink": "", + "is_external": False, + } + ), + } + + +def _app( + tmp_path: Path, + include_external: bool, + requirement_types: str = "tool_req", + needs_types: list[dict[str, object]] | None = None, +) -> SimpleNamespace: + discovered_types = needs_types or [ + {"directive": "tool_req", "tags": ["requirement_excl_process"]}, + {"directive": "feat_req", "tags": ["requirement"]}, + {"directive": "workflow", "tags": []}, + ] return SimpleNamespace( env=object(), outdir=str(tmp_path), config=SimpleNamespace( - score_metamodel_requirement_types="tool_req", + score_metamodel_requirement_types=requirement_types, score_metamodel_include_external_needs=include_external, + needs_types=discovered_types, ), ) @@ -94,3 +191,202 @@ def test_write_metrics_json_can_include_external( assert metrics["include_external"] is True assert metrics["requirements"]["total"] == 2 + + +def test_explicit_requirement_types_disable_autodiscovery( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) + + metamodel_init._write_metrics_json( + cast( + Sphinx, + _app( + tmp_path, + include_external=False, + requirement_types="tool_req", + ), + ), + None, + ) + + payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) + assert set(payload["metrics_by_type"].keys()) == {"tool_req"} + + +def test_write_metrics_json_autodiscovers_when_types_unset( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) + + metamodel_init._write_metrics_json( + cast( + Sphinx, + _app(tmp_path, include_external=False, requirement_types=""), + ), + None, + ) + + payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) + assert payload["schema_version"] == "1" + assert set(payload["metrics_by_type"].keys()) == {"feat_req", "tool_req"} + + +def test_autodiscovery_excludes_tagged_types_not_present_in_needs( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) + + metamodel_init._write_metrics_json( + cast( + Sphinx, + _app( + tmp_path, + include_external=False, + requirement_types="", + needs_types=[ + {"directive": "tool_req", "tags": ["requirement_excl_process"]}, + {"directive": "feat_req", "tags": ["requirement"]}, + {"directive": "aou_req", "tags": ["requirement"]}, + ], + ), + ), + None, + ) + + payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) + assert set(payload["metrics_by_type"].keys()) == {"feat_req", "tool_req"} + + +def test_write_metrics_json_empty_when_no_types_configured_or_discovered( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNonReqNeedsData) + + metamodel_init._write_metrics_json( + cast( + Sphinx, + _app( + tmp_path, + include_external=False, + requirement_types="", + needs_types=[{"directive": "workflow", "tags": []}], + ), + ), + None, + ) + + payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) + assert payload["schema_version"] == "1" + assert payload["metrics_by_type"] == {} + + +def test_autodiscovery_falls_back_to_present_req_suffix_types( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) + + metamodel_init._write_metrics_json( + cast( + Sphinx, + _app( + tmp_path, + include_external=False, + requirement_types="", + needs_types=[{"directive": "workflow", "tags": []}], + ), + ), + None, + ) + + payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) + assert set(payload["metrics_by_type"].keys()) == {"feat_req", "tool_req"} + + +def test_autodiscovery_respects_include_external_scope( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) + + metamodel_init._write_metrics_json( + cast( + Sphinx, + _app( + tmp_path, + include_external=True, + requirement_types="", + needs_types=[{"directive": "workflow", "tags": []}], + ), + ), + None, + ) + + payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) + assert set(payload["metrics_by_type"].keys()) == {"feat_req", "gd_req", "tool_req"} + + +def test_autodiscovery_handles_needitem_like_objects( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeObjectNeedsData) + + metamodel_init._write_metrics_json( + cast( + Sphinx, + _app( + tmp_path, + include_external=False, + requirement_types="", + needs_types=[{"directive": "workflow", "tags": []}], + ), + ), + None, + ) + + payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) + assert set(payload["metrics_by_type"].keys()) == {"feat_req", "tool_req"} + + +@pytest.mark.parametrize( + ("requirement_types", "include_external", "should_exist", "expected_totals"), + [ + ("tool_req", False, True, {"tool_req": 1}), + ("feat_req,tool_req", False, True, {"feat_req": 1, "tool_req": 1}), + ("", False, True, {"feat_req": 1, "tool_req": 1}), + (" ", False, True, {"feat_req": 1, "tool_req": 1}), + ("tool_req", True, True, {"tool_req": 2}), + ], +) +def test_write_metrics_json_settings_matrix( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + requirement_types: str, + include_external: bool, + should_exist: bool, + expected_totals: dict[str, int], +) -> None: + monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) + + metamodel_init._write_metrics_json( + cast( + Sphinx, + _app( + tmp_path, + include_external=include_external, + requirement_types=requirement_types, + ), + ), + None, + ) + + metrics_file = tmp_path / "metrics.json" + assert metrics_file.exists() is should_exist + if not should_exist: + return + + payload = json.loads(metrics_file.read_text(encoding="utf-8")) + by_type = payload["metrics_by_type"] + assert set(by_type.keys()) == set(expected_totals.keys()) + + for req_type, expected_total in expected_totals.items(): + assert by_type[req_type]["requirements"]["total"] == expected_total From 3ba6ee067e359960d6feca5239ac655cebecc086 Mon Sep 17 00:00:00 2001 From: Frank Scholter Peres Date: Wed, 20 May 2026 11:18:03 +0000 Subject: [PATCH 2/8] docs: clarify requirement type autodiscovery override --- docs/how-to/dashboards_and_quality_gates.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/how-to/dashboards_and_quality_gates.rst b/docs/how-to/dashboards_and_quality_gates.rst index ea0797579..a0aa58fde 100644 --- a/docs/how-to/dashboards_and_quality_gates.rst +++ b/docs/how-to/dashboards_and_quality_gates.rst @@ -50,6 +50,11 @@ In ``docs/conf.py``: score_metamodel_requirement_types = "feat_req,comp_req,aou_req" score_metamodel_include_external_needs = False +By default, ``score_metamodel`` autodiscovers requirement types from the +repository needs that are present in the current build. If +``score_metamodel_requirement_types`` is set, that explicit list overrides +autodiscovery. + Use ``score_metamodel_include_external_needs = True`` only in repositories that intentionally aggregate requirements across module dependencies, such as integration repositories. Use ``False`` for module repositories to gate only on From 2c638a340ae0618feba4867abb7a6f9e44684418 Mon Sep 17 00:00:00 2001 From: Frank Scholter Peres Date: Wed, 20 May 2026 11:30:21 +0000 Subject: [PATCH 3/8] lint fix --- src/extensions/score_metamodel/__init__.py | 29 ++++++++++++++-------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/extensions/score_metamodel/__init__.py b/src/extensions/score_metamodel/__init__.py index f9fea3a9d..f196cc4a6 100644 --- a/src/extensions/score_metamodel/__init__.py +++ b/src/extensions/score_metamodel/__init__.py @@ -118,7 +118,9 @@ def _write_metrics_json(app: Sphinx, exception: Exception | None) -> None: raw = str(getattr(app.config, "score_metamodel_requirement_types", "")).strip() requirement_types = {t.strip() for t in raw.split(",") if t.strip()} if not requirement_types: - requirement_types = _discover_requirement_types(app, all_needs, include_external) + requirement_types = _discover_requirement_types( + app, all_needs, include_external + ) include_not_implemented = True metrics_by_type: dict[str, Any] = {} @@ -164,6 +166,19 @@ def _get_need_value(need: Any, key: str, default: Any = None) -> Any: return default +def _as_requirement_directive(need_type: Any) -> str | None: + if not isinstance(need_type, dict): + return None + directive = need_type.get("directive") + tags = need_type.get("tags", []) + if not isinstance(directive, str) or not isinstance(tags, list): + return None + normalized = {str(tag).strip() for tag in tags} + if "requirement_excl_process" in normalized or "requirement" in normalized: + return directive + return None + + def _discover_requirement_types( app: Sphinx, all_needs: list[Any], include_external: bool ) -> set[str]: @@ -171,16 +186,8 @@ def _discover_requirement_types( tagged_requirements: set[str] = set() needs_types = getattr(app.config, "needs_types", []) for need_type in needs_types or []: - if not isinstance(need_type, dict): - continue - directive = need_type.get("directive") - tags = need_type.get("tags", []) - if not isinstance(directive, str): - continue - if not isinstance(tags, list): - continue - normalized = {str(tag).strip() for tag in tags} - if "requirement_excl_process" in normalized or "requirement" in normalized: + directive = _as_requirement_directive(need_type) + if directive: tagged_requirements.add(directive) present_types: set[str] = set() From d892ca6a0c39479323d6edf980a9eb417be79bf9 Mon Sep 17 00:00:00 2001 From: Frank Scholter Peres Date: Fri, 22 May 2026 08:57:59 +0000 Subject: [PATCH 4/8] Refine requirement type autodiscovery --- docs/conf.py | 4 -- docs/how-to/dashboards_and_quality_gates.rst | 30 +++++--- src/extensions/score_metamodel/__init__.py | 20 +++--- ...st_traceability_metrics_json_generation.py | 71 +++---------------- 4 files changed, 37 insertions(+), 88 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 488b0c86b..0255915c2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -18,7 +18,3 @@ extensions = [ "score_sphinx_bundle", ] - -# Configure traceability metrics explicitly for this repository. -score_metamodel_requirement_types = "tool_req" -score_metamodel_include_external_needs = False diff --git a/docs/how-to/dashboards_and_quality_gates.rst b/docs/how-to/dashboards_and_quality_gates.rst index a0aa58fde..418fc6228 100644 --- a/docs/how-to/dashboards_and_quality_gates.rst +++ b/docs/how-to/dashboards_and_quality_gates.rst @@ -40,25 +40,35 @@ Typical Setup For details, see :ref:`setup`. -Minimal Configuration Example ------------------------------ +Configuration +------------- + +Default Behavior (No Configuration Needed) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default, ``score_metamodel`` autodiscovers requirement types from the +repository needs present in the current build. Requirement types are identified +from ``needs_types`` entries tagged with ``requirement`` or +``requirement_excl_process``. -In ``docs/conf.py``: +This is the recommended setup for most repositories. + +Optional Override for Requirement Types +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If a repository needs to force a specific set of requirement types, set an +explicit override in ``docs/conf.py``: .. code-block:: python score_metamodel_requirement_types = "feat_req,comp_req,aou_req" - score_metamodel_include_external_needs = False -By default, ``score_metamodel`` autodiscovers requirement types from the -repository needs that are present in the current build. If -``score_metamodel_requirement_types`` is set, that explicit list overrides +When this setting is provided, the explicit list is used instead of autodiscovery. Use ``score_metamodel_include_external_needs = True`` only in repositories that intentionally aggregate requirements across module dependencies, such as -integration repositories. Use ``False`` for module repositories to gate only on -local traceability. +integration repositories. Building the Dashboard ---------------------- @@ -92,7 +102,7 @@ There are two common modes: **Module repository** -- Set ``score_metamodel_include_external_needs = False``. +- No setting needed. Local-only scope is the default. - Gate only on the needs owned by the repository itself. - Use this for per-module implementation progress and traceability. diff --git a/src/extensions/score_metamodel/__init__.py b/src/extensions/score_metamodel/__init__.py index f196cc4a6..4d4eaa77a 100644 --- a/src/extensions/score_metamodel/__init__.py +++ b/src/extensions/score_metamodel/__init__.py @@ -157,13 +157,7 @@ def _write_metrics_json(app: Sphinx, exception: Exception | None) -> None: def _get_need_value(need: Any, key: str, default: Any = None) -> Any: - getter = getattr(need, "get", None) - if callable(getter): - return getter(key, default) - try: - return need[key] - except Exception: - return default + return need.get(key, default) def _as_requirement_directive(need_type: Any) -> str | None: @@ -198,11 +192,15 @@ def _discover_requirement_types( need_type: Any = _get_need_value(need, "type", None) if isinstance(need_type, str): present_types.add(need_type) + discovered = tagged_requirements.intersection(present_types) - if not discovered: - # Fallback for repositories that use *_req directives but do not tag - # requirement types in needs_types. - discovered = {t for t in present_types if t.endswith("_req")} + + if tagged_requirements and not discovered: + logger.warning( + "No requirement types discovered in current build for tagged " + "needs_types requirement directives." + ) + if discovered: logger.info( "score_metamodel_requirement_types is not configured; " diff --git a/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py b/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py index 7d5e98885..47af209cb 100644 --- a/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py +++ b/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py @@ -98,47 +98,10 @@ def get_needs_view(self) -> dict[str, dict[str, object]]: } -class _NeedObj: - def __init__(self, payload: dict[str, object]): - self._payload = payload - - def get(self, key: str, default: object | None = None) -> object | None: - return self._payload.get(key, default) - - -class _FakeObjectNeedsData: - def __init__(self, env: object): - self._env = env - - def get_needs_view(self) -> dict[str, _NeedObj]: - return { - "LOCAL_REQ": _NeedObj( - { - "id": "LOCAL_REQ", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "", - "testlink": "", - "is_external": False, - } - ), - "LOCAL_FEAT": _NeedObj( - { - "id": "LOCAL_FEAT", - "type": "feat_req", - "implemented": "YES", - "source_code_link": "", - "testlink": "", - "is_external": False, - } - ), - } - - def _app( tmp_path: Path, include_external: bool, - requirement_types: str = "tool_req", + requirement_types: str = "", needs_types: list[dict[str, object]] | None = None, ) -> SimpleNamespace: discovered_types = needs_types or [ @@ -281,7 +244,7 @@ def test_write_metrics_json_empty_when_no_types_configured_or_discovered( assert payload["metrics_by_type"] == {} -def test_autodiscovery_falls_back_to_present_req_suffix_types( +def test_autodiscovery_without_tagged_requirement_types_is_empty( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) @@ -300,7 +263,7 @@ def test_autodiscovery_falls_back_to_present_req_suffix_types( ) payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) - assert set(payload["metrics_by_type"].keys()) == {"feat_req", "tool_req"} + assert payload["metrics_by_type"] == {} def test_autodiscovery_respects_include_external_scope( @@ -315,7 +278,11 @@ def test_autodiscovery_respects_include_external_scope( tmp_path, include_external=True, requirement_types="", - needs_types=[{"directive": "workflow", "tags": []}], + needs_types=[ + {"directive": "tool_req", "tags": ["requirement_excl_process"]}, + {"directive": "feat_req", "tags": ["requirement"]}, + {"directive": "gd_req", "tags": ["requirement"]}, + ], ), ), None, @@ -325,28 +292,6 @@ def test_autodiscovery_respects_include_external_scope( assert set(payload["metrics_by_type"].keys()) == {"feat_req", "gd_req", "tool_req"} -def test_autodiscovery_handles_needitem_like_objects( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeObjectNeedsData) - - metamodel_init._write_metrics_json( - cast( - Sphinx, - _app( - tmp_path, - include_external=False, - requirement_types="", - needs_types=[{"directive": "workflow", "tags": []}], - ), - ), - None, - ) - - payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) - assert set(payload["metrics_by_type"].keys()) == {"feat_req", "tool_req"} - - @pytest.mark.parametrize( ("requirement_types", "include_external", "should_exist", "expected_totals"), [ From a2c640f19a5df72214a4f459eb87a26002d3fef1 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Mon, 1 Jun 2026 17:30:23 +0200 Subject: [PATCH 5/8] WIP: Re-doing everything --- .../requirements/implementation_state.rst | 10 +- src/BUILD | 1 + src/extensions/score_metamodel/BUILD | 1 + src/extensions/score_metamodel/__init__.py | 163 ----------- .../score_metamodel/checks/standards.py | 2 +- .../checks/traceability_dashboard.py | 181 ------------ .../tests/test_traceability_dashboard.py | 218 --------------- .../tests/test_traceability_metrics.py | 206 -------------- .../score_metamodel/traceability_metrics.py | 258 ------------------ src/extensions/score_metrics/BUILD | 40 +++ src/extensions/score_metrics/__init__.py | 152 +++++++++++ .../sphinx_filters.py | 15 +- .../tests/test_sphinx_filters.py | 10 +- .../tests/test_traceability_dashboard.py | 218 +++++++++++++++ .../tests/test_traceability_metrics.py | 206 ++++++++++++++ ...st_traceability_metrics_json_generation.py | 0 .../score_metrics/traceability_dashboard.py | 183 +++++++++++++ .../score_metrics/traceability_metrics.py | 184 +++++++++++++ .../score_source_code_linker/xml_parser.py | 34 +-- src/extensions/score_sphinx_bundle/BUILD | 1 + .../score_sphinx_bundle/__init__.py | 1 + 21 files changed, 1025 insertions(+), 1059 deletions(-) delete mode 100644 src/extensions/score_metamodel/checks/traceability_dashboard.py delete mode 100644 src/extensions/score_metamodel/tests/test_traceability_dashboard.py delete mode 100644 src/extensions/score_metamodel/tests/test_traceability_metrics.py delete mode 100644 src/extensions/score_metamodel/traceability_metrics.py create mode 100644 src/extensions/score_metrics/BUILD create mode 100644 src/extensions/score_metrics/__init__.py rename src/extensions/{score_metamodel => score_metrics}/sphinx_filters.py (92%) rename src/extensions/{score_metamodel => score_metrics}/tests/test_sphinx_filters.py (89%) create mode 100644 src/extensions/score_metrics/tests/test_traceability_dashboard.py create mode 100644 src/extensions/score_metrics/tests/test_traceability_metrics.py rename src/extensions/{score_metamodel => score_metrics}/tests/test_traceability_metrics_json_generation.py (100%) create mode 100644 src/extensions/score_metrics/traceability_dashboard.py create mode 100644 src/extensions/score_metrics/traceability_metrics.py diff --git a/docs/internals/requirements/implementation_state.rst b/docs/internals/requirements/implementation_state.rst index e46df74fa..d2ecc9c24 100644 --- a/docs/internals/requirements/implementation_state.rst +++ b/docs/internals/requirements/implementation_state.rst @@ -29,7 +29,7 @@ Overview .. needpie:: Tool Requirements Status :labels: not implemented, implemented but incomplete traceability, fully linked :colors: red,yellow, green - :filter-func: src.extensions.score_metamodel.checks.traceability_dashboard.pie_requirements_status(tool_req) + :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_requirements_status(tool_req) Jump to evidence tables: @@ -85,28 +85,28 @@ In Detail .. needpie:: Requirements with Codelinks :labels: no codelink, with codelink :colors: red, green - :filter-func: src.extensions.score_metamodel.checks.traceability_dashboard.pie_requirements_with_code_links(tool_req) + :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_requirements_with_code_links(tool_req) .. grid-item-card:: .. needpie:: Requirements with linked tests :labels: no test link, with test link :colors: red, green - :filter-func: src.extensions.score_metamodel.checks.traceability_dashboard.pie_requirements_with_test_links(tool_req) + :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_requirements_with_test_links(tool_req) .. grid-item-card:: .. needpie:: Requirements fully linked (code + tests) :labels: not fully linked, fully linked :colors: orange, green - :filter-func: src.extensions.score_metamodel.checks.traceability_dashboard.pie_requirements_fully_linked(tool_req) + :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_requirements_fully_linked(tool_req) .. grid-item-card:: .. needpie:: Process requirements linked by tool requirements :labels: not linked, linked :colors: red, green - :filter-func: src.extensions.score_metamodel.checks.traceability_dashboard.pie_process_requirements_linked(tool_req,true) + :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_process_requirements_linked(tool_req,true) Process-to-Tool Mapping diff --git a/src/BUILD b/src/BUILD index f12df2fa2..c4515ce09 100644 --- a/src/BUILD +++ b/src/BUILD @@ -41,6 +41,7 @@ filegroup( "//src/extensions/score_source_code_linker:all_sources", "//src/extensions/score_sphinx_bundle:all_sources", "//src/extensions/score_sync_toml:all_sources", + "//src/extensions/score_metrics:all_sources", "//src/helper_lib:all_sources", ], visibility = ["//visibility:public"], diff --git a/src/extensions/score_metamodel/BUILD b/src/extensions/score_metamodel/BUILD index 82fe3eaed..2a182a195 100644 --- a/src/extensions/score_metamodel/BUILD +++ b/src/extensions/score_metamodel/BUILD @@ -56,6 +56,7 @@ py_library( # TODO: Figure out if all requirements are needed or if we can break it down a bit deps = all_requirements + [ "@score_docs_as_code//src/helper_lib", + "@score_docs_as_code//src/extensions/score_metrics:score_metrics", ], ) diff --git a/src/extensions/score_metamodel/__init__.py b/src/extensions/score_metamodel/__init__.py index 8ce2be614..3f837ed84 100644 --- a/src/extensions/score_metamodel/__init__.py +++ b/src/extensions/score_metamodel/__init__.py @@ -31,9 +31,6 @@ ProhibitedWordCheck as ProhibitedWordCheck, ScoreNeedType as ScoreNeedType, ) -from src.extensions.score_metamodel.traceability_metrics import ( - compute_traceability_summary, -) from src.extensions.score_metamodel.yaml_parser import ( default_options as default_options, load_metamodel_data as load_metamodel_data, @@ -99,116 +96,6 @@ def graph_check(func: graph_check_function): return func -def _write_metrics_json(app: Sphinx, exception: Exception | None) -> None: - """Write a schema-v1 metrics.json alongside needs.json in the build output. - - This is the single source of truth for traceability metrics. It runs - inside the Sphinx build so it has access to all needs (local + external) - and produces the same metrics the dashboard pie charts display. - The traceability_gate reads this file to enforce CI thresholds. - """ - if exception: - return - - all_needs: list[Any] = list(SphinxNeedsData(app.env).get_needs_view().values()) - include_external: bool = bool( - getattr(app.config, "score_metamodel_include_external_needs", False) - ) - - raw = str(getattr(app.config, "score_metamodel_requirement_types", "")).strip() - requirement_types = {t.strip() for t in raw.split(",") if t.strip()} - if not requirement_types: - requirement_types = _discover_requirement_types( - app, all_needs, include_external - ) - include_not_implemented = True - - metrics_by_type: dict[str, Any] = {} - if not requirement_types: - logger.info( - "No requirement types configured or discovered; writing empty metrics.json." - ) - else: - for req_type in sorted(requirement_types): - type_summary = compute_traceability_summary( - all_needs=all_needs, - requirement_types={req_type}, - include_not_implemented=include_not_implemented, - filtered_test_types=set(), - include_external=include_external, - ) - metrics_by_type[req_type] = { - "include_not_implemented": type_summary["include_not_implemented"], - "include_external": type_summary["include_external"], - "requirements": type_summary["requirements"], - "tests": type_summary["tests"], - } - - output: dict[str, Any] = { - "schema_version": "1", - "generated_by": "sphinx_build", - "metrics_by_type": metrics_by_type, - } - - out_path = Path(app.outdir) / "metrics.json" - out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_text(json.dumps(output, indent=2), encoding="utf-8") - logger.info(f"Traceability metrics written to: {out_path}") - - -def _get_need_value(need: Any, key: str, default: Any = None) -> Any: - return need.get(key, default) - - -def _as_requirement_directive(need_type: Any) -> str | None: - if not isinstance(need_type, dict): - return None - directive = need_type.get("directive") - tags = need_type.get("tags", []) - if not isinstance(directive, str) or not isinstance(tags, list): - return None - normalized = {str(tag).strip() for tag in tags} - if "requirement_excl_process" in normalized or "requirement" in normalized: - return directive - return None - - -def _discover_requirement_types( - app: Sphinx, all_needs: list[Any], include_external: bool -) -> set[str]: - """Discover requirement directives that are both tagged and present.""" - tagged_requirements: set[str] = set() - needs_types = getattr(app.config, "needs_types", []) - for need_type in needs_types or []: - directive = _as_requirement_directive(need_type) - if directive: - tagged_requirements.add(directive) - - present_types: set[str] = set() - for need in all_needs: - is_external = bool(_get_need_value(need, "is_external", False)) - if not include_external and is_external: - continue - need_type: Any = _get_need_value(need, "type", None) - if isinstance(need_type, str): - present_types.add(need_type) - - discovered = tagged_requirements.intersection(present_types) - - if tagged_requirements and not discovered: - logger.warning( - "No requirement types discovered in current build for tagged " - "needs_types requirement directives." - ) - - if discovered: - logger.info( - "score_metamodel_requirement_types is not configured; " - f"using discovered requirement types: {', '.join(sorted(discovered))}" - ) - return discovered - - def _run_checks(app: Sphinx, exception: Exception | None) -> None: # Do not run checks if an exception occurred during build if exception: @@ -276,32 +163,6 @@ def is_check_enabled(check: local_check_function | graph_check_function): ) -def _configure_traceability_dashboard(app: Sphinx, config: object) -> None: - """Propagate repo-level traceability settings to dashboard filters.""" - from src.extensions.score_metamodel.checks.traceability_dashboard import ( - set_default_include_external, - ) - - include_external = bool( - getattr(config, "score_metamodel_include_external_needs", False) - ) - set_default_include_external(include_external) - - -def _remove_prefix(word: str, prefixes: list[str]) -> str: - for prefix in prefixes or []: - if isinstance(word, str) and word.startswith(prefix): - return word.removeprefix(prefix) - return word - - -def _get_need_type_for_need(app: Sphinx, need: NeedItem) -> ScoreNeedType: - for nt in app.config.needs_types: - if nt["directive"] == need["type"]: - return nt - raise ValueError(f"Need type {need['type']} not found in needs_types") - - def _resolve_linkable_types( link_name: str, link_value: str, @@ -415,30 +276,6 @@ def setup(app: Sphinx) -> dict[str, str | bool]: ), ) - app.add_config_value( - "score_metamodel_requirement_types", - "", - rebuild="env", - description=( - "Comma-separated list of need types treated as requirements for " - "traceability metrics. If empty, requirement types are autodiscovered " - "from needs_types tags (requirement, requirement_excl_process)." - ), - ) - - app.add_config_value( - "score_metamodel_include_external_needs", - False, - rebuild="env", - description=( - "When True, include external requirements in dashboard and CI metrics. " - "Default is False so each repo gates only its own needs." - ), - ) - - _ = app.connect("config-inited", _configure_traceability_dashboard, priority=498) - - _ = app.connect("build-finished", _write_metrics_json) _ = app.connect("build-finished", _run_checks) return { diff --git a/src/extensions/score_metamodel/checks/standards.py b/src/extensions/score_metamodel/checks/standards.py index a04d176ba..70261641f 100644 --- a/src/extensions/score_metamodel/checks/standards.py +++ b/src/extensions/score_metamodel/checks/standards.py @@ -18,7 +18,7 @@ from sphinx_needs.need_item import NeedItem -from ..sphinx_filters import ( +from score_metrics.sphinx_filters import ( generic_pie_items_by_tag, generic_pie_items_in_relationships, generic_pie_linked_items, diff --git a/src/extensions/score_metamodel/checks/traceability_dashboard.py b/src/extensions/score_metamodel/checks/traceability_dashboard.py deleted file mode 100644 index a59087928..000000000 --- a/src/extensions/score_metamodel/checks/traceability_dashboard.py +++ /dev/null @@ -1,181 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -# ╓ ╖ -# ║ Some portions generated by Github Copilot ║ -# ╙ ╜ - -"""Needpie filter functions backed by shared traceability metric calculations.""" - -from __future__ import annotations - -from collections.abc import Sequence -from typing import Any - -from sphinx_needs.need_item import NeedItem - -from ..traceability_metrics import compute_traceability_summary, filter_requirements - -_DEFAULT_INCLUDE_EXTERNAL = False - - -def set_default_include_external(include_external: bool) -> None: - """Configure default behaviour for including external requirements.""" - global _DEFAULT_INCLUDE_EXTERNAL - _DEFAULT_INCLUDE_EXTERNAL = bool(include_external) - - -def _include_external(kwargs: dict[str, str | int | float]) -> bool: - """Read include_external override from filter args, else use configured default.""" - raw = kwargs.get("arg2") - if raw is None: - return _DEFAULT_INCLUDE_EXTERNAL - text = str(raw).strip().lower() - return text in {"1", "true", "yes", "on"} - - -def _requirement_types(kwargs: dict[str, str | int | float]) -> set[str]: - raw = str(kwargs.get("arg1", "tool_req")).strip() - values = {value.strip() for value in raw.split(",") if value.strip()} - return values or {"tool_req"} - - -def pie_requirements_status( - needs: Sequence[NeedItem | dict[str, Any]], - results: list[int], - **kwargs: str | int | float, -) -> None: - """Dashboard status split: not implemented, implemented/incomplete, fully linked.""" - req_types = _requirement_types(kwargs) - include_external = _include_external(kwargs) - - all_requirements = filter_requirements( - needs, - requirement_types=req_types, - include_not_implemented=True, - include_external=include_external, - ) - implemented_requirements = filter_requirements( - needs, - requirement_types=req_types, - include_not_implemented=False, - include_external=include_external, - ) - summary = compute_traceability_summary( - all_needs=needs, - requirement_types=req_types, - include_not_implemented=False, - filtered_test_types=set(), - include_external=include_external, - ) - - not_implemented = len(all_requirements) - len(implemented_requirements) - fully_linked = int(summary["requirements"]["fully_linked"]) - implemented_incomplete = len(implemented_requirements) - fully_linked - - results.append(not_implemented) - results.append(implemented_incomplete) - results.append(fully_linked) - - -def pie_requirements_with_code_links( - needs: Sequence[NeedItem | dict[str, Any]], - results: list[int], - **kwargs: str | int | float, -) -> None: - """Dashboard split: requirements with and without source code links.""" - req_types = _requirement_types(kwargs) - include_external = _include_external(kwargs) - summary = compute_traceability_summary( - all_needs=needs, - requirement_types=req_types, - include_not_implemented=True, - filtered_test_types=set(), - include_external=include_external, - ) - - total = int(summary["requirements"]["total"]) - with_code = int(summary["requirements"]["with_code_link"]) - - results.append(total - with_code) - results.append(with_code) - - -def pie_requirements_with_test_links( - needs: Sequence[NeedItem | dict[str, Any]], - results: list[int], - **kwargs: str | int | float, -) -> None: - """Dashboard split: requirements with and without testcase links.""" - req_types = _requirement_types(kwargs) - include_external = _include_external(kwargs) - summary = compute_traceability_summary( - all_needs=needs, - requirement_types=req_types, - include_not_implemented=True, - filtered_test_types=set(), - include_external=include_external, - ) - - total = int(summary["requirements"]["total"]) - with_test = int(summary["requirements"]["with_test_link"]) - - results.append(total - with_test) - results.append(with_test) - - -def pie_requirements_fully_linked( - needs: Sequence[NeedItem | dict[str, Any]], - results: list[int], - **kwargs: str | int | float, -) -> None: - """Dashboard split: requirements fully linked vs incomplete.""" - req_types = _requirement_types(kwargs) - include_external = _include_external(kwargs) - summary = compute_traceability_summary( - all_needs=needs, - requirement_types=req_types, - include_not_implemented=True, - filtered_test_types=set(), - include_external=include_external, - ) - - total = int(summary["requirements"]["total"]) - fully_linked = int(summary["requirements"]["fully_linked"]) - - results.append(total - fully_linked) - results.append(fully_linked) - - -def pie_process_requirements_linked( - needs: Sequence[NeedItem | dict[str, Any]], - results: list[int], - **kwargs: str | int | float, -) -> None: - """Dashboard split: process requirements linked vs not linked.""" - req_types = _requirement_types(kwargs) - include_external = _include_external(kwargs) - summary = compute_traceability_summary( - all_needs=needs, - requirement_types=req_types, - include_not_implemented=True, - filtered_test_types=set(), - include_external=include_external, - ) - - process_requirements = summary["process_requirements"] - total = int(process_requirements["total"]) - linked = int(process_requirements["linked"]) - - results.append(total - linked) - results.append(linked) diff --git a/src/extensions/score_metamodel/tests/test_traceability_dashboard.py b/src/extensions/score_metamodel/tests/test_traceability_dashboard.py deleted file mode 100644 index 2e06b3b7d..000000000 --- a/src/extensions/score_metamodel/tests/test_traceability_dashboard.py +++ /dev/null @@ -1,218 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -"""Tests that dashboard filters follow local/external settings.""" - -from collections.abc import Sequence -from typing import Any - -import pytest - -from src.extensions.score_metamodel.checks import traceability_dashboard -from src.extensions.score_metamodel.checks.traceability_dashboard import ( - pie_process_requirements_linked, - pie_requirements_fully_linked, - pie_requirements_with_code_links, - pie_requirements_with_test_links, - set_default_include_external, -) -from src.extensions.score_metamodel.traceability_metrics import ( - compute_traceability_summary, -) - - -def _needs() -> list[dict[str, object]]: - return [ - { - "id": "LOCAL_REQ", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "", - "testlink": "", - "is_external": False, - }, - { - "id": "LOCAL_SYS_REQ", - "type": "sys_req", - "implemented": "YES", - "source_code_link": "", - "testlink": "T_LOCAL", - "is_external": False, - }, - { - "id": "EXT_REQ", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "src/ext.py:10", - "testlink": "T_EXT", - "is_external": True, - }, - ] - - -def test_dashboard_defaults_to_local_only() -> None: - set_default_include_external(False) - - results: list[int] = [] - pie_requirements_with_code_links(_needs(), results, arg1="tool_req") - - summary = compute_traceability_summary( - all_needs=_needs(), - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=False, - ) - - assert results == [1, 0] - assert results == [ - summary["requirements"]["total"] - summary["requirements"]["with_code_link"], - summary["requirements"]["with_code_link"], - ] - - -def test_dashboard_can_include_external_via_default_flag() -> None: - set_default_include_external(True) - - results: list[int] = [] - pie_requirements_with_code_links(_needs(), results, arg1="tool_req") - - summary = compute_traceability_summary( - all_needs=_needs(), - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=True, - ) - - assert results == [1, 1] - assert results == [ - summary["requirements"]["total"] - summary["requirements"]["with_code_link"], - summary["requirements"]["with_code_link"], - ] - - -def test_dashboard_filter_arg_can_override_default() -> None: - set_default_include_external(True) - - results: list[int] = [] - pie_requirements_with_code_links(_needs(), results, arg1="tool_req", arg2="false") - - assert results == [1, 0] - - -def test_requirements_with_test_links_default_local_only() -> None: - set_default_include_external(False) - - results: list[int] = [] - pie_requirements_with_test_links(_needs(), results, arg1="tool_req") - - summary = compute_traceability_summary( - all_needs=_needs(), - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=False, - ) - - assert results == [1, 0] - assert results == [ - summary["requirements"]["total"] - summary["requirements"]["with_test_link"], - summary["requirements"]["with_test_link"], - ] - - -def test_requirements_with_test_links_can_override_include_external() -> None: - set_default_include_external(False) - - results: list[int] = [] - pie_requirements_with_test_links(_needs(), results, arg1="tool_req", arg2="true") - - assert results == [1, 1] - - -def test_requirements_with_test_links_parses_multiple_types() -> None: - set_default_include_external(False) - - results: list[int] = [] - pie_requirements_with_test_links(_needs(), results, arg1="tool_req,sys_req") - - assert results == [1, 1] - - -def test_requirements_fully_linked_uses_shared_summary() -> None: - set_default_include_external(False) - - results: list[int] = [] - pie_requirements_fully_linked(_needs(), results, arg1="tool_req") - - summary = compute_traceability_summary( - all_needs=_needs(), - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=False, - ) - - assert results == [1, 0] - assert results == [ - summary["requirements"]["total"] - summary["requirements"]["fully_linked"], - summary["requirements"]["fully_linked"], - ] - - -def test_requirements_fully_linked_can_include_external() -> None: - set_default_include_external(True) - - results: list[int] = [] - pie_requirements_fully_linked(_needs(), results, arg1="tool_req") - - assert results == [1, 1] - - -def test_process_requirements_linked_uses_stream_a_process_requirement_totals( - monkeypatch: pytest.MonkeyPatch, -) -> None: - captured: dict[str, object] = {} - - def _fake_summary( - all_needs: Sequence[dict[str, Any]], - requirement_types: set[str], - include_not_implemented: bool, - filtered_test_types: set[str], - include_external: bool, - ) -> dict[str, dict[str, int]]: - captured["all_needs"] = all_needs - captured["requirement_types"] = requirement_types - captured["include_not_implemented"] = include_not_implemented - captured["filtered_test_types"] = filtered_test_types - captured["include_external"] = include_external - return { - "requirements": {"total": 99, "linked": 0}, - "process_requirements": {"total": 4, "linked": 3}, - } - - monkeypatch.setattr( - traceability_dashboard, "compute_traceability_summary", _fake_summary - ) - - results: list[int] = [] - pie_process_requirements_linked( - _needs(), results, arg1="tool_req,sys_req", arg2="true" - ) - - assert results == [1, 3] - assert captured["requirement_types"] == {"tool_req", "sys_req"} - assert captured["include_not_implemented"] is True - assert captured["filtered_test_types"] == set() - assert captured["include_external"] is True diff --git a/src/extensions/score_metamodel/tests/test_traceability_metrics.py b/src/extensions/score_metamodel/tests/test_traceability_metrics.py deleted file mode 100644 index 850f064f6..000000000 --- a/src/extensions/score_metamodel/tests/test_traceability_metrics.py +++ /dev/null @@ -1,206 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -"""Unit tests for traceability_metrics include_external handling.""" - -from src.extensions.score_metamodel.traceability_metrics import ( - compute_traceability_summary, - filter_requirements, -) - - -def _needs() -> list[dict[str, object]]: - return [ - { - "id": "LOCAL_REQ", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "src/local.py:1", - "testlink": "tests/test_local.py::test_ok", - "is_external": False, - }, - { - "id": "EXT_REQ", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "src/external.py:9", - "testlink": "tests/test_external.py::test_ok", - "is_external": True, - }, - { - "id": "TC_1", - "type": "testcase", - "partially_verifies": "LOCAL_REQ", - "fully_verifies": "", - "is_external": False, - }, - ] - - -def test_filter_requirements_defaults_to_local_only() -> None: - filtered = filter_requirements( - _needs(), - requirement_types={"tool_req"}, - include_not_implemented=True, - ) - - assert [need["id"] for need in filtered] == ["LOCAL_REQ"] - - -def test_filter_requirements_can_include_external_needs() -> None: - filtered = filter_requirements( - _needs(), - requirement_types={"tool_req"}, - include_not_implemented=True, - include_external=True, - ) - - assert sorted(need["id"] for need in filtered) == ["EXT_REQ", "LOCAL_REQ"] - - -def test_compute_traceability_summary_propagates_include_external() -> None: - summary_local = compute_traceability_summary( - all_needs=_needs(), - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=False, - ) - summary_all = compute_traceability_summary( - all_needs=_needs(), - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=True, - ) - - assert summary_local["include_external"] is False - assert summary_local["requirements"]["total"] == 1 - assert summary_all["include_external"] is True - assert summary_all["requirements"]["total"] == 2 - - -def test_compute_traceability_summary_process_requirements_summary() -> None: - summary = compute_traceability_summary( - all_needs=[ - { - "id": "TOOL_REQ_1", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "src/req.py:10", - "testlink": "tests/test_req.py::test_ok", - "satisfies": "PR_LOCAL_1,OTHER_REQ", - "is_external": False, - }, - { - "id": "TOOL_REQ_2", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "src/req.py:20", - "testlink": "tests/test_req.py::test_ok_2", - "satisfies": ["PR_LOCAL_1", "PR_LOCAL_2"], - "is_external": False, - }, - { - "id": "PR_LOCAL_1", - "type": "process_req", - "is_external": False, - }, - { - "id": "PR_LOCAL_2", - "type": "gd_req", - "is_external": False, - }, - { - "id": "PR_LOCAL_3", - "type": "gd_req", - "is_external": False, - }, - ], - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=False, - ) - - process_requirements = summary["process_requirements"] - - assert process_requirements["total"] == 3 - assert process_requirements["linked_by_tool_requirements"] == 2 - assert process_requirements["linked_by_tool_requirements_pct"] == (2 / 3) * 100 - assert process_requirements["unlinked_ids"] == ["PR_LOCAL_3"] - - -def test_compute_traceability_summary_process_requirements_respects_include_external() -> ( - None -): - all_needs = [ - { - "id": "TOOL_REQ_LOCAL", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "src/local.py:1", - "testlink": "tests/test_local.py::test_ok", - "satisfies": "PR_LOCAL", - "is_external": False, - }, - { - "id": "TOOL_REQ_EXTERNAL", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "src/external.py:1", - "testlink": "tests/test_external.py::test_ok", - "satisfies": "PR_EXTERNAL", - "is_external": True, - }, - { - "id": "PR_LOCAL", - "type": "gd_req", - "is_external": False, - }, - { - "id": "PR_EXTERNAL", - "type": "gd_req", - "is_external": True, - }, - ] - - summary_local = compute_traceability_summary( - all_needs=all_needs, - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=False, - ) - summary_all = compute_traceability_summary( - all_needs=all_needs, - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=True, - ) - - assert summary_local["process_requirements"] == { - "total": 1, - "linked": 1, - "linked_by_tool_requirements": 1, - "linked_by_tool_requirements_pct": 100.0, - "unlinked_ids": [], - } - assert summary_all["process_requirements"] == { - "total": 2, - "linked": 2, - "linked_by_tool_requirements": 2, - "linked_by_tool_requirements_pct": 100.0, - "unlinked_ids": [], - } diff --git a/src/extensions/score_metamodel/traceability_metrics.py b/src/extensions/score_metamodel/traceability_metrics.py deleted file mode 100644 index 08e1723e9..000000000 --- a/src/extensions/score_metamodel/traceability_metrics.py +++ /dev/null @@ -1,258 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -# ╓ ╖ -# ║ Some portions generated by Github Copilot ║ -# ╙ ╜ - -"""Shared traceability metric calculations for CI checks and dashboards.""" - -from __future__ import annotations - -from collections.abc import Sequence -from typing import Any - - -def is_non_empty(value: Any) -> bool: - """Return True if value should be treated as present for traceability checks.""" - if isinstance(value, str): - return bool(value.strip()) - return bool(value) - - -def parse_need_id_list(value: Any) -> list[str]: - """Normalize need-id lists encoded as CSV strings or string lists.""" - if value is None: - return [] - if isinstance(value, str): - return [item.strip() for item in value.split(",") if item.strip()] - if isinstance(value, list): - out: list[str] = [] - for item in value: - if isinstance(item, str) and item.strip(): - out.append(item.strip()) - return out - return [] - - -def safe_percent(numerator: int, denominator: int) -> float: - """Return percentage in range [0, 100], treating empty denominator as 100%.""" - if denominator == 0: - return 100.0 - return (numerator / denominator) * 100.0 - - -def filter_requirements( - all_needs: Sequence[Any], - requirement_types: set[str], - include_not_implemented: bool, - include_external: bool = False, -) -> list[Any]: - """Extract requirements by type, implementation state, and origin.""" - requirements: list[dict[str, Any]] = [] - for need in all_needs: - need_type = str(need.get("type", "")).strip() - if need_type not in requirement_types: - continue - if not include_external and need.get("is_external", False): - continue - if not include_not_implemented: - implemented = str(need.get("implemented", "")).upper().strip() - if implemented not in {"YES", "PARTIAL"}: - continue - requirements.append(need) - return requirements - - -def calculate_requirement_metrics( - requirements: Sequence[Any], -) -> dict[str, Any]: - """Calculate requirement traceability statistics for links and completeness.""" - total = len(requirements) - with_code = sum( - 1 for need in requirements if is_non_empty(need.get("source_code_link")) - ) - with_test = sum(1 for need in requirements if is_non_empty(need.get("testlink"))) - fully_linked = sum( - 1 - for need in requirements - if is_non_empty(need.get("source_code_link")) - and is_non_empty(need.get("testlink")) - ) - - missing_code_ids = [ - str(need.get("id", "")) - for need in requirements - if not is_non_empty(need.get("source_code_link")) and need.get("id") - ] - missing_test_ids = [ - str(need.get("id", "")) - for need in requirements - if not is_non_empty(need.get("testlink")) and need.get("id") - ] - not_fully_linked_ids = [ - str(need.get("id", "")) - for need in requirements - if ( - ( - not is_non_empty(need.get("source_code_link")) - or not is_non_empty(need.get("testlink")) - ) - and need.get("id") - ) - ] - - return { - "total": total, - "with_code_link": with_code, - "with_test_link": with_test, - "fully_linked": fully_linked, - "with_code_link_pct": safe_percent(with_code, total), - "with_test_link_pct": safe_percent(with_test, total), - "fully_linked_pct": safe_percent(fully_linked, total), - "missing_code_link_ids": sorted(missing_code_ids), - "missing_test_link_ids": sorted(missing_test_ids), - "not_fully_linked_ids": sorted(not_fully_linked_ids), - } - - -def calculate_test_metrics( - all_needs: Sequence[Any], - requirement_ids: set[str], - filtered_test_types: set[str], -) -> dict[str, Any]: - """Calculate testcase linkage and broken testcase-reference statistics.""" - testcases = [ - need for need in all_needs if str(need.get("type", "")).strip() == "testcase" - ] - if filtered_test_types: - testcases = [ - need - for need in testcases - if str(need.get("test_type", need.get("TestType", ""))).strip() - in filtered_test_types - ] - - tests_total = len(testcases) - tests_linked = 0 - broken_references: list[dict[str, str]] = [] - - for test_need in testcases: - test_id = str(test_need.get("id", "")) - partially = parse_need_id_list( - test_need.get("partially_verifies", test_need.get("PartiallyVerifies")) - ) - fully = parse_need_id_list( - test_need.get("fully_verifies", test_need.get("FullyVerifies")) - ) - refs = partially + fully - if refs: - tests_linked += 1 - for ref in refs: - if ref not in requirement_ids: - broken_references.append({"testcase": test_id, "missing_need": ref}) - - return { - "total": tests_total, - "filtered_test_types": sorted(filtered_test_types), - "linked_to_requirements": tests_linked, - "linked_to_requirements_pct": safe_percent(tests_linked, tests_total), - "broken_references": broken_references, - } - - -def calculate_process_requirement_metrics( - all_needs: Sequence[Any], - include_not_implemented: bool, - include_external: bool, -) -> dict[str, Any]: - """Calculate process-requirement coverage via tool_req ``satisfies`` links.""" - process_requirements = [ - need - for need in all_needs - if str(need.get("type", "")).strip() in {"gd_req", "process_req"} - and (include_external or not need.get("is_external", False)) - ] - process_requirement_ids = { - str(need.get("id", "")).strip() - for need in process_requirements - if need.get("id") - } - - tool_requirements = filter_requirements( - all_needs, - requirement_types={"tool_req"}, - include_not_implemented=include_not_implemented, - include_external=include_external, - ) - - linked_process_requirement_ids: set[str] = set() - for need in tool_requirements: - satisfies_ids = parse_need_id_list(need.get("satisfies", need.get("Satisfies"))) - for ref_id in satisfies_ids: - if ref_id in process_requirement_ids: - linked_process_requirement_ids.add(ref_id) - - total = len(process_requirement_ids) - linked_by_tool_requirements = len(linked_process_requirement_ids) - unlinked_ids = sorted(process_requirement_ids - linked_process_requirement_ids) - - return { - "total": total, - "linked": linked_by_tool_requirements, - "linked_by_tool_requirements": linked_by_tool_requirements, - "linked_by_tool_requirements_pct": safe_percent( - linked_by_tool_requirements, total - ), - "unlinked_ids": unlinked_ids, - } - - -def compute_traceability_summary( - all_needs: Sequence[Any], - requirement_types: set[str], - include_not_implemented: bool, - filtered_test_types: set[str], - include_external: bool = False, -) -> dict[str, Any]: - """Return full CI/dashboard summary using one shared metric implementation.""" - requirements = filter_requirements( - all_needs, - requirement_types=requirement_types, - include_not_implemented=include_not_implemented, - include_external=include_external, - ) - requirement_ids = { - str(need.get("id", "")).strip() for need in requirements if need.get("id") - } - - req_metrics = calculate_requirement_metrics(requirements) - test_metrics = calculate_test_metrics( - all_needs, - requirement_ids=requirement_ids, - filtered_test_types=filtered_test_types, - ) - process_requirement_metrics = calculate_process_requirement_metrics( - all_needs, - include_not_implemented=include_not_implemented, - include_external=include_external, - ) - - return { - "requirement_types": sorted(requirement_types), - "include_not_implemented": include_not_implemented, - "include_external": include_external, - "requirements": req_metrics, - "tests": test_metrics, - "process_requirements": process_requirement_metrics, - } diff --git a/src/extensions/score_metrics/BUILD b/src/extensions/score_metrics/BUILD new file mode 100644 index 000000000..b288353ec --- /dev/null +++ b/src/extensions/score_metrics/BUILD @@ -0,0 +1,40 @@ +load("@aspect_rules_py//py:defs.bzl", "py_library") +load("@docs_as_code_hub_env//:requirements.bzl", "all_requirements") +load("//:score_pytest.bzl", "score_pytest") + +filegroup( + name = "all_sources", + srcs = glob( + [ + "*.py", + ], + ), + visibility = ["//visibility:public"], +) + +filegroup( + name = "tests", + srcs = glob( + ["tests/*.py"], + ), +) + +py_library( + name = "score_metrics", + srcs = [":all_sources"], + imports = ["."], + visibility = ["//visibility:public"], + # TODO: Figure out if all requirements are needed or if we can break it down a bit + deps = all_requirements + [ + "@score_docs_as_code//src/helper_lib", + ], +) + +score_pytest( + name = "score_metrics_test", + size = "small", + srcs = glob(["tests/*.py"]), + pytest_config = "//:pyproject.toml", + # All requirements already in the library so no need to have it double + deps = [":score_metrics"], +) diff --git a/src/extensions/score_metrics/__init__.py b/src/extensions/score_metrics/__init__.py new file mode 100644 index 000000000..b572d044f --- /dev/null +++ b/src/extensions/score_metrics/__init__.py @@ -0,0 +1,152 @@ +import json +import os +from pathlib import Path +from typing import Any + +from sphinx.application import Sphinx +from sphinx_needs import logging +from sphinx_needs.data import NeedsView, SphinxNeedsData +from sphinx_needs.need_item import NeedItem +from score_metrics.traceability_metrics import ( + compute_traceability_summary, +) + +logger = logging.get_logger(__name__) + + +def _configure_traceability_dashboard(app: Sphinx, config: object) -> None: + """Propagate repo-level traceability settings to dashboard filters.""" + from src.extensions.score_metrics.traceability_dashboard import ( + set_default_include_external, + ) + + include_external = bool( + getattr(config, "score_metamodel_include_external_needs", False) + ) + set_default_include_external(include_external) + + +def get_need_types_by_tags(needs_types: list[NeedItem], tags: set[str]) -> list[str]: + found_need_types: list[str] = [] + for need_type in needs_types: + found_tag_set = set(need_type["tags"]) + if tags.intersection(found_tag_set): + found_need_types.append(need_type["type"]) + return found_need_types + + +def _discover_requirement_types( + app: Sphinx, all_needs: list[NeedItem] +) -> set[NeedItem]: + """Discover requirement directives that are both tagged and present.""" + tagged_requirements_types = get_need_types_by_tags( + all_needs, {"requirement", "requirement_excl_process"} + ) + all_tagged_requirements = ( + SphinxNeedsData(app.env) + .get_needs_view() + .filter_types(tagged_requirements_types) + .values() + ) + if not all_tagged_requirements: + logger.warning( + "No requirement types discovered in current build for tagged " + "needs_types requirement directives." + ) + + return set(all_tagged_requirements) + + +def _write_metrics_json(app: Sphinx, exception: Exception | None) -> None: + """Write a schema-v1 metrics.json alongside needs.json in the build output. + + This is the single source of truth for traceability metrics. It runs + inside the Sphinx build so it has access to all needs (local + external) + and produces the same metrics the dashboard pie charts display. + The traceability_gate reads this file to enforce CI thresholds. + """ + print("WE ARE IN WRITE METRICS JSON") + if exception: + logger.error("===================================") + logger.error("===================================") + logger.error("===================================") + logger.error(f"EXCEPTION RAISED: {exception}") + return + + include_external: bool = bool( + getattr(app.config, "score_metamodel_include_external_needs", False) + ) + all_needs: NeedsView = SphinxNeedsData(app.env).get_needs_view() + + raw = getattr(app.config, "score_metamodel_requirement_types", "").strip() + requirement_types = [t.strip() for t in raw.split(",") if t.strip()] + if requirement_types: + requirement_types = all_needs.filter_types(requirement_types) + + metrics_by_type: dict[str, Any] = {} + if not requirement_types: + logger.info( + "No requirement types configured or discovered; writing empty metrics.json." + ) + else: + for req_type in sorted(requirement_types): + type_summary = compute_traceability_summary( + all_needs=all_needs, + filtered_test_types=set(), + include_external=include_external, + ) + metrics_by_type[req_type] = { + "include_not_implemented": type_summary["include_not_implemented"], + "include_external": type_summary["include_external"], + "requirements": type_summary["requirements"], + "tests": type_summary["tests"], + } + + output: dict[str, Any] = { + "schema_version": "1", + "generated_by": "sphinx_build", + "metrics_by_type": metrics_by_type, + } + print("===================================") + print("===================================") + print("===================================") + print(f"writing metrics.json to {Path(app.outdir) / 'metrics.json'}") + print("===================================") + print("===================================") + print("===================================") + out_path = Path(app.outdir) / "metrics.json" + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(output, indent=2), encoding="utf-8") + print(f"Traceability metrics written to: {out_path}") + + +def setup(app: Sphinx) -> dict[str, str | bool]: + app.add_config_value( + "score_metamodel_requirement_types", + "", + rebuild="env", + description=( + "Comma-separated list of need types treated as requirements for " + "traceability metrics. If empty, requirement types are autodiscovered " + "from needs_types tags (requirement, requirement_excl_process)." + ), + ) + + app.add_config_value( + "score_metamodel_include_external_needs", + False, + rebuild="env", + description=( + "When True, include external requirements in dashboard and CI metrics. " + "Default is False so each repo gates only its own needs." + ), + ) + + _ = app.connect("config-inited", _configure_traceability_dashboard, priority=498) + _ = app.connect("build-finished", _write_metrics_json, priority=499) + + return { + "version": "0.1", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/src/extensions/score_metamodel/sphinx_filters.py b/src/extensions/score_metrics/sphinx_filters.py similarity index 92% rename from src/extensions/score_metamodel/sphinx_filters.py rename to src/extensions/score_metrics/sphinx_filters.py index 8fd5fca19..bf6f2d685 100644 --- a/src/extensions/score_metamodel/sphinx_filters.py +++ b/src/extensions/score_metrics/sphinx_filters.py @@ -55,7 +55,7 @@ def _matches_source_selector(need: NeedItem, selector: str) -> bool: def generic_pie_linked_items( - needs: list[NeedItem], results: list[int], **kwargs: str | int | float + needs: list[NeedItem], results: list[int], **kwargs: dict[str, str | int | float] ) -> None: """Count target IDs by whether they are linked by selected source needs. @@ -63,6 +63,7 @@ def generic_pie_linked_items( selector prefix, matched against source ``type`` and ``id``), and ``arg3`` (link field name, default ``complies``). """ + results = [] id_prefix = str(kwargs.get("arg1", "")) source_selector = str(kwargs.get("arg2", "")) link_field = str(kwargs.get("arg3", "complies")) @@ -89,7 +90,7 @@ def generic_pie_linked_items( def generic_pie_items_by_tag( - needs: list[NeedItem], results: list[int], **kwargs: str | int | float + needs: list[NeedItem], results: list[int], **kwargs: dict[str, str | int | float] ) -> None: """Count tagged items split by whether selected source needs link them. @@ -123,7 +124,7 @@ def generic_pie_items_by_tag( def generic_pie_items_in_relationships( - needs: list[NeedItem], results: list[int], **kwargs: str | int | float + needs: list[NeedItem], results: list[int], **kwargs: dict[str, str | int | float] ) -> None: """Count items of a given type by how many container items reference them. @@ -164,3 +165,11 @@ def generic_pie_items_in_relationships( results.append(not_referenced) results.append(referenced_once) results.append(referenced_multiple) + + +def get_metrics_from_generated_json( + needs: list[NeedItem], results: list[int], **kwargs: str | int | float +) -> None: + metrics_json_path = str(kwargs.get("arg1", "_build/metrics.json")) + # TODO: Read the Metrics.json and give back the data + results diff --git a/src/extensions/score_metamodel/tests/test_sphinx_filters.py b/src/extensions/score_metrics/tests/test_sphinx_filters.py similarity index 89% rename from src/extensions/score_metamodel/tests/test_sphinx_filters.py rename to src/extensions/score_metrics/tests/test_sphinx_filters.py index c065a01b7..94de5ed12 100644 --- a/src/extensions/score_metamodel/tests/test_sphinx_filters.py +++ b/src/extensions/score_metrics/tests/test_sphinx_filters.py @@ -15,7 +15,7 @@ from sphinx_needs.need_item import NeedItem -from src.extensions.score_metamodel.sphinx_filters import ( +from score_metrics.sphinx_filters import ( generic_pie_items_by_tag, generic_pie_linked_items, ) @@ -39,9 +39,7 @@ def test_generic_pie_linked_items_matches_source_by_id_prefix() -> None: generic_pie_linked_items( needs, results, - arg1="std_req__iso26262__", - arg2="gd_", - arg3="complies", + kwargs={"arg1": "std_req__iso26262__", "arg2": "gd_", "arg3": "complies"}, ) assert results == [1, 0] @@ -66,9 +64,7 @@ def test_generic_pie_items_by_tag_matches_source_by_id_prefix() -> None: generic_pie_items_by_tag( needs, results, - arg1="aspice40_man5", - arg2="gd_", - arg3="complies", + kwargs={"arg1": "aspice40_man5", "arg2": "gd_", "arg3": "complies"}, ) assert results == [1, 1] diff --git a/src/extensions/score_metrics/tests/test_traceability_dashboard.py b/src/extensions/score_metrics/tests/test_traceability_dashboard.py new file mode 100644 index 000000000..763f5a0b0 --- /dev/null +++ b/src/extensions/score_metrics/tests/test_traceability_dashboard.py @@ -0,0 +1,218 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Tests that dashboard filters follow local/external settings.""" + +from collections.abc import Sequence +from typing import Any + +import pytest + +#from src.extensions.score_metamodel.checks import traceability_dashboard +# from src.extensions.score_metrics.traceability_dashboard import ( +# pie_process_requirements_linked, +# pie_requirements_fully_linked, +# pie_requirements_with_code_links, +# pie_requirements_with_test_links, +# set_default_include_external, +# ) +# from src.extensions.score_metrics.traceability_metrics import ( +# compute_traceability_summary, +# ) +# +# +# def _needs() -> list[dict[str, object]]: +# return [ +# { +# "id": "LOCAL_REQ", +# "type": "tool_req", +# "implemented": "YES", +# "source_code_link": "", +# "testlink": "", +# "is_external": False, +# }, +# { +# "id": "LOCAL_SYS_REQ", +# "type": "sys_req", +# "implemented": "YES", +# "source_code_link": "", +# "testlink": "T_LOCAL", +# "is_external": False, +# }, +# { +# "id": "EXT_REQ", +# "type": "tool_req", +# "implemented": "YES", +# "source_code_link": "src/ext.py:10", +# "testlink": "T_EXT", +# "is_external": True, +# }, +# ] +# +# +# def test_dashboard_defaults_to_local_only() -> None: +# set_default_include_external(False) +# +# results: list[int] = [] +# pie_requirements_with_code_links(_needs(), results, arg1="tool_req") +# +# summary = compute_traceability_summary( +# all_needs=_needs(), +# requirement_types={"tool_req"}, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=False, +# ) +# +# assert results == [1, 0] +# assert results == [ +# summary["requirements"]["total"] - summary["requirements"]["with_code_link"], +# summary["requirements"]["with_code_link"], +# ] +# +# +# def test_dashboard_can_include_external_via_default_flag() -> None: +# set_default_include_external(True) +# +# results: list[int] = [] +# pie_requirements_with_code_links(_needs(), results, arg1="tool_req") +# +# summary = compute_traceability_summary( +# all_needs=_needs(), +# requirement_types={"tool_req"}, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=True, +# ) +# +# assert results == [1, 1] +# assert results == [ +# summary["requirements"]["total"] - summary["requirements"]["with_code_link"], +# summary["requirements"]["with_code_link"], +# ] +# +# +# def test_dashboard_filter_arg_can_override_default() -> None: +# set_default_include_external(True) +# +# results: list[int] = [] +# pie_requirements_with_code_links(_needs(), results, arg1="tool_req", arg2="false") +# +# assert results == [1, 0] +# +# +# def test_requirements_with_test_links_default_local_only() -> None: +# set_default_include_external(False) +# +# results: list[int] = [] +# pie_requirements_with_test_links(_needs(), results, arg1="tool_req") +# +# summary = compute_traceability_summary( +# all_needs=_needs(), +# requirement_types={"tool_req"}, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=False, +# ) +# +# assert results == [1, 0] +# assert results == [ +# summary["requirements"]["total"] - summary["requirements"]["with_test_link"], +# summary["requirements"]["with_test_link"], +# ] +# +# +# def test_requirements_with_test_links_can_override_include_external() -> None: +# set_default_include_external(False) +# +# results: list[int] = [] +# pie_requirements_with_test_links(_needs(), results, arg1="tool_req", arg2="true") +# +# assert results == [1, 1] +# +# +# def test_requirements_with_test_links_parses_multiple_types() -> None: +# set_default_include_external(False) +# +# results: list[int] = [] +# pie_requirements_with_test_links(_needs(), results, arg1="tool_req,sys_req") +# +# assert results == [1, 1] +# +# +# def test_requirements_fully_linked_uses_shared_summary() -> None: +# set_default_include_external(False) +# +# results: list[int] = [] +# pie_requirements_fully_linked(_needs(), results, arg1="tool_req") +# +# summary = compute_traceability_summary( +# all_needs=_needs(), +# requirement_types={"tool_req"}, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=False, +# ) +# +# assert results == [1, 0] +# assert results == [ +# summary["requirements"]["total"] - summary["requirements"]["fully_linked"], +# summary["requirements"]["fully_linked"], +# ] +# +# +# def test_requirements_fully_linked_can_include_external() -> None: +# set_default_include_external(True) +# +# results: list[int] = [] +# pie_requirements_fully_linked(_needs(), results, arg1="tool_req") +# +# assert results == [1, 1] +# +# +# def test_process_requirements_linked_uses_stream_a_process_requirement_totals( +# monkeypatch: pytest.MonkeyPatch, +# ) -> None: +# captured: dict[str, object] = {} +# +# def _fake_summary( +# all_needs: Sequence[dict[str, Any]], +# requirement_types: set[str], +# include_not_implemented: bool, +# filtered_test_types: set[str], +# include_external: bool, +# ) -> dict[str, dict[str, int]]: +# captured["all_needs"] = all_needs +# captured["requirement_types"] = requirement_types +# captured["include_not_implemented"] = include_not_implemented +# captured["filtered_test_types"] = filtered_test_types +# captured["include_external"] = include_external +# return { +# "requirements": {"total": 99, "linked": 0}, +# "process_requirements": {"total": 4, "linked": 3}, +# } +# +# monkeypatch.setattr( +# traceability_dashboard, "compute_traceability_summary", _fake_summary +# ) +# +# results: list[int] = [] +# pie_process_requirements_linked( +# _needs(), results, arg1="tool_req,sys_req", arg2="true" +# ) +# +# assert results == [1, 3] +# assert captured["requirement_types"] == {"tool_req", "sys_req"} +# assert captured["include_not_implemented"] is True +# assert captured["filtered_test_types"] == set() +# assert captured["include_external"] is True diff --git a/src/extensions/score_metrics/tests/test_traceability_metrics.py b/src/extensions/score_metrics/tests/test_traceability_metrics.py new file mode 100644 index 000000000..87478eeb7 --- /dev/null +++ b/src/extensions/score_metrics/tests/test_traceability_metrics.py @@ -0,0 +1,206 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Unit tests for traceability_metrics include_external handling.""" + +from src.extensions.score_metrics.traceability_metrics import ( + compute_traceability_summary, + #filter_requirements, +) + + +def _needs() -> list[dict[str, object]]: + return [ + { + "id": "LOCAL_REQ", + "type": "tool_req", + "implemented": "YES", + "source_code_link": "src/local.py:1", + "testlink": "tests/test_local.py::test_ok", + "is_external": False, + }, + { + "id": "EXT_REQ", + "type": "tool_req", + "implemented": "YES", + "source_code_link": "src/external.py:9", + "testlink": "tests/test_external.py::test_ok", + "is_external": True, + }, + { + "id": "TC_1", + "type": "testcase", + "partially_verifies": "LOCAL_REQ", + "fully_verifies": "", + "is_external": False, + }, + ] + + +# def test_filter_requirements_defaults_to_local_only() -> None: +# filtered = filter_requirements( +# _needs(), +# requirement_types={"tool_req"}, +# include_not_implemented=True, +# ) +# +# assert [need["id"] for need in filtered] == ["LOCAL_REQ"] +# +# +# def test_filter_requirements_can_include_external_needs() -> None: +# filtered = filter_requirements( +# _needs(), +# requirement_types={"tool_req"}, +# include_not_implemented=True, +# include_external=True, +# ) +# +# assert sorted(need["id"] for need in filtered) == ["EXT_REQ", "LOCAL_REQ"] +# +# +# def test_compute_traceability_summary_propagates_include_external() -> None: +# summary_local = compute_traceability_summary( +# all_needs=_needs(), +# requirement_types={"tool_req"}, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=False, +# ) +# summary_all = compute_traceability_summary( +# all_needs=_needs(), +# requirement_types={"tool_req"}, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=True, +# ) +# +# assert summary_local["include_external"] is False +# assert summary_local["requirements"]["total"] == 1 +# assert summary_all["include_external"] is True +# assert summary_all["requirements"]["total"] == 2 +# +# +# def test_compute_traceability_summary_process_requirements_summary() -> None: +# summary = compute_traceability_summary( +# all_needs=[ +# { +# "id": "TOOL_REQ_1", +# "type": "tool_req", +# "implemented": "YES", +# "source_code_link": "src/req.py:10", +# "testlink": "tests/test_req.py::test_ok", +# "satisfies": "PR_LOCAL_1,OTHER_REQ", +# "is_external": False, +# }, +# { +# "id": "TOOL_REQ_2", +# "type": "tool_req", +# "implemented": "YES", +# "source_code_link": "src/req.py:20", +# "testlink": "tests/test_req.py::test_ok_2", +# "satisfies": ["PR_LOCAL_1", "PR_LOCAL_2"], +# "is_external": False, +# }, +# { +# "id": "PR_LOCAL_1", +# "type": "process_req", +# "is_external": False, +# }, +# { +# "id": "PR_LOCAL_2", +# "type": "gd_req", +# "is_external": False, +# }, +# { +# "id": "PR_LOCAL_3", +# "type": "gd_req", +# "is_external": False, +# }, +# ], +# requirement_types={"tool_req"}, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=False, +# ) +# +# process_requirements = summary["process_requirements"] +# +# assert process_requirements["total"] == 3 +# assert process_requirements["linked_by_tool_requirements"] == 2 +# assert process_requirements["linked_by_tool_requirements_pct"] == (2 / 3) * 100 +# assert process_requirements["unlinked_ids"] == ["PR_LOCAL_3"] +# +# +# def test_compute_traceability_summary_process_requirements_respects_include_external() -> ( +# None +# ): +# all_needs = [ +# { +# "id": "TOOL_REQ_LOCAL", +# "type": "tool_req", +# "implemented": "YES", +# "source_code_link": "src/local.py:1", +# "testlink": "tests/test_local.py::test_ok", +# "satisfies": "PR_LOCAL", +# "is_external": False, +# }, +# { +# "id": "TOOL_REQ_EXTERNAL", +# "type": "tool_req", +# "implemented": "YES", +# "source_code_link": "src/external.py:1", +# "testlink": "tests/test_external.py::test_ok", +# "satisfies": "PR_EXTERNAL", +# "is_external": True, +# }, +# { +# "id": "PR_LOCAL", +# "type": "gd_req", +# "is_external": False, +# }, +# { +# "id": "PR_EXTERNAL", +# "type": "gd_req", +# "is_external": True, +# }, +# ] +# +# summary_local = compute_traceability_summary( +# all_needs=all_needs, +# requirement_types={"tool_req"}, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=False, +# ) +# summary_all = compute_traceability_summary( +# all_needs=all_needs, +# requirement_types={"tool_req"}, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=True, +# ) +# +# assert summary_local["process_requirements"] == { +# "total": 1, +# "linked": 1, +# "linked_by_tool_requirements": 1, +# "linked_by_tool_requirements_pct": 100.0, +# "unlinked_ids": [], +# } +# assert summary_all["process_requirements"] == { +# "total": 2, +# "linked": 2, +# "linked_by_tool_requirements": 2, +# "linked_by_tool_requirements_pct": 100.0, +# "unlinked_ids": [], +# } diff --git a/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py b/src/extensions/score_metrics/tests/test_traceability_metrics_json_generation.py similarity index 100% rename from src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py rename to src/extensions/score_metrics/tests/test_traceability_metrics_json_generation.py diff --git a/src/extensions/score_metrics/traceability_dashboard.py b/src/extensions/score_metrics/traceability_dashboard.py new file mode 100644 index 000000000..5d8b41182 --- /dev/null +++ b/src/extensions/score_metrics/traceability_dashboard.py @@ -0,0 +1,183 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +# ╓ ╖ +# ║ Some portions generated by Github Copilot ║ +# ╙ ╜ + +"""Needpie filter functions backed by shared traceability metric calculations.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any + +from sphinx_needs.need_item import NeedItem + +from score_metrics.traceability_metrics import ( + compute_traceability_summary, +) + +_DEFAULT_INCLUDE_EXTERNAL = False + + +def set_default_include_external(include_external: bool) -> None: + """Configure default behaviour for including external requirements.""" + global _DEFAULT_INCLUDE_EXTERNAL + _DEFAULT_INCLUDE_EXTERNAL = bool(include_external) + + +# def _include_external(kwargs: dict[str, str | int | float]) -> bool: +# """Read include_external override from filter args, else use configured default.""" +# raw = kwargs.get("arg2") +# if raw is None: +# return _DEFAULT_INCLUDE_EXTERNAL +# text = str(raw).strip().lower() +# return text in {"1", "true", "yes", "on"} +# +# +# def _requirement_types(kwargs: dict[str, str | int | float]) -> set[str]: +# raw = str(kwargs.get("arg1", "tool_req")).strip() +# values = {value.strip() for value in raw.split(",") if value.strip()} +# return values or {"tool_req"} +# +# +# def pie_requirements_status( +# needs: Sequence[NeedItem | dict[str, Any]], +# results: list[int], +# **kwargs: str | int | float, +# ) -> None: +# """Dashboard status split: not implemented, implemented/incomplete, fully linked.""" +# req_types = _requirement_types(kwargs) +# include_external = _include_external(kwargs) +# +# all_requirements = filter_requirements( +# needs, +# requirement_types=req_types, +# include_not_implemented=True, +# include_external=include_external, +# ) +# implemented_requirements = filter_requirements( +# needs, +# requirement_types=req_types, +# include_not_implemented=False, +# include_external=include_external, +# ) +# summary = compute_traceability_summary( +# all_needs=needs, +# requirement_types=req_types, +# include_not_implemented=False, +# filtered_test_types=set(), +# include_external=include_external, +# ) +# +# not_implemented = len(all_requirements) - len(implemented_requirements) +# fully_linked = int(summary["requirements"]["fully_linked"]) +# implemented_incomplete = len(implemented_requirements) - fully_linked +# +# results.append(not_implemented) +# results.append(implemented_incomplete) +# results.append(fully_linked) +# +# +# def pie_requirements_with_code_links( +# needs: Sequence[NeedItem | dict[str, Any]], +# results: list[int], +# **kwargs: str | int | float, +# ) -> None: +# """Dashboard split: requirements with and without source code links.""" +# req_types = _requirement_types(kwargs) +# include_external = _include_external(kwargs) +# summary = compute_traceability_summary( +# all_needs=needs, +# requirement_types=req_types, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=include_external, +# ) +# +# total = int(summary["requirements"]["total"]) +# with_code = int(summary["requirements"]["with_code_link"]) +# +# results.append(total - with_code) +# results.append(with_code) +# +# +# def pie_requirements_with_test_links( +# needs: Sequence[NeedItem | dict[str, Any]], +# results: list[int], +# **kwargs: str | int | float, +# ) -> None: +# """Dashboard split: requirements with and without testcase links.""" +# req_types = _requirement_types(kwargs) +# include_external = _include_external(kwargs) +# summary = compute_traceability_summary( +# all_needs=needs, +# requirement_types=req_types, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=include_external, +# ) +# +# total = int(summary["requirements"]["total"]) +# with_test = int(summary["requirements"]["with_test_link"]) +# +# results.append(total - with_test) +# results.append(with_test) +# +# +# def pie_requirements_fully_linked( +# needs: Sequence[NeedItem | dict[str, Any]], +# results: list[int], +# **kwargs: str | int | float, +# ) -> None: +# """Dashboard split: requirements fully linked vs incomplete.""" +# req_types = _requirement_types(kwargs) +# include_external = _include_external(kwargs) +# summary = compute_traceability_summary( +# all_needs=needs, +# requirement_types=req_types, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=include_external, +# ) +# +# total = int(summary["requirements"]["total"]) +# fully_linked = int(summary["requirements"]["fully_linked"]) +# +# results.append(total - fully_linked) +# results.append(fully_linked) +# +# +# def pie_process_requirements_linked( +# needs: Sequence[NeedItem | dict[str, Any]], +# results: list[int], +# **kwargs: str | int | float, +# ) -> None: +# """Dashboard split: process requirements linked vs not linked.""" +# req_types = _requirement_types(kwargs) +# include_external = _include_external(kwargs) +# summary = compute_traceability_summary( +# all_needs=needs, +# requirement_types=req_types, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=include_external, +# ) +# +# process_requirements = summary["process_requirements"] +# total = int(process_requirements["total"]) +# linked = int(process_requirements["linked"]) +# +# results.append(total - linked) +# results.append(linked) diff --git a/src/extensions/score_metrics/traceability_metrics.py b/src/extensions/score_metrics/traceability_metrics.py new file mode 100644 index 000000000..3fe78019b --- /dev/null +++ b/src/extensions/score_metrics/traceability_metrics.py @@ -0,0 +1,184 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +# ╓ ╖ +# ║ Some portions generated by Github Copilot ║ +# ╙ ╜ + +"""Shared traceability metric calculations for CI checks and dashboards.""" + +from __future__ import annotations + +from typing import Any +from sphinx_needs.need_item import NeedItem +from sphinx_needs.data import NeedsView + + +def is_non_empty(value: Any) -> bool: + """Return True if value should be treated as present for traceability checks.""" + if isinstance(value, str): + return bool(value.strip()) + return bool(value) + + +def safe_percent(numerator: int, denominator: int) -> float: + """Return percentage in range [0, 100], treating empty denominator as 100%.""" + if denominator == 0: + return 100.0 + return (numerator / denominator) * 100.0 + + +def calculate_requirement_metrics( + all_requirement_needs: list[NeedItem], +) -> dict[str, Any]: + """Calculate requirement traceability statistics for links and completeness.""" + total = len(all_requirement_needs) + reqs_with_code_link = 0 + reqs_with_test_link = 0 + reqs_fully_linked = 0 + missing_code_links_need_ids: set[str] = set() + missing_test_links_need_ids: set[str] = set() + missing_both_links_need_ids: set[str] = set() + + for need in all_requirement_needs: + # Sourcecode link check + if is_non_empty(need.get("source_code_link")): + reqs_with_code_link += 1 + else: + missing_code_links_need_ids.add(need.get("id")) + + # Testlink check + if is_non_empty(need.get("testlink")): + reqs_with_test_link += 1 + else: + missing_test_links_need_ids.add(need.get("id")) + # Negative check (both missing) + if is_non_empty(need.get("testlink")) and is_non_empty( + need.get("source_code_link") + ): + reqs_fully_linked += 1 + + missing_both_links_need_ids = missing_code_links_need_ids.intersection( + missing_test_links_need_ids + ) + + return { + "total": total, + "with_code_link": reqs_with_code_link, + "with_test_link": reqs_with_test_link, + "fully_linked": reqs_fully_linked, + "with_code_link_pct": safe_percent(reqs_with_code_link, total), + "with_test_link_pct": safe_percent(reqs_with_test_link, total), + "fully_linked_pct": safe_percent(reqs_fully_linked, total), + "missing_code_link_ids": sorted(missing_code_links_need_ids), + "missing_test_link_ids": sorted(missing_test_links_need_ids), + "not_fully_linked_ids": sorted(missing_both_links_need_ids), + } + + +def calculate_test_metrics( + all_needs: NeedsView, + filtered_test_types: set[str], +) -> dict[str, Any]: + """Calculate testcase linkage and broken testcase-reference statistics.""" + test_needs = all_needs.filter_types(["testcase"]).values() + tests_total = len(test_needs) + tests_linked = 0 + broken_references: list[dict[str, str]] = [] + + for test_need in test_needs: + # This should never happen (that it has unknown_testcase id + # TODO: Remove this: + test_id = str(test_need.get("id", "")) + partially: list[str] = test_need.get("partially_verifies") + fully: list[str] = test_need.get("fully_verifies") + refs = set(partially + fully) + if refs: + tests_linked += 1 + for ref in refs: + if ref not in all_needs: + broken_references.append({"testcase": test_id, "missing_need": ref}) + + return { + "total": tests_total, + "filtered_test_types": sorted(filtered_test_types), + "linked_to_requirements": tests_linked, + "linked_to_requirements_pct": safe_percent(tests_linked, tests_total), + "broken_references": broken_references, + } + + +def calculate_process_requirement_metrics( + all_needs: NeedsView, + include_external: bool, +) -> dict[str, Any]: + """Calculate process-requirement coverage via tool_req ``satisfies`` links.""" + + process_requirements = all_needs.filter_types(["gd_req"]).filter_is_external( + include_external + ) + process_requirement_ids = set(process_requirements.keys()) + tool_requirements = all_needs.filter_types(["tool_req"]).filter_is_external( + include_external + ) + + linked_process_requirement_ids: set[str] = set() + for need in tool_requirements.values(): + satisfies_ids = need.get("satisfies") + for ref_id in satisfies_ids: + if ref_id in process_requirement_ids: + linked_process_requirement_ids.add(ref_id) + + total = len(process_requirement_ids) + linked_by_tool_requirements = len(linked_process_requirement_ids) + unlinked_ids = sorted(process_requirement_ids - linked_process_requirement_ids) + + return { + "total": total, + "linked": linked_by_tool_requirements, + "linked_by_tool_requirements": linked_by_tool_requirements, + "linked_by_tool_requirements_pct": safe_percent( + linked_by_tool_requirements, total + ), + "unlinked_ids": unlinked_ids, + } + + +def compute_traceability_summary( + all_needs: NeedsView, + filtered_test_types: set[str], + include_external: bool = False, +) -> dict[str, Any]: + """Return full CI/dashboard summary using one shared metric implementation.""" + requirements = all_needs + req_metrics = calculate_requirement_metrics( + list(requirements.filter_is_external(False).values()) + ) + + test_metrics = calculate_test_metrics( + all_needs, + filtered_test_types=filtered_test_types, + ) + process_requirement_metrics = calculate_process_requirement_metrics( + all_needs, + include_external=include_external, + ) + + return { + # Note: Unsure if we truly need this here. It seems to me to create too much spam + # "requirement_types": sorted(requirement_types), + "include_external": include_external, + "requirements": req_metrics, + "tests": test_metrics, + "process_requirements": process_requirement_metrics, + } diff --git a/src/extensions/score_source_code_linker/xml_parser.py b/src/extensions/score_source_code_linker/xml_parser.py index 302d87469..48c9277ca 100644 --- a/src/extensions/score_source_code_linker/xml_parser.py +++ b/src/extensions/score_source_code_linker/xml_parser.py @@ -219,9 +219,10 @@ def read_test_xml_file(file: Path) -> tuple[list[DataOfTestCase], list[str], lis properties_element = testcase.find("properties") # HINT: This list is hard coded here, might not be ideal to have that in the # long run. - if properties_element is None: - non_prop_tests.append(testname) - continue + # Even if we have no properties we still want to create test_case needs + # if properties_element is None: + # non_prop_tests.append(testname) + # continue # ╓ ╖ # ║ Disabled Temporarily ║ @@ -236,12 +237,13 @@ def read_test_xml_file(file: Path) -> tuple[list[DataOfTestCase], list[str], lis # I think it should be possible to save the 'from_dict' operation # If the is_valid method would return 'False' anyway. # I just can't think of it right now, leaving this for future me - case_properties = parse_properties(case_properties, properties_element) - case_properties.update(md) + if properties_element: + case_properties = parse_properties(case_properties, properties_element) + case_properties.update(md) test_case = DataOfTestCase.from_dict(case_properties) + # We will check if the testcase is valid, but still will generate the TestCaseNeed if not test_case.is_valid(): missing_prop_tests.append(testname) - continue test_case_needs.append(test_case) return test_case_needs, non_prop_tests, missing_prop_tests @@ -332,23 +334,21 @@ def short_hash(input_str: str, length: int = 5) -> str: def construct_and_add_need(app: Sphinx, tn: DataOfTestCase): - # Asserting worldview to a peace Language Server - # And ensure non crashing due to non string concatenation - # Everything but 'result_text', - # and either 'Fully' or 'PartiallyVerifies' should not be None here - assert tn.file is not None + # We will now allow file to be empty in case of non fully fleshed out testcases + file = tn.file if tn.file is not None else "" assert tn.name is not None + name = tn.name external_url = "" if tn.repo_name is None or tn.hash is None or tn.url is None: logger.info( "Creating testcase need with fallback URL due to incomplete repo metadata: " - f"name={tn.name}, file={tn.file}, repo_name={tn.repo_name}, " + f"name={name}, file={file}, repo_name={tn.repo_name}, " f"hash={tn.hash}, url={tn.url}", type="score_source_code_linker", ) line = tn.line if tn.line is not None else 1 external_url = ( - f"https://github.com/placeholder/placeholder/blob/unknown/{tn.file}#L{line}" + f"https://github.com/placeholder/placeholder/blob/unknown/{file}#L{line}" ) else: # Have to build metadata here for the gh link func @@ -359,10 +359,10 @@ def construct_and_add_need(app: Sphinx, tn: DataOfTestCase): _ = add_external_need( app=app, need_type="testcase", - title=tn.name, + title=name, tags="TEST", - id=f"testcase__{tn.name}_{short_hash(tn.file + tn.name)}", - name=tn.name, + id=f"testcase__{name}_{short_hash(file + name)}", + name=name, external_url=external_url, fully_verifies=tn.FullyVerifies if tn.FullyVerifies is not None else "", partially_verifies=tn.PartiallyVerifies @@ -370,7 +370,7 @@ def construct_and_add_need(app: Sphinx, tn: DataOfTestCase): else "", test_type=tn.TestType, derivation_technique=tn.DerivationTechnique, - file=tn.file, + file=file, line=tn.line, result=tn.result, # We just want the 'failed' or whatever result_text=tn.result_text if tn.result_text else "", diff --git a/src/extensions/score_sphinx_bundle/BUILD b/src/extensions/score_sphinx_bundle/BUILD index 832e35a61..c34d2907e 100644 --- a/src/extensions/score_sphinx_bundle/BUILD +++ b/src/extensions/score_sphinx_bundle/BUILD @@ -29,6 +29,7 @@ py_library( "@score_docs_as_code//src/extensions/score_layout", "@score_docs_as_code//src/extensions/score_metamodel", "@score_docs_as_code//src/extensions/score_source_code_linker", + "@score_docs_as_code//src/extensions/score_metrics", "@score_docs_as_code//src/extensions/score_sync_toml", "@score_docs_as_code//src/helper_lib", ], diff --git a/src/extensions/score_sphinx_bundle/__init__.py b/src/extensions/score_sphinx_bundle/__init__.py index 6ae04008b..57b2d6892 100644 --- a/src/extensions/score_sphinx_bundle/__init__.py +++ b/src/extensions/score_sphinx_bundle/__init__.py @@ -31,6 +31,7 @@ "sphinxcontrib.mermaid", "needs_config_writer", "score_sync_toml", + "score_metrics" ] From ff4a6ee1c6a51724b08234ae2ba251028ce85ead Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Wed, 3 Jun 2026 09:57:31 +0200 Subject: [PATCH 6/8] WIP: Saving before del process reqs Saving metrics before deleting process reqs --- .../requirements/implementation_state.rst | 201 +++++++++--------- src/extensions/score_metrics/__init__.py | 157 +++++--------- .../score_metrics/traceability_dashboard.py | 44 +++- .../score_metrics/traceability_metrics.py | 121 ++++++++--- 4 files changed, 282 insertions(+), 241 deletions(-) diff --git a/docs/internals/requirements/implementation_state.rst b/docs/internals/requirements/implementation_state.rst index d2ecc9c24..6bc2ca9f7 100644 --- a/docs/internals/requirements/implementation_state.rst +++ b/docs/internals/requirements/implementation_state.rst @@ -23,105 +23,106 @@ requirements. It focuses on tooling capabilities offered to downstream repositories rather than on product-specific traceability inside those repositories. -Overview --------- +.. Overview +.. -------- +.. .. needpie:: Tool Requirements Status - :labels: not implemented, implemented but incomplete traceability, fully linked - :colors: red,yellow, green - :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_requirements_status(tool_req) - -Jump to evidence tables: - -- :ref:`Tool Requirement Implementation and Links table ` -- :ref:`Process Requirement to Tool Requirement mapping table ` - -How To Read These Levels ------------------------- - -The overview pie combines implementation state and traceability evidence: - -- ``not implemented``: - requirement has ``implemented == NO``. -- ``implemented but incomplete traceability``: - requirement has ``implemented == YES`` or ``implemented == PARTIAL``, - but is missing at least one traceability link (code link and/or test link). -- ``fully linked``: - requirement is implemented and has both ``source_code_link`` and ``testlink``. - -Implementation labels used on this page: - -- ``NO``: requirement is not implemented. -- ``PARTIAL``: requirement is partly implemented. -- ``YES``: requirement is implemented. - -Why multiple pies are shown: - -- ``Requirements with Codelinks`` shows requirement-to-implementation traceability. -- ``Requirements with linked tests`` shows requirement-to-verification traceability. -- ``Requirements fully linked`` is the strict roll-up (both links present). - -These are intentionally separate because they answer different diagnostics: -missing code links, missing test links, or both. - -In Detail ---------- - -.. grid:: 2 - :class-container: score-grid - - .. grid-item-card:: - - .. needpie:: Requirements marked as Implemented - :labels: not implemented, partial, implemented - :colors: red, orange, green - - type == 'tool_req' and implemented == 'NO' - type == 'tool_req' and implemented == 'PARTIAL' - type == 'tool_req' and implemented == 'YES' - - .. grid-item-card:: - - .. needpie:: Requirements with Codelinks - :labels: no codelink, with codelink - :colors: red, green - :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_requirements_with_code_links(tool_req) - - .. grid-item-card:: - - .. needpie:: Requirements with linked tests - :labels: no test link, with test link - :colors: red, green - :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_requirements_with_test_links(tool_req) - - .. grid-item-card:: - - .. needpie:: Requirements fully linked (code + tests) - :labels: not fully linked, fully linked - :colors: orange, green - :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_requirements_fully_linked(tool_req) - - .. grid-item-card:: - - .. needpie:: Process requirements linked by tool requirements - :labels: not linked, linked - :colors: red, green - :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_process_requirements_linked(tool_req,true) - - -Process-to-Tool Mapping ------------------------ - -.. _tooling_coverage_table_process_mapping: - -.. needtable:: Process requirement -> tool requirement mapping - :types: tool_req - :columns: satisfies as "Process Requirement";id as "Tool Requirement" - :style: table - -.. _tooling_coverage_table_impl_links: - -.. needtable:: Tool requirement implementation and links - :types: tool_req - :columns: id as "Tool Requirement";implemented;source_code_link;testlink - :style: table + :labels: total, linked with code + :colors: blue, yellow + :filter-func: src.extensions.score_metrics.traceability_dashboard.get_linked_metrics_for_type(_build/metrics,overall_metrics:total;overall_metrics:with_test_link) +.. +.. Jump to evidence tables: +.. +.. - :ref:`Tool Requirement Implementation and Links table ` +.. - :ref:`Process Requirement to Tool Requirement mapping table ` +.. +.. How To Read These Levels +.. ------------------------ +.. +.. The overview pie combines implementation state and traceability evidence: +.. +.. - ``not implemented``: +.. requirement has ``implemented == NO``. +.. - ``implemented but incomplete traceability``: +.. requirement has ``implemented == YES`` or ``implemented == PARTIAL``, +.. but is missing at least one traceability link (code link and/or test link). +.. - ``fully linked``: +.. requirement is implemented and has both ``source_code_link`` and ``testlink``. +.. +.. Implementation labels used on this page: +.. +.. - ``NO``: requirement is not implemented. +.. - ``PARTIAL``: requirement is partly implemented. +.. - ``YES``: requirement is implemented. +.. +.. Why multiple pies are shown: +.. +.. - ``Requirements with Codelinks`` shows requirement-to-implementation traceability. +.. - ``Requirements with linked tests`` shows requirement-to-verification traceability. +.. - ``Requirements fully linked`` is the strict roll-up (both links present). +.. +.. These are intentionally separate because they answer different diagnostics: +.. missing code links, missing test links, or both. +.. +.. In Detail +.. --------- +.. +.. .. grid:: 2 +.. :class-container: score-grid +.. +.. .. grid-item-card:: +.. +.. .. needpie:: Requirements marked as Implemented +.. :labels: not implemented, partial, implemented +.. :colors: red, orange, green +.. +.. type == 'tool_req' and implemented == 'NO' +.. type == 'tool_req' and implemented == 'PARTIAL' +.. type == 'tool_req' and implemented == 'YES' +.. +.. .. grid-item-card:: +.. +.. .. needpie:: Requirements with Codelinks +.. :labels: no codelink, with codelink +.. :colors: red, green +.. :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_requirements_with_code_links(tool_req) +.. +.. .. grid-item-card:: +.. +.. .. needpie:: Requirements with linked tests +.. :labels: no test link, with test link +.. :colors: red, green +.. :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_requirements_with_test_links(tool_req) +.. +.. .. grid-item-card:: +.. +.. .. needpie:: Requirements fully linked (code + tests) +.. :labels: not fully linked, fully linked +.. :colors: orange, green +.. :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_requirements_fully_linked(tool_req) +.. +.. .. grid-item-card:: +.. +.. .. needpie:: Process requirements linked by tool requirements +.. :labels: not linked, linked +.. :colors: red, green +.. :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_process_requirements_linked(tool_req,true) +.. +.. +.. Process-to-Tool Mapping +.. ----------------------- +.. +.. .. _tooling_coverage_table_process_mapping: +.. +.. .. needtable:: Process requirement -> tool requirement mapping +.. :types: tool_req +.. :columns: satisfies as "Process Requirement";id as "Tool Requirement" +.. :style: table +.. +.. .. _tooling_coverage_table_impl_links: +.. +.. .. needtable:: Tool requirement implementation and links +.. :types: tool_req +.. :columns: id as "Tool Requirement";implemented;source_code_link;testlink +.. :style: table diff --git a/src/extensions/score_metrics/__init__.py b/src/extensions/score_metrics/__init__.py index b572d044f..d42a01d1e 100644 --- a/src/extensions/score_metrics/__init__.py +++ b/src/extensions/score_metrics/__init__.py @@ -5,59 +5,25 @@ from sphinx.application import Sphinx from sphinx_needs import logging -from sphinx_needs.data import NeedsView, SphinxNeedsData -from sphinx_needs.need_item import NeedItem -from score_metrics.traceability_metrics import ( - compute_traceability_summary, -) +from score_metrics.traceability_metrics import calculate_full_need_metrics +from sphinx.environment import BuildEnvironment logger = logging.get_logger(__name__) -def _configure_traceability_dashboard(app: Sphinx, config: object) -> None: - """Propagate repo-level traceability settings to dashboard filters.""" - from src.extensions.score_metrics.traceability_dashboard import ( - set_default_include_external, - ) - - include_external = bool( - getattr(config, "score_metamodel_include_external_needs", False) - ) - set_default_include_external(include_external) - +# def _configure_traceability_dashboard(app: Sphinx, config: object) -> None: +# """Propagate repo-level traceability settings to dashboard filters.""" +# from src.extensions.score_metrics.traceability_dashboard import ( +# set_default_include_external, +# ) +# +# include_external = bool( +# getattr(config, "score_metamodel_include_external_needs", False) +# ) +# set_default_include_external(include_external) -def get_need_types_by_tags(needs_types: list[NeedItem], tags: set[str]) -> list[str]: - found_need_types: list[str] = [] - for need_type in needs_types: - found_tag_set = set(need_type["tags"]) - if tags.intersection(found_tag_set): - found_need_types.append(need_type["type"]) - return found_need_types - - -def _discover_requirement_types( - app: Sphinx, all_needs: list[NeedItem] -) -> set[NeedItem]: - """Discover requirement directives that are both tagged and present.""" - tagged_requirements_types = get_need_types_by_tags( - all_needs, {"requirement", "requirement_excl_process"} - ) - all_tagged_requirements = ( - SphinxNeedsData(app.env) - .get_needs_view() - .filter_types(tagged_requirements_types) - .values() - ) - if not all_tagged_requirements: - logger.warning( - "No requirement types discovered in current build for tagged " - "needs_types requirement directives." - ) - return set(all_tagged_requirements) - - -def _write_metrics_json(app: Sphinx, exception: Exception | None) -> None: +def _write_metrics_json(app: Sphinx, env: BuildEnvironment) -> None: """Write a schema-v1 metrics.json alongside needs.json in the build output. This is the single source of truth for traceability metrics. It runs @@ -65,59 +31,52 @@ def _write_metrics_json(app: Sphinx, exception: Exception | None) -> None: and produces the same metrics the dashboard pie charts display. The traceability_gate reads this file to enforce CI thresholds. """ - print("WE ARE IN WRITE METRICS JSON") - if exception: - logger.error("===================================") - logger.error("===================================") - logger.error("===================================") - logger.error(f"EXCEPTION RAISED: {exception}") - return - include_external: bool = bool( getattr(app.config, "score_metamodel_include_external_needs", False) ) - all_needs: NeedsView = SphinxNeedsData(app.env).get_needs_view() - - raw = getattr(app.config, "score_metamodel_requirement_types", "").strip() - requirement_types = [t.strip() for t in raw.split(",") if t.strip()] - if requirement_types: - requirement_types = all_needs.filter_types(requirement_types) - - metrics_by_type: dict[str, Any] = {} - if not requirement_types: - logger.info( - "No requirement types configured or discovered; writing empty metrics.json." - ) - else: - for req_type in sorted(requirement_types): - type_summary = compute_traceability_summary( - all_needs=all_needs, - filtered_test_types=set(), - include_external=include_external, - ) - metrics_by_type[req_type] = { - "include_not_implemented": type_summary["include_not_implemented"], - "include_external": type_summary["include_external"], - "requirements": type_summary["requirements"], - "tests": type_summary["tests"], - } - - output: dict[str, Any] = { - "schema_version": "1", - "generated_by": "sphinx_build", - "metrics_by_type": metrics_by_type, - } - print("===================================") - print("===================================") - print("===================================") - print(f"writing metrics.json to {Path(app.outdir) / 'metrics.json'}") - print("===================================") - print("===================================") - print("===================================") - out_path = Path(app.outdir) / "metrics.json" - out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_text(json.dumps(output, indent=2), encoding="utf-8") - print(f"Traceability metrics written to: {out_path}") + calculate_full_need_metrics(app=app, include_external=include_external) + # all_needs: NeedsView = SphinxNeedsData(app.env).get_needs_view() + # + # raw_metamodel_path = app.config.score_metamodel_yaml + # override_path = Path(raw_metamodel_path) if raw_metamodel_path else None + # metamodel = load_metamodel_data(override_path) + # + # raw = getattr(app.config, "score_metamodel_requirement_types", "").strip() + # filter_reqs = [t.strip() for t in raw.split(",") if t.strip()] + # if not filter_reqs: + # filter_reqs = get_need_types_by_tags( + # metamodel.needs_types, {"reqiurement", "requirement_excl_process"} + # ) + # metrics_by_type: dict[str, Any] = {} + # test_stats + # + # test_needs = list(all_needs.filter_types(["testcase"]).values()) + # test_metrics = calculate_test_metrics(test_needs, current_reqtype_needs) + # for req_type in sorted(filter_reqs): + # needs_of_req_type = all_needs.filter_types([req_type]).filter_is_external(False) + # if not list(needs_of_req_type.values()): + # continue + # type_summary = compute_traceability_summary( + # all_needs=all_needs, + # current_reqtype=req_type, + # current_reqtype_needs=needs_of_req_type, + # include_external=include_external, + # ) + # metrics_by_type[req_type] = { + # "include_external": type_summary["include_external"], + # "requirements": type_summary["requirements"], + # "tests": type_summary["tests"], + # } + # + # output: dict[str, Any] = { + # "schema_version": "1", + # "generated_by": "sphinx_build", + # "metrics_by_type": metrics_by_type, + # } + # out_path = Path(app.outdir) / "metrics.json" + # out_path.parent.mkdir(parents=True, exist_ok=True) + # out_path.write_text(json.dumps(output, indent=2), encoding="utf-8") + # print(f"Traceability metrics written to: {out_path}") def setup(app: Sphinx) -> dict[str, str | bool]: @@ -142,8 +101,8 @@ def setup(app: Sphinx) -> dict[str, str | bool]: ), ) - _ = app.connect("config-inited", _configure_traceability_dashboard, priority=498) - _ = app.connect("build-finished", _write_metrics_json, priority=499) + # _ = app.connect("config-inited", _configure_traceability_dashboard, priority=498) + _ = app.connect("env-updated", _write_metrics_json, priority=550) return { "version": "0.1", diff --git a/src/extensions/score_metrics/traceability_dashboard.py b/src/extensions/score_metrics/traceability_dashboard.py index 5d8b41182..4e135380d 100644 --- a/src/extensions/score_metrics/traceability_dashboard.py +++ b/src/extensions/score_metrics/traceability_dashboard.py @@ -19,22 +19,50 @@ from __future__ import annotations +import json +from pathlib import Path + from collections.abc import Sequence from typing import Any from sphinx_needs.need_item import NeedItem - -from score_metrics.traceability_metrics import ( - compute_traceability_summary, -) +from score_metrics.traceability_metrics import CALCULATED_METRICS _DEFAULT_INCLUDE_EXTERNAL = False -def set_default_include_external(include_external: bool) -> None: - """Configure default behaviour for including external requirements.""" - global _DEFAULT_INCLUDE_EXTERNAL - _DEFAULT_INCLUDE_EXTERNAL = bool(include_external) +def get_linked_metrics_for_type( + needs: list[Any], results: list[int], **kwargs: Any +) -> None: + """Append dynamically selected metrics from a nested JSON file to ``results``. + + The ``metric_paths`` argument must be a semicolon-separated list of dot paths, + for example: ``overall_metrics.total;overall_metrics.with_test_link``. + """ + # metrics_path = str(kwargs["arg1"]) + ".json" + # metrics_json = json.loads(Path(metrics_path).read_text(encoding="utf-8")) + metrics_json = CALCULATED_METRICS + + # arg1 = _build/metrics, + # arg2 = overall_metrics, :total,overall_metrics:with_test_link + + metric_values = "overall_metrics.total" + str(kwargs["arg2"]) + for raw_path in metric_values.split(";"): + path = raw_path.strip() + if not path: + continue + + current: Any = metrics_json + for key in path.split(":"): + current = current[key] + results.append(int(current)) + results[0] -= sum(results[1:]) + + +# def set_default_include_external(include_external: bool) -> None: +# """Configure default behaviour for including external requirements.""" +# global _DEFAULT_INCLUDE_EXTERNAL +# _DEFAULT_INCLUDE_EXTERNAL = bool(include_external) # def _include_external(kwargs: dict[str, str | int | float]) -> bool: diff --git a/src/extensions/score_metrics/traceability_metrics.py b/src/extensions/score_metrics/traceability_metrics.py index 3fe78019b..cc2310255 100644 --- a/src/extensions/score_metrics/traceability_metrics.py +++ b/src/extensions/score_metrics/traceability_metrics.py @@ -19,9 +19,25 @@ from __future__ import annotations +import json from typing import Any from sphinx_needs.need_item import NeedItem from sphinx_needs.data import NeedsView +from sphinx.application import Sphinx +from score_metamodel import ScoreNeedType +from pathlib import Path +from sphinx_needs.data import SphinxNeedsData +from score_metamodel.yaml_parser import load_metamodel_data + +CALCULATED_METRICS = {} + +def get_need_types_by_tags(needs: list[ScoreNeedType], tags: set[str]) -> list[str]: + found_need_types: list[str] = [] + for need_type in needs: + found_tag_set = set(need_type["tags"]) + if tags.intersection(found_tag_set): + found_need_types.append(need_type["directive"]) + return found_need_types def is_non_empty(value: Any) -> bool: @@ -39,10 +55,11 @@ def safe_percent(numerator: int, denominator: int) -> float: def calculate_requirement_metrics( - all_requirement_needs: list[NeedItem], + current_requirement_needs: list[NeedItem], ) -> dict[str, Any]: """Calculate requirement traceability statistics for links and completeness.""" - total = len(all_requirement_needs) + total = len(current_requirement_needs) + print("===============") reqs_with_code_link = 0 reqs_with_test_link = 0 reqs_fully_linked = 0 @@ -50,7 +67,7 @@ def calculate_requirement_metrics( missing_test_links_need_ids: set[str] = set() missing_both_links_need_ids: set[str] = set() - for need in all_requirement_needs: + for need in current_requirement_needs: # Sourcecode link check if is_non_empty(need.get("source_code_link")): reqs_with_code_link += 1 @@ -87,11 +104,9 @@ def calculate_requirement_metrics( def calculate_test_metrics( - all_needs: NeedsView, - filtered_test_types: set[str], + test_needs: list[NeedItem], all_needs: NeedsView ) -> dict[str, Any]: """Calculate testcase linkage and broken testcase-reference statistics.""" - test_needs = all_needs.filter_types(["testcase"]).values() tests_total = len(test_needs) tests_linked = 0 broken_references: list[dict[str, str]] = [] @@ -111,7 +126,6 @@ def calculate_test_metrics( return { "total": tests_total, - "filtered_test_types": sorted(filtered_test_types), "linked_to_requirements": tests_linked, "linked_to_requirements_pct": safe_percent(tests_linked, tests_total), "broken_references": broken_references, @@ -120,17 +134,12 @@ def calculate_test_metrics( def calculate_process_requirement_metrics( all_needs: NeedsView, - include_external: bool, ) -> dict[str, Any]: """Calculate process-requirement coverage via tool_req ``satisfies`` links.""" - process_requirements = all_needs.filter_types(["gd_req"]).filter_is_external( - include_external - ) + process_requirements = all_needs.filter_types(["gd_req"]) process_requirement_ids = set(process_requirements.keys()) - tool_requirements = all_needs.filter_types(["tool_req"]).filter_is_external( - include_external - ) + tool_requirements = all_needs.filter_types(["tool_req"]).filter_is_external(False) linked_process_requirement_ids: set[str] = set() for need in tool_requirements.values(): @@ -154,31 +163,75 @@ def calculate_process_requirement_metrics( } -def compute_traceability_summary( - all_needs: NeedsView, - filtered_test_types: set[str], - include_external: bool = False, -) -> dict[str, Any]: - """Return full CI/dashboard summary using one shared metric implementation.""" - requirements = all_needs - req_metrics = calculate_requirement_metrics( - list(requirements.filter_is_external(False).values()) +def calculate_full_need_metrics(app: Sphinx, include_external: bool): + # ───────────────[ Getting configuration values ]─────────────── + global CALCULATED_METRICS + # if CALCULATED_METRICS: + # logger.info("Metrics calculated already, skipping re-execution") + # return + all_needs: NeedsView = SphinxNeedsData(app.env).get_needs_view() + + raw_metamodel_path = app.config.score_metamodel_yaml + override_path = Path(raw_metamodel_path) if raw_metamodel_path else None + metamodel = load_metamodel_data(override_path) + + raw = getattr(app.config, "score_metamodel_requirement_types", "").strip() + filter_reqs = [t.strip() for t in raw.split(",") if t.strip()] + if not filter_reqs: + filter_reqs = get_need_types_by_tags( + metamodel.needs_types, {"reqiurement", "requirement_excl_process"} + ) + # ──────────────────[ Calculate Test Metrics ]────────────────── + + test_needs = list(all_needs.filter_types(["testcase"]).values()) + test_metrics = calculate_test_metrics(test_needs, all_needs) + process_requirement_metrics = calculate_process_requirement_metrics( + all_needs, ) - test_metrics = calculate_test_metrics( - all_needs, - filtered_test_types=filtered_test_types, + metrics_by_type: dict[str, Any] = {} + overall_metrics: dict[str, Any] = { + "total": 0, + "with_code_link": 0, + "with_test_link": 0, + "fully_linked": 0, + } + for req_type in sorted(filter_reqs): + needs_of_req_type = all_needs.filter_types([req_type]).filter_is_external( + include_external + ) + # We do not care if there is no requirements of this type. + if not list(needs_of_req_type.values()): + continue + req_metrics = calculate_requirement_metrics(list(needs_of_req_type.values())) + overall_metrics["total"] += req_metrics["total"] + overall_metrics["with_code_link"] += req_metrics["with_code_link"] + overall_metrics["with_test_link"] += req_metrics["with_test_link"] + overall_metrics["fully_linked"] += req_metrics["fully_linked"] + metrics_by_type[req_type] = req_metrics + overall_metrics["with_code_link_pct"] = safe_percent( + overall_metrics["with_code_link"], overall_metrics["total"] ) - process_requirement_metrics = calculate_process_requirement_metrics( - all_needs, - include_external=include_external, + overall_metrics["with_test_link_pct"] = safe_percent( + overall_metrics["with_test_link"], overall_metrics["total"] + ) + overall_metrics["fully_linked_pct"] = safe_percent( + overall_metrics["fully_linked"], overall_metrics["total"] ) - return { - # Note: Unsure if we truly need this here. It seems to me to create too much spam - # "requirement_types": sorted(requirement_types), - "include_external": include_external, - "requirements": req_metrics, + output: dict[str, Any] = { + "schema_version": "1", + "generated_by": "sphinx_build", + "overall_metrics": overall_metrics, + "metrics_by_type": metrics_by_type, "tests": test_metrics, "process_requirements": process_requirement_metrics, } + app.config.calculated_metrics = output + CALCULATED_METRICS = output + + out_path = Path(app.outdir) / "metrics.json" + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(output, indent=2), encoding="utf-8") + print(f"Traceability metrics written to: {out_path}") + From 81f94df9c8ff069ed4caf17f62e3d2c1a40b1fdf Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Wed, 3 Jun 2026 14:06:09 +0200 Subject: [PATCH 7/8] WIP: Enable saving of metrics to global var --- docs/conf.py | 2 + docs/how-to/dashboards_and_quality_gates.rst | 290 ++++++++++++++++++ .../requirements/implementation_state.rst | 18 +- src/extensions/score_metrics/__init__.py | 66 ++-- .../score_metrics/traceability_dashboard.py | 71 ++++- .../score_metrics/traceability_metrics.py | 73 ++--- 6 files changed, 403 insertions(+), 117 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 0255915c2..56031d451 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,6 +10,7 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* +import matplotlib project = "Score Docs-as-Code" project_url = "https://eclipse-score.github.io/docs-as-code/" @@ -18,3 +19,4 @@ extensions = [ "score_sphinx_bundle", ] +matplotlib.rcParamsDefault["savefig.bbox"] = "tight" diff --git a/docs/how-to/dashboards_and_quality_gates.rst b/docs/how-to/dashboards_and_quality_gates.rst index 418fc6228..bc485734d 100644 --- a/docs/how-to/dashboards_and_quality_gates.rst +++ b/docs/how-to/dashboards_and_quality_gates.rst @@ -151,6 +151,296 @@ For a new consumer repository: 4. Introduce modest thresholds in CI. 5. Raise thresholds over time as the repository matures. +.. + ╓ ╖ + ║ .. The Following part has been generated by Copilot ║ + ╙ ╜ + +Metrics Needpie functions +========================= + +Overview +-------- + +These helpers read values from a nested dictionary named ``CALCULATED_METRICS``. +A metric is selected by a **colon-separated path**, for example: + +- ``overall_metrics:total`` +- ``overall_metrics:with_test_link`` +- ``types:bug:total`` + +All resolved values are converted to ``int`` and appended to a mutable +``results`` list passed by the caller. + +The ``needs`` parameter is accepted for integration compatibility, but it is not +used by the current implementations. + +Path format +----------- + +Paths are split using ``:`` and then resolved step by step. + +Example: + +.. code-block:: text + + path = "overall_metrics:total" + +is resolved like: + +.. code-block:: python + + current = CALCULATED_METRICS + current = current["overall_metrics"] + current = current["total"] + +Function reference +------------------ + +_get_key_values(results, argument_paths) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Purpose +^^^^^^^ + +Internal helper that appends metric values for a list of paths. + +Behavior +^^^^^^^^ + +1. Iterate over each item in ``argument_paths``. +2. Strip whitespace. +3. Skip empty paths. +4. Resolve the path inside ``CALCULATED_METRICS``. +5. Convert to integer and append to ``results``. + +Notes +^^^^^ + +- Modifies ``results`` in place. +- Does not return a new list. +- Raises ``KeyError`` if a path key does not exist. +- Raises ``ValueError`` if a resolved value cannot be converted to ``int``. + +get_metrics_with_overall_total_considered(...) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Purpose +^^^^^^^ + +Compute a remainder against global overall total. + +Behavior +^^^^^^^^ + +1. Append ``CALCULATED_METRICS["overall_metrics"]["total"]`` as first value. +2. Append each metric referenced by ``kwargs.values()``. +3. Replace the first value with: + + ``overall_total - sum(all other appended values)`` + +Typical use case +^^^^^^^^^^^^^^^^ + +Use this when your pie/chart needs an "Other" bucket based on the global total. + +Example +^^^^^^^ + +Assume: + +.. code-block:: python + + CALCULATED_METRICS = { + "overall_metrics": { + "total": 100, + "with_test_link": 30, + "with_review": 20, + } + } + +Call: + +.. code-block:: python + + results = [] + get_metrics_with_overall_total_considered( + needs=[], + results=results, + a="overall_metrics:with_test_link", + b="overall_metrics:with_review", + ) + +Result: + +.. code-block:: python + + # Step 1 append total: [100] + # Step 2 append selected: [100, 30, 20] + # Step 3 remainder: [50, 30, 20] + results == [50, 30, 20] + +get_metrics_with_custom_type_total_considered(...) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Purpose +^^^^^^^ + +Support custom total path if provided as the last kwarg value. + +Behavior +^^^^^^^^ + +If the **last** provided path ends with ``:total``: + +1. Append that last path as baseline total. +2. Append all preceding paths as components. +3. Replace the first value with: + + ``baseline_total - sum(components)`` + +Otherwise: + +- Append all provided paths directly (no subtraction). + +Important ordering rule +^^^^^^^^^^^^^^^^^^^^^^^ + +The special total path must be the **last** kwarg value, for example: + +.. code-block:: python + + get_metrics_with_custom_type_total_considered( + needs=[], + results=results, + part_a="types:feature:done", + part_b="types:feature:in_progress", + total="types:feature:total", # must be last + ) + +Example with custom total +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Assume: + +.. code-block:: python + + CALCULATED_METRICS = { + "types": { + "feature": { + "total": 40, + "done": 10, + "in_progress": 5, + } + } + } + +Call: + +.. code-block:: python + + results = [] + get_metrics_with_custom_type_total_considered( + needs=[], + results=results, + done="types:feature:done", + in_progress="types:feature:in_progress", + total="types:feature:total", + ) + +Result: + +.. code-block:: python + + # baseline total: 40 + # components: 10, 5 + # remainder: 40 - (10 + 5) = 25 + results == [25, 10, 5] + +Example without custom total +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Call: + +.. code-block:: python + + results = [] + get_metrics_with_custom_type_total_considered( + needs=[], + results=results, + done="types:feature:done", + in_progress="types:feature:in_progress", + ) + +Result: + +.. code-block:: python + + # no subtraction mode + results == [10, 5] + +get_just_metrics(...) +~~~~~~~~~~~~~~~~~~~~ + +Purpose +^^^^^^^ + +Append only the selected metric values. + +Behavior +^^^^^^^^ + +- Interprets each kwarg value as a path. +- Resolves and appends each value. +- No total baseline and no remainder calculation. + +Example +^^^^^^^ + +.. code-block:: python + + results = [] + get_just_metrics( + needs=[], + results=results, + a="overall_metrics:with_test_link", + b="overall_metrics:with_review", + ) + # results might become [30, 20] + +How to use these functions correctly +------------------------------------ + +1. Pass a mutable list in ``results`` (usually ``[]`` initially). +2. Provide metric paths through keyword argument values. +3. Use colon-separated keys (``a:b:c``), not dot notation. +4. For custom-total mode, ensure the total path is the last kwarg value. +5. Expect in-place mutation of ``results``. + +Common pitfalls +--------------- + +- Empty kwargs in custom-total function can lead to index errors + (because ``values[-1]`` is accessed). +- Invalid paths raise ``KeyError``. +- Non-integer values raise ``ValueError`` during ``int(...)`` conversion. +- ``print(results)`` currently causes side effects during execution; remove if + quiet behavior is preferred in production. + +Testing recommendations +----------------------- + +Add unit tests for: + +- Basic path resolution with one and multiple paths. +- Whitespace and empty-path handling. +- Overall total remainder logic. +- Custom total behavior when last path ends with ``:total``. +- Behavior when no custom total is provided. +- Error handling for missing keys and non-integer values. +- Empty ``kwargs`` behavior in custom-total function. + + Related Guides -------------- diff --git a/docs/internals/requirements/implementation_state.rst b/docs/internals/requirements/implementation_state.rst index 6bc2ca9f7..92adbc39f 100644 --- a/docs/internals/requirements/implementation_state.rst +++ b/docs/internals/requirements/implementation_state.rst @@ -27,11 +27,19 @@ repositories. .. -------- .. -.. needpie:: Tool Requirements Status - :labels: total, linked with code - :colors: blue, yellow - :filter-func: src.extensions.score_metrics.traceability_dashboard.get_linked_metrics_for_type(_build/metrics,overall_metrics:total;overall_metrics:with_test_link) -.. +.. needpie:: Overall Metrics with Total incooperated + :labels: without any link, overall with test link, overall with code, overall fully linked + :colors: red, yellow, blue, green + :filter-func: src.extensions.score_metrics.traceability_dashboard.get_metrics_with_overall_total_considered(overall_metrics:with_test_link,overall_metrics:with_code_link,overall_metrics:fully_linked) + + + +.. needpie:: Metrics without any total incooperated + :labels: tool req with test link, tool req with code, overall fully linked + :colors: yellow, blue, green + :filter-func: src.extensions.score_metrics.traceability_dashboard.get_just_metrics(metrics_by_type:tool_req:with_test_link,metrics_by_type:tool_req:with_code_link,overall_metrics:fully_linked) + + .. Jump to evidence tables: .. .. - :ref:`Tool Requirement Implementation and Links table ` diff --git a/src/extensions/score_metrics/__init__.py b/src/extensions/score_metrics/__init__.py index d42a01d1e..39b2674e2 100644 --- a/src/extensions/score_metrics/__init__.py +++ b/src/extensions/score_metrics/__init__.py @@ -8,6 +8,8 @@ from score_metrics.traceability_metrics import calculate_full_need_metrics from sphinx.environment import BuildEnvironment +from src.extensions.score_metrics.traceability_metrics import CALCULATED_METRICS + logger = logging.get_logger(__name__) @@ -21,11 +23,9 @@ # getattr(config, "score_metamodel_include_external_needs", False) # ) # set_default_include_external(include_external) - - -def _write_metrics_json(app: Sphinx, env: BuildEnvironment) -> None: - """Write a schema-v1 metrics.json alongside needs.json in the build output. - +# +def calculate_need_metrics(app: Sphinx, env:BuildEnvironment) -> None: + """ This is the single source of truth for traceability metrics. It runs inside the Sphinx build so it has access to all needs (local + external) and produces the same metrics the dashboard pie charts display. @@ -35,48 +35,17 @@ def _write_metrics_json(app: Sphinx, env: BuildEnvironment) -> None: getattr(app.config, "score_metamodel_include_external_needs", False) ) calculate_full_need_metrics(app=app, include_external=include_external) - # all_needs: NeedsView = SphinxNeedsData(app.env).get_needs_view() - # - # raw_metamodel_path = app.config.score_metamodel_yaml - # override_path = Path(raw_metamodel_path) if raw_metamodel_path else None - # metamodel = load_metamodel_data(override_path) - # - # raw = getattr(app.config, "score_metamodel_requirement_types", "").strip() - # filter_reqs = [t.strip() for t in raw.split(",") if t.strip()] - # if not filter_reqs: - # filter_reqs = get_need_types_by_tags( - # metamodel.needs_types, {"reqiurement", "requirement_excl_process"} - # ) - # metrics_by_type: dict[str, Any] = {} - # test_stats - # - # test_needs = list(all_needs.filter_types(["testcase"]).values()) - # test_metrics = calculate_test_metrics(test_needs, current_reqtype_needs) - # for req_type in sorted(filter_reqs): - # needs_of_req_type = all_needs.filter_types([req_type]).filter_is_external(False) - # if not list(needs_of_req_type.values()): - # continue - # type_summary = compute_traceability_summary( - # all_needs=all_needs, - # current_reqtype=req_type, - # current_reqtype_needs=needs_of_req_type, - # include_external=include_external, - # ) - # metrics_by_type[req_type] = { - # "include_external": type_summary["include_external"], - # "requirements": type_summary["requirements"], - # "tests": type_summary["tests"], - # } - # - # output: dict[str, Any] = { - # "schema_version": "1", - # "generated_by": "sphinx_build", - # "metrics_by_type": metrics_by_type, - # } - # out_path = Path(app.outdir) / "metrics.json" - # out_path.parent.mkdir(parents=True, exist_ok=True) - # out_path.write_text(json.dumps(output, indent=2), encoding="utf-8") - # print(f"Traceability metrics written to: {out_path}") + + +def _write_metrics_json(app: Sphinx, exception: Any|None) -> None: + """ + Write a schema-v1 metrics.json alongside needs.json in the build output. + """ + + out_path = Path(app.outdir) / "metrics.json" + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(CALCULATED_METRICS, indent=2), encoding="utf-8") + print(f"Traceability metrics written to: {out_path}") def setup(app: Sphinx) -> dict[str, str | bool]: @@ -102,7 +71,8 @@ def setup(app: Sphinx) -> dict[str, str | bool]: ) # _ = app.connect("config-inited", _configure_traceability_dashboard, priority=498) - _ = app.connect("env-updated", _write_metrics_json, priority=550) + _ = app.connect("env-updated", calculate_need_metrics, priority=550) + _ = app.connect("build-finished", _write_metrics_json, priority=501) return { "version": "0.1", diff --git a/src/extensions/score_metrics/traceability_dashboard.py b/src/extensions/score_metrics/traceability_dashboard.py index 4e135380d..2fd592402 100644 --- a/src/extensions/score_metrics/traceability_dashboard.py +++ b/src/extensions/score_metrics/traceability_dashboard.py @@ -30,24 +30,21 @@ _DEFAULT_INCLUDE_EXTERNAL = False +# ╓ ╖ +# ║ Docstrings of functions generated by Copilot ║ +# ╙ ╜ -def get_linked_metrics_for_type( - needs: list[Any], results: list[int], **kwargs: Any -) -> None: - """Append dynamically selected metrics from a nested JSON file to ``results``. - The ``metric_paths`` argument must be a semicolon-separated list of dot paths, - for example: ``overall_metrics.total;overall_metrics.with_test_link``. +def _get_key_values(results: list[int], argument_paths: list[str]): + """Append integer metric values selected by colon-separated paths. + + Each path is resolved against ``CALCULATED_METRICS`` (for example: + ``"overall_metrics:total"``), converted to ``int``, and appended to + ``results``. """ - # metrics_path = str(kwargs["arg1"]) + ".json" - # metrics_json = json.loads(Path(metrics_path).read_text(encoding="utf-8")) metrics_json = CALCULATED_METRICS - # arg1 = _build/metrics, - # arg2 = overall_metrics, :total,overall_metrics:with_test_link - - metric_values = "overall_metrics.total" + str(kwargs["arg2"]) - for raw_path in metric_values.split(";"): + for raw_path in argument_paths: path = raw_path.strip() if not path: continue @@ -55,8 +52,56 @@ def get_linked_metrics_for_type( current: Any = metrics_json for key in path.split(":"): current = current[key] + results.append(int(current)) + + +def get_metrics_with_overall_total_considered( + needs: list[Any], results: list[int], **kwargs: Any +) -> None: + """Append selected metrics and compute remainder from overall total. + + This function appends ``overall_metrics:total`` first, then appends all + metrics referenced by ``kwargs`` values. Finally, it replaces the first + appended value with the remainder after subtracting all other appended + values. + """ + # metrics_path = str(kwargs["arg1"]) + ".json" + # metrics_json = json.loads(Path(metrics_path).read_text(encoding="utf-8")) + metrics_json = CALCULATED_METRICS + results.append(int(metrics_json["overall_metrics"]["total"])) + _get_key_values(results, [str(value) for value in kwargs.values()]) results[0] -= sum(results[1:]) + print(results) + + +def get_metrics_with_custom_type_total_considered( + needs: list[Any], results: list[int], **kwargs: Any +) -> None: + """Append selected metrics, optionally using a custom total path. + + If the last kwarg value ends with ``":total"``, that path is used as + baseline total and all preceding paths are treated as components; the first + result becomes ``total - sum(components)``. Otherwise, all paths are simply + appended as-is. + """ + # Get the 'total' that was specified as the first value + values = [str(value) for value in kwargs.values()] + if values[-1].endswith(":total"): + _get_key_values(results, [values[-1]]) # baseline total + _get_key_values(results, values[:-1]) # components + results[0] -= sum(results[1:]) + else: + _get_key_values(results, values) + + +def get_just_metrics(needs: list[Any], results: list[int], **kwargs: Any) -> None: + """Append selected metric values without any total/remainder calculation. + + All kwarg values are interpreted as colon-separated metric paths and + appended to ``results`` in insertion order. + """ + _get_key_values(results, [str(value) for value in kwargs.values()]) # def set_default_include_external(include_external: bool) -> None: diff --git a/src/extensions/score_metrics/traceability_metrics.py b/src/extensions/score_metrics/traceability_metrics.py index cc2310255..15af520f5 100644 --- a/src/extensions/score_metrics/traceability_metrics.py +++ b/src/extensions/score_metrics/traceability_metrics.py @@ -31,7 +31,12 @@ CALCULATED_METRICS = {} + def get_need_types_by_tags(needs: list[ScoreNeedType], tags: set[str]) -> list[str]: + """ + Takes a list of 'ScoreNeedTypes' and filters out any that have one or more of the + specified tags given. + """ found_need_types: list[str] = [] for need_type in needs: found_tag_set = set(need_type["tags"]) @@ -59,7 +64,6 @@ def calculate_requirement_metrics( ) -> dict[str, Any]: """Calculate requirement traceability statistics for links and completeness.""" total = len(current_requirement_needs) - print("===============") reqs_with_code_link = 0 reqs_with_test_link = 0 reqs_fully_linked = 0 @@ -112,9 +116,7 @@ def calculate_test_metrics( broken_references: list[dict[str, str]] = [] for test_need in test_needs: - # This should never happen (that it has unknown_testcase id - # TODO: Remove this: - test_id = str(test_need.get("id", "")) + test_id = str(test_need.get("id")) partially: list[str] = test_need.get("partially_verifies") fully: list[str] = test_need.get("fully_verifies") refs = set(partially + fully) @@ -132,70 +134,42 @@ def calculate_test_metrics( } -def calculate_process_requirement_metrics( - all_needs: NeedsView, -) -> dict[str, Any]: - """Calculate process-requirement coverage via tool_req ``satisfies`` links.""" - - process_requirements = all_needs.filter_types(["gd_req"]) - process_requirement_ids = set(process_requirements.keys()) - tool_requirements = all_needs.filter_types(["tool_req"]).filter_is_external(False) - - linked_process_requirement_ids: set[str] = set() - for need in tool_requirements.values(): - satisfies_ids = need.get("satisfies") - for ref_id in satisfies_ids: - if ref_id in process_requirement_ids: - linked_process_requirement_ids.add(ref_id) - - total = len(process_requirement_ids) - linked_by_tool_requirements = len(linked_process_requirement_ids) - unlinked_ids = sorted(process_requirement_ids - linked_process_requirement_ids) - - return { - "total": total, - "linked": linked_by_tool_requirements, - "linked_by_tool_requirements": linked_by_tool_requirements, - "linked_by_tool_requirements_pct": safe_percent( - linked_by_tool_requirements, total - ), - "unlinked_ids": unlinked_ids, - } - - def calculate_full_need_metrics(app: Sphinx, include_external: bool): + """ + Calculate all tracked metrics for requirements and tests. + Will save the result in a global variable 'CALCULATED_METRICS' + """ # ───────────────[ Getting configuration values ]─────────────── global CALCULATED_METRICS - # if CALCULATED_METRICS: - # logger.info("Metrics calculated already, skipping re-execution") - # return all_needs: NeedsView = SphinxNeedsData(app.env).get_needs_view() raw_metamodel_path = app.config.score_metamodel_yaml override_path = Path(raw_metamodel_path) if raw_metamodel_path else None metamodel = load_metamodel_data(override_path) + # We either get the types that should be considered from the configuration + # If none are specified the need types declared in the Metamodel with + # the tags 'requirement' are taken raw = getattr(app.config, "score_metamodel_requirement_types", "").strip() filter_reqs = [t.strip() for t in raw.split(",") if t.strip()] if not filter_reqs: - filter_reqs = get_need_types_by_tags( - metamodel.needs_types, {"reqiurement", "requirement_excl_process"} - ) + filter_reqs = get_need_types_by_tags(metamodel.needs_types, {"reqiurement"}) # ──────────────────[ Calculate Test Metrics ]────────────────── test_needs = list(all_needs.filter_types(["testcase"]).values()) test_metrics = calculate_test_metrics(test_needs, all_needs) - process_requirement_metrics = calculate_process_requirement_metrics( - all_needs, - ) metrics_by_type: dict[str, Any] = {} + + # Metrics accumulated over all requirements types overall_metrics: dict[str, Any] = { "total": 0, "with_code_link": 0, "with_test_link": 0, "fully_linked": 0, } + + # ─────[ Calculating Metrics for each requirement_type ]─── for req_type in sorted(filter_reqs): needs_of_req_type = all_needs.filter_types([req_type]).filter_is_external( include_external @@ -209,6 +183,7 @@ def calculate_full_need_metrics(app: Sphinx, include_external: bool): overall_metrics["with_test_link"] += req_metrics["with_test_link"] overall_metrics["fully_linked"] += req_metrics["fully_linked"] metrics_by_type[req_type] = req_metrics + # Calculating % of each category for the overall metrics overall_metrics["with_code_link_pct"] = safe_percent( overall_metrics["with_code_link"], overall_metrics["total"] ) @@ -225,13 +200,9 @@ def calculate_full_need_metrics(app: Sphinx, include_external: bool): "overall_metrics": overall_metrics, "metrics_by_type": metrics_by_type, "tests": test_metrics, - "process_requirements": process_requirement_metrics, } - app.config.calculated_metrics = output + # Save the metrics in a Global Variable to enable access from other parts. + # Not a great solution but it is needed, as needpie filter functions for example + # can not access 'app'. CALCULATED_METRICS = output - out_path = Path(app.outdir) / "metrics.json" - out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_text(json.dumps(output, indent=2), encoding="utf-8") - print(f"Traceability metrics written to: {out_path}") - From 91ac8c5e0cf43f9aba9ef7a06224769e50b4da15 Mon Sep 17 00:00:00 2001 From: MaximilianSoerenPollak Date: Wed, 3 Jun 2026 17:15:06 +0200 Subject: [PATCH 8/8] WIP: REfactor --- docs/how-to/dashboards_and_quality_gates.rst | 2 +- .../requirements/implementation_state.rst | 4 +- .../score_metamodel/checks/standards.py | 3 +- src/extensions/score_metrics/BUILD | 17 +- src/extensions/score_metrics/__init__.py | 24 +- .../score_metrics/sphinx_filters.py | 84 ++- .../tests/test_sphinx_filters.py | 149 ++++- .../tests/test_traceability_dashboard.py | 6 +- .../tests/test_traceability_metrics.py | 5 - ...st_traceability_metrics_json_generation.py | 628 +++++++++--------- .../score_metrics/traceability_dashboard.py | 72 -- .../score_metrics/traceability_metrics.py | 14 +- .../tests/test_xml_parser.py | 8 +- .../score_source_code_linker/xml_parser.py | 13 +- .../score_sphinx_bundle/__init__.py | 2 +- 15 files changed, 591 insertions(+), 440 deletions(-) diff --git a/docs/how-to/dashboards_and_quality_gates.rst b/docs/how-to/dashboards_and_quality_gates.rst index bc485734d..0bdaf9465 100644 --- a/docs/how-to/dashboards_and_quality_gates.rst +++ b/docs/how-to/dashboards_and_quality_gates.rst @@ -380,7 +380,7 @@ Result: results == [10, 5] get_just_metrics(...) -~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~ Purpose ^^^^^^^ diff --git a/docs/internals/requirements/implementation_state.rst b/docs/internals/requirements/implementation_state.rst index 92adbc39f..95998ae99 100644 --- a/docs/internals/requirements/implementation_state.rst +++ b/docs/internals/requirements/implementation_state.rst @@ -30,14 +30,14 @@ repositories. .. needpie:: Overall Metrics with Total incooperated :labels: without any link, overall with test link, overall with code, overall fully linked :colors: red, yellow, blue, green - :filter-func: src.extensions.score_metrics.traceability_dashboard.get_metrics_with_overall_total_considered(overall_metrics:with_test_link,overall_metrics:with_code_link,overall_metrics:fully_linked) + :filter-func: src.extensions.score_metrics.sphinx_filter.get_metrics_with_overall_total_considered(overall_metrics:with_test_link,overall_metrics:with_code_link,overall_metrics:fully_linked) .. needpie:: Metrics without any total incooperated :labels: tool req with test link, tool req with code, overall fully linked :colors: yellow, blue, green - :filter-func: src.extensions.score_metrics.traceability_dashboard.get_just_metrics(metrics_by_type:tool_req:with_test_link,metrics_by_type:tool_req:with_code_link,overall_metrics:fully_linked) + :filter-func: src.extensions.score_metrics.sphinx_filter.get_just_metrics(metrics_by_type:tool_req:with_test_link,metrics_by_type:tool_req:with_code_link,overall_metrics:fully_linked) .. Jump to evidence tables: diff --git a/src/extensions/score_metamodel/checks/standards.py b/src/extensions/score_metamodel/checks/standards.py index 70261641f..da32a3ce2 100644 --- a/src/extensions/score_metamodel/checks/standards.py +++ b/src/extensions/score_metamodel/checks/standards.py @@ -16,13 +16,12 @@ # ╙ ╜ # from sphinx.application import Sphinx -from sphinx_needs.need_item import NeedItem - from score_metrics.sphinx_filters import ( generic_pie_items_by_tag, generic_pie_items_in_relationships, generic_pie_linked_items, ) +from sphinx_needs.need_item import NeedItem # from score_metamodel import ( # CheckLogger, diff --git a/src/extensions/score_metrics/BUILD b/src/extensions/score_metrics/BUILD index b288353ec..59ca9d7b3 100644 --- a/src/extensions/score_metrics/BUILD +++ b/src/extensions/score_metrics/BUILD @@ -1,3 +1,15 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* load("@aspect_rules_py//py:defs.bzl", "py_library") load("@docs_as_code_hub_env//:requirements.bzl", "all_requirements") load("//:score_pytest.bzl", "score_pytest") @@ -36,5 +48,8 @@ score_pytest( srcs = glob(["tests/*.py"]), pytest_config = "//:pyproject.toml", # All requirements already in the library so no need to have it double - deps = [":score_metrics"], + deps = [ + ":score_metrics", + "@score_docs_as_code//src/extensions/score_metamodel", + ], ) diff --git a/src/extensions/score_metrics/__init__.py b/src/extensions/score_metrics/__init__.py index 39b2674e2..fbc76b5c8 100644 --- a/src/extensions/score_metrics/__init__.py +++ b/src/extensions/score_metrics/__init__.py @@ -1,12 +1,24 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* import json import os from pathlib import Path from typing import Any -from sphinx.application import Sphinx -from sphinx_needs import logging from score_metrics.traceability_metrics import calculate_full_need_metrics +from sphinx.application import Sphinx from sphinx.environment import BuildEnvironment +from sphinx_needs import logging from src.extensions.score_metrics.traceability_metrics import CALCULATED_METRICS @@ -24,7 +36,7 @@ # ) # set_default_include_external(include_external) # -def calculate_need_metrics(app: Sphinx, env:BuildEnvironment) -> None: +def calculate_need_metrics(app: Sphinx, env: BuildEnvironment) -> None: """ This is the single source of truth for traceability metrics. It runs inside the Sphinx build so it has access to all needs (local + external) @@ -37,7 +49,7 @@ def calculate_need_metrics(app: Sphinx, env:BuildEnvironment) -> None: calculate_full_need_metrics(app=app, include_external=include_external) -def _write_metrics_json(app: Sphinx, exception: Any|None) -> None: +def _write_metrics_json(app: Sphinx, exception: Any | None) -> None: """ Write a schema-v1 metrics.json alongside needs.json in the build output. """ @@ -71,8 +83,8 @@ def setup(app: Sphinx) -> dict[str, str | bool]: ) # _ = app.connect("config-inited", _configure_traceability_dashboard, priority=498) - _ = app.connect("env-updated", calculate_need_metrics, priority=550) - _ = app.connect("build-finished", _write_metrics_json, priority=501) + _ = app.connect("env-updated", calculate_need_metrics, priority=600) + _ = app.connect("build-finished", _write_metrics_json, priority=550) return { "version": "0.1", diff --git a/src/extensions/score_metrics/sphinx_filters.py b/src/extensions/score_metrics/sphinx_filters.py index bf6f2d685..e2fe08a4d 100644 --- a/src/extensions/score_metrics/sphinx_filters.py +++ b/src/extensions/score_metrics/sphinx_filters.py @@ -38,8 +38,10 @@ def func(needs: list[NeedItem], results: list[int], **kwargs) -> None: ... """ from __future__ import annotations +from typing import Any from sphinx_needs.need_item import NeedItem +from score_metrics.traceability_metrics import CALCULATED_METRICS def _matches_source_selector(need: NeedItem, selector: str) -> bool: @@ -55,7 +57,7 @@ def _matches_source_selector(need: NeedItem, selector: str) -> bool: def generic_pie_linked_items( - needs: list[NeedItem], results: list[int], **kwargs: dict[str, str | int | float] + needs: list[NeedItem], results: list[int], **kwargs: Any ) -> None: """Count target IDs by whether they are linked by selected source needs. @@ -63,7 +65,7 @@ def generic_pie_linked_items( selector prefix, matched against source ``type`` and ``id``), and ``arg3`` (link field name, default ``complies``). """ - results = [] + results.clear() id_prefix = str(kwargs.get("arg1", "")) source_selector = str(kwargs.get("arg2", "")) link_field = str(kwargs.get("arg3", "complies")) @@ -90,7 +92,7 @@ def generic_pie_linked_items( def generic_pie_items_by_tag( - needs: list[NeedItem], results: list[int], **kwargs: dict[str, str | int | float] + needs: list[NeedItem], results: list[int], **kwargs: Any ) -> None: """Count tagged items split by whether selected source needs link them. @@ -98,6 +100,7 @@ def generic_pie_items_by_tag( matched against source ``type`` and ``id``), and ``arg3`` (link field name, default ``complies``). """ + results.clear() tag = str(kwargs.get("arg1", "")) source_selector = str(kwargs.get("arg2", "")) link_field = str(kwargs.get("arg3", "complies")) @@ -124,7 +127,7 @@ def generic_pie_items_by_tag( def generic_pie_items_in_relationships( - needs: list[NeedItem], results: list[int], **kwargs: dict[str, str | int | float] + needs: list[NeedItem], results: list[int], **kwargs: Any ) -> None: """Count items of a given type by how many container items reference them. @@ -143,6 +146,7 @@ def generic_pie_items_in_relationships( Appends to *results*: ``[not_referenced_count, referenced_once_count, referenced_multiple_count]`` """ + results.clear() container_type = str(kwargs.get("arg1", "")) field = str(kwargs.get("arg2", "")) item_type = str(kwargs.get("arg3", "")) @@ -167,9 +171,71 @@ def generic_pie_items_in_relationships( results.append(referenced_multiple) -def get_metrics_from_generated_json( - needs: list[NeedItem], results: list[int], **kwargs: str | int | float +def _get_key_values(results: list[int], argument_paths: list[str]): + """Append integer metric values selected by colon-separated paths. + + Each path is resolved against ``CALCULATED_METRICS`` (for example: + ``"overall_metrics:total"``), converted to ``int``, and appended to + ``results``. + """ + metrics_json = CALCULATED_METRICS + for raw_path in argument_paths: + path = raw_path.strip() + if not path: + continue + + current: Any = metrics_json + for key in path.split(":"): + current = current[key] + + results.append(int(current)) + + +def get_metrics_with_overall_total_considered( + needs: list[Any], results: list[int], **kwargs: Any +) -> None: + """Append selected metrics and compute remainder from overall total. + + This function appends ``overall_metrics:total`` first, then appends all + metrics referenced by ``kwargs`` values. Finally, it replaces the first + appended value with the remainder after subtracting all other appended + values. + """ + results.clear() + metrics_json = CALCULATED_METRICS + results.append(int(metrics_json["overall_metrics"]["total"])) + _get_key_values(results, [str(value) for value in kwargs.values()]) + results[0] -= sum(results[1:]) + print(results) + + +def get_metrics_with_custom_type_total_considered( + needs: list[Any], results: list[int], **kwargs: Any ) -> None: - metrics_json_path = str(kwargs.get("arg1", "_build/metrics.json")) - # TODO: Read the Metrics.json and give back the data - results + """Append selected metrics, optionally using a custom total path. + + If the last kwarg value ends with ``":total"``, that path is used as + baseline total and all preceding paths are treated as components; the first + result becomes ``total - sum(components)``. Otherwise, all paths are simply + appended as-is. + """ + # Get the 'total' that was specified as the first value + + results.clear() + values = [str(value) for value in kwargs.values()] + if values[-1].endswith(":total"): + _get_key_values(results, [values[-1]]) # baseline total + _get_key_values(results, values[:-1]) # components + results[0] -= sum(results[1:]) + else: + _get_key_values(results, values) + + +def get_just_metrics(needs: list[Any], results: list[int], **kwargs: Any) -> None: + """Append selected metric values without any total/remainder calculation. + + All kwarg values are interpreted as colon-separated metric paths and + appended to ``results`` in insertion order. + """ + results.clear() + _get_key_values(results, [str(value) for value in kwargs.values()]) diff --git a/src/extensions/score_metrics/tests/test_sphinx_filters.py b/src/extensions/score_metrics/tests/test_sphinx_filters.py index 94de5ed12..3b08c3522 100644 --- a/src/extensions/score_metrics/tests/test_sphinx_filters.py +++ b/src/extensions/score_metrics/tests/test_sphinx_filters.py @@ -12,13 +12,16 @@ # ******************************************************************************* from typing import cast +import pytest -from sphinx_needs.need_item import NeedItem +# noqa +import score_metrics.sphinx_filters as sphinx_filters from score_metrics.sphinx_filters import ( generic_pie_items_by_tag, generic_pie_linked_items, ) +from sphinx_needs.need_item import NeedItem def test_generic_pie_linked_items_matches_source_by_id_prefix() -> None: @@ -37,9 +40,7 @@ def test_generic_pie_linked_items_matches_source_by_id_prefix() -> None: results: list[int] = [] generic_pie_linked_items( - needs, - results, - kwargs={"arg1": "std_req__iso26262__", "arg2": "gd_", "arg3": "complies"}, + needs, results, arg1="std_req__iso26262__", arg2="gd_", arg3="complies" ) assert results == [1, 0] @@ -62,9 +63,143 @@ def test_generic_pie_items_by_tag_matches_source_by_id_prefix() -> None: results: list[int] = [] generic_pie_items_by_tag( - needs, - results, - kwargs={"arg1": "aspice40_man5", "arg2": "gd_", "arg3": "complies"}, + needs, results, arg1="aspice40_man5", arg2="gd_", arg3="complies" ) assert results == [1, 1] + + +EXAMPLE_METRICS = { + "schema_version": "1", + "generated_by": "sphinx_build", + "overall_metrics": { + "total": 61, + "with_code_link": 46, + "with_test_link": 3, + "fully_linked": 2, + "with_code_link_pct": 75.40983606557377, + "with_test_link_pct": 4.918032786885246, + "fully_linked_pct": 3.278688524590164, + }, + "metrics_by_type": { + "tool_req": { + "total": 61, + "with_code_link": 46, + "with_test_link": 3, + "fully_linked": 2, + } + }, + "tests": { + "total": 208, + "linked_to_requirements": 16, + "linked_to_requirements_pct": 7.6923076923076925, + "broken_references": [], + }, +} + + +@pytest.fixture(autouse=True) +def reset_global_metrics(): + """Reset global CALCULATED_METRICS before and after each test.""" + sphinx_filters.CALCULATED_METRICS = {} + yield + sphinx_filters.CALCULATED_METRICS = {} + + +def test_get_key_values_raises_key_error_when_global_is_empty() -> None: + """It raises KeyError if CALCULATED_METRICS is still empty.""" + results: list[int] = [] + with pytest.raises(KeyError): + sphinx_filters._get_key_values(results, ["overall_metrics:total"]) + + +def test_get_key_values_appends_values_when_metrics_loaded() -> None: + """It appends resolved integer values once metrics data is loaded.""" + sphinx_filters.CALCULATED_METRICS = EXAMPLE_METRICS + results: list[int] = [] + + sphinx_filters._get_key_values( + results, + [ + "overall_metrics:total", + "overall_metrics:with_code_link", + "tests:linked_to_requirements", + ], + ) + + assert results == [61, 46, 16] + + +def test_get_metrics_with_overall_total_considered_when_metrics_loaded() -> None: + """It computes remainder from overall total and selected metrics.""" + sphinx_filters.CALCULATED_METRICS = EXAMPLE_METRICS + results: list[int] = [] + + sphinx_filters.get_metrics_with_overall_total_considered( + needs=[], + results=results, + code="overall_metrics:with_code_link", + test="overall_metrics:with_test_link", + fully="overall_metrics:fully_linked", + ) + + assert results == [10, 46, 3, 2] + + +def test_get_metrics_with_custom_type_total_considered_with_total_suffix() -> None: + """It uses trailing ':total' as baseline and computes remainder.""" + sphinx_filters.CALCULATED_METRICS = EXAMPLE_METRICS + results: list[int] = [] + + sphinx_filters.get_metrics_with_custom_type_total_considered( + needs=[], + results=results, + code="metrics_by_type:tool_req:with_code_link", + test="metrics_by_type:tool_req:with_test_link", + total="metrics_by_type:tool_req:total", + ) + + assert results == [12, 46, 3] + + +def test_get_metrics_with_custom_type_total_considered_without_total_suffix() -> None: + """It appends values directly when trailing ':total' is not provided.""" + sphinx_filters.CALCULATED_METRICS = EXAMPLE_METRICS + results: list[int] = [] + + sphinx_filters.get_metrics_with_custom_type_total_considered( + needs=[], + results=results, + code="metrics_by_type:tool_req:with_code_link", + test="metrics_by_type:tool_req:with_test_link", + ) + + assert results == [46, 3] + + +def test_get_just_metrics_appends_values_when_metrics_loaded() -> None: + """It appends selected values without remainder logic.""" + sphinx_filters.CALCULATED_METRICS = EXAMPLE_METRICS + results: list[int] = [] + + sphinx_filters.get_just_metrics( + needs=[], + results=results, + total="overall_metrics:total", + linked_tests="tests:linked_to_requirements", + ) + + assert results == [61, 16] + + +def test_get_metrics_with_custom_type_total_considered_empty_kwargs_raises_index_error() -> ( + None +): + """Current behavior: empty kwargs raises IndexError.""" + sphinx_filters.CALCULATED_METRICS = EXAMPLE_METRICS + results: list[int] = [] + + with pytest.raises(IndexError): + sphinx_filters.get_metrics_with_custom_type_total_considered( + needs=[], results=results + ) diff --git a/src/extensions/score_metrics/tests/test_traceability_dashboard.py b/src/extensions/score_metrics/tests/test_traceability_dashboard.py index 763f5a0b0..adb3a47b2 100644 --- a/src/extensions/score_metrics/tests/test_traceability_dashboard.py +++ b/src/extensions/score_metrics/tests/test_traceability_dashboard.py @@ -13,12 +13,8 @@ """Tests that dashboard filters follow local/external settings.""" -from collections.abc import Sequence -from typing import Any -import pytest - -#from src.extensions.score_metamodel.checks import traceability_dashboard +# from src.extensions.score_metamodel.checks import traceability_dashboard # from src.extensions.score_metrics.traceability_dashboard import ( # pie_process_requirements_linked, # pie_requirements_fully_linked, diff --git a/src/extensions/score_metrics/tests/test_traceability_metrics.py b/src/extensions/score_metrics/tests/test_traceability_metrics.py index 87478eeb7..e9b4cb4d7 100644 --- a/src/extensions/score_metrics/tests/test_traceability_metrics.py +++ b/src/extensions/score_metrics/tests/test_traceability_metrics.py @@ -13,11 +13,6 @@ """Unit tests for traceability_metrics include_external handling.""" -from src.extensions.score_metrics.traceability_metrics import ( - compute_traceability_summary, - #filter_requirements, -) - def _needs() -> list[dict[str, object]]: return [ diff --git a/src/extensions/score_metrics/tests/test_traceability_metrics_json_generation.py b/src/extensions/score_metrics/tests/test_traceability_metrics_json_generation.py index 47af209cb..2e2e355de 100644 --- a/src/extensions/score_metrics/tests/test_traceability_metrics_json_generation.py +++ b/src/extensions/score_metrics/tests/test_traceability_metrics_json_generation.py @@ -21,317 +21,317 @@ import pytest from sphinx.application import Sphinx -import src.extensions.score_metamodel.__init__ as metamodel_init - - -class _FakeNeedsData: - def __init__(self, env: object): - self._env = env - - def get_needs_view(self) -> dict[str, dict[str, object]]: - return { - "LOCAL_REQ": { - "id": "LOCAL_REQ", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "", - "testlink": "", - "is_external": False, - }, - "LOCAL_FEAT": { - "id": "LOCAL_FEAT", - "type": "feat_req", - "implemented": "YES", - "source_code_link": "", - "testlink": "", - "is_external": False, - }, - "EXT_REQ": { - "id": "EXT_REQ", - "type": "tool_req", - "implemented": "NO", - "source_code_link": "src/ext.py:1", - "testlink": "", - "is_external": True, - }, - "EXT_FEAT": { - "id": "EXT_FEAT", - "type": "feat_req", - "implemented": "NO", - "source_code_link": "src/ext_feat.py:1", - "testlink": "", - "is_external": True, - }, - "EXT_GD": { - "id": "EXT_GD", - "type": "gd_req", - "implemented": "NO", - "source_code_link": "src/ext_gd.py:1", - "testlink": "", - "is_external": True, - }, - } - - -class _FakeNonReqNeedsData: - def __init__(self, env: object): - self._env = env - - def get_needs_view(self) -> dict[str, dict[str, object]]: - return { - "LOCAL_COMP": { - "id": "LOCAL_COMP", - "type": "comp", - "implemented": "YES", - "source_code_link": "", - "testlink": "", - "is_external": False, - }, - "LOCAL_DOC": { - "id": "LOCAL_DOC", - "type": "document", - "implemented": "YES", - "source_code_link": "", - "testlink": "", - "is_external": False, - }, - } - - -def _app( - tmp_path: Path, - include_external: bool, - requirement_types: str = "", - needs_types: list[dict[str, object]] | None = None, -) -> SimpleNamespace: - discovered_types = needs_types or [ - {"directive": "tool_req", "tags": ["requirement_excl_process"]}, - {"directive": "feat_req", "tags": ["requirement"]}, - {"directive": "workflow", "tags": []}, - ] - return SimpleNamespace( - env=object(), - outdir=str(tmp_path), - config=SimpleNamespace( - score_metamodel_requirement_types=requirement_types, - score_metamodel_include_external_needs=include_external, - needs_types=discovered_types, - ), - ) - - -def test_write_metrics_json_defaults_to_local_only( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) - - metamodel_init._write_metrics_json( - cast(Sphinx, _app(tmp_path, include_external=False)), - None, - ) - - payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) - metrics = payload["metrics_by_type"]["tool_req"] - - assert payload["schema_version"] == "1" - assert metrics["include_not_implemented"] is True - assert metrics["include_external"] is False - assert metrics["requirements"]["total"] == 1 - - -def test_write_metrics_json_can_include_external( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) - - metamodel_init._write_metrics_json( - cast(Sphinx, _app(tmp_path, include_external=True)), - None, - ) - - payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) - metrics = payload["metrics_by_type"]["tool_req"] - - assert metrics["include_external"] is True - assert metrics["requirements"]["total"] == 2 - - -def test_explicit_requirement_types_disable_autodiscovery( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) - - metamodel_init._write_metrics_json( - cast( - Sphinx, - _app( - tmp_path, - include_external=False, - requirement_types="tool_req", - ), - ), - None, - ) - - payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) - assert set(payload["metrics_by_type"].keys()) == {"tool_req"} - - -def test_write_metrics_json_autodiscovers_when_types_unset( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) - - metamodel_init._write_metrics_json( - cast( - Sphinx, - _app(tmp_path, include_external=False, requirement_types=""), - ), - None, - ) - - payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) - assert payload["schema_version"] == "1" - assert set(payload["metrics_by_type"].keys()) == {"feat_req", "tool_req"} - - -def test_autodiscovery_excludes_tagged_types_not_present_in_needs( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) - - metamodel_init._write_metrics_json( - cast( - Sphinx, - _app( - tmp_path, - include_external=False, - requirement_types="", - needs_types=[ - {"directive": "tool_req", "tags": ["requirement_excl_process"]}, - {"directive": "feat_req", "tags": ["requirement"]}, - {"directive": "aou_req", "tags": ["requirement"]}, - ], - ), - ), - None, - ) - - payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) - assert set(payload["metrics_by_type"].keys()) == {"feat_req", "tool_req"} - - -def test_write_metrics_json_empty_when_no_types_configured_or_discovered( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNonReqNeedsData) - - metamodel_init._write_metrics_json( - cast( - Sphinx, - _app( - tmp_path, - include_external=False, - requirement_types="", - needs_types=[{"directive": "workflow", "tags": []}], - ), - ), - None, - ) - - payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) - assert payload["schema_version"] == "1" - assert payload["metrics_by_type"] == {} - - -def test_autodiscovery_without_tagged_requirement_types_is_empty( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) - - metamodel_init._write_metrics_json( - cast( - Sphinx, - _app( - tmp_path, - include_external=False, - requirement_types="", - needs_types=[{"directive": "workflow", "tags": []}], - ), - ), - None, - ) - - payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) - assert payload["metrics_by_type"] == {} - - -def test_autodiscovery_respects_include_external_scope( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) - - metamodel_init._write_metrics_json( - cast( - Sphinx, - _app( - tmp_path, - include_external=True, - requirement_types="", - needs_types=[ - {"directive": "tool_req", "tags": ["requirement_excl_process"]}, - {"directive": "feat_req", "tags": ["requirement"]}, - {"directive": "gd_req", "tags": ["requirement"]}, - ], - ), - ), - None, - ) - - payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) - assert set(payload["metrics_by_type"].keys()) == {"feat_req", "gd_req", "tool_req"} - - -@pytest.mark.parametrize( - ("requirement_types", "include_external", "should_exist", "expected_totals"), - [ - ("tool_req", False, True, {"tool_req": 1}), - ("feat_req,tool_req", False, True, {"feat_req": 1, "tool_req": 1}), - ("", False, True, {"feat_req": 1, "tool_req": 1}), - (" ", False, True, {"feat_req": 1, "tool_req": 1}), - ("tool_req", True, True, {"tool_req": 2}), - ], -) -def test_write_metrics_json_settings_matrix( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, - requirement_types: str, - include_external: bool, - should_exist: bool, - expected_totals: dict[str, int], -) -> None: - monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) - - metamodel_init._write_metrics_json( - cast( - Sphinx, - _app( - tmp_path, - include_external=include_external, - requirement_types=requirement_types, - ), - ), - None, - ) - - metrics_file = tmp_path / "metrics.json" - assert metrics_file.exists() is should_exist - if not should_exist: - return - - payload = json.loads(metrics_file.read_text(encoding="utf-8")) - by_type = payload["metrics_by_type"] - assert set(by_type.keys()) == set(expected_totals.keys()) - - for req_type, expected_total in expected_totals.items(): - assert by_type[req_type]["requirements"]["total"] == expected_total +# import src.extensions.score_metamodel.__init__ as metamodel_init +# +# +# class _FakeNeedsData: +# def __init__(self, env: object): +# self._env = env +# +# def get_needs_view(self) -> dict[str, dict[str, object]]: +# return { +# "LOCAL_REQ": { +# "id": "LOCAL_REQ", +# "type": "tool_req", +# "implemented": "YES", +# "source_code_link": "", +# "testlink": "", +# "is_external": False, +# }, +# "LOCAL_FEAT": { +# "id": "LOCAL_FEAT", +# "type": "feat_req", +# "implemented": "YES", +# "source_code_link": "", +# "testlink": "", +# "is_external": False, +# }, +# "EXT_REQ": { +# "id": "EXT_REQ", +# "type": "tool_req", +# "implemented": "NO", +# "source_code_link": "src/ext.py:1", +# "testlink": "", +# "is_external": True, +# }, +# "EXT_FEAT": { +# "id": "EXT_FEAT", +# "type": "feat_req", +# "implemented": "NO", +# "source_code_link": "src/ext_feat.py:1", +# "testlink": "", +# "is_external": True, +# }, +# "EXT_GD": { +# "id": "EXT_GD", +# "type": "gd_req", +# "implemented": "NO", +# "source_code_link": "src/ext_gd.py:1", +# "testlink": "", +# "is_external": True, +# }, +# } +# +# +# class _FakeNonReqNeedsData: +# def __init__(self, env: object): +# self._env = env +# +# def get_needs_view(self) -> dict[str, dict[str, object]]: +# return { +# "LOCAL_COMP": { +# "id": "LOCAL_COMP", +# "type": "comp", +# "implemented": "YES", +# "source_code_link": "", +# "testlink": "", +# "is_external": False, +# }, +# "LOCAL_DOC": { +# "id": "LOCAL_DOC", +# "type": "document", +# "implemented": "YES", +# "source_code_link": "", +# "testlink": "", +# "is_external": False, +# }, +# } +# +# +# def _app( +# tmp_path: Path, +# include_external: bool, +# requirement_types: str = "", +# needs_types: list[dict[str, object]] | None = None, +# ) -> SimpleNamespace: +# discovered_types = needs_types or [ +# {"directive": "tool_req", "tags": ["requirement_excl_process"]}, +# {"directive": "feat_req", "tags": ["requirement"]}, +# {"directive": "workflow", "tags": []}, +# ] +# return SimpleNamespace( +# env=object(), +# outdir=str(tmp_path), +# config=SimpleNamespace( +# score_metamodel_requirement_types=requirement_types, +# score_metamodel_include_external_needs=include_external, +# needs_types=discovered_types, +# ), +# ) +# +# +# def test_write_metrics_json_defaults_to_local_only( +# monkeypatch: pytest.MonkeyPatch, tmp_path: Path +# ) -> None: +# monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) +# +# metamodel_init._write_metrics_json( +# cast(Sphinx, _app(tmp_path, include_external=False)), +# None, +# ) +# +# payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) +# metrics = payload["metrics_by_type"]["tool_req"] +# +# assert payload["schema_version"] == "1" +# assert metrics["include_not_implemented"] is True +# assert metrics["include_external"] is False +# assert metrics["requirements"]["total"] == 1 +# +# +# def test_write_metrics_json_can_include_external( +# monkeypatch: pytest.MonkeyPatch, tmp_path: Path +# ) -> None: +# monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) +# +# metamodel_init._write_metrics_json( +# cast(Sphinx, _app(tmp_path, include_external=True)), +# None, +# ) +# +# payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) +# metrics = payload["metrics_by_type"]["tool_req"] +# +# assert metrics["include_external"] is True +# assert metrics["requirements"]["total"] == 2 +# +# +# def test_explicit_requirement_types_disable_autodiscovery( +# monkeypatch: pytest.MonkeyPatch, tmp_path: Path +# ) -> None: +# monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) +# +# metamodel_init._write_metrics_json( +# cast( +# Sphinx, +# _app( +# tmp_path, +# include_external=False, +# requirement_types="tool_req", +# ), +# ), +# None, +# ) +# +# payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) +# assert set(payload["metrics_by_type"].keys()) == {"tool_req"} +# +# +# def test_write_metrics_json_autodiscovers_when_types_unset( +# monkeypatch: pytest.MonkeyPatch, tmp_path: Path +# ) -> None: +# monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) +# +# metamodel_init._write_metrics_json( +# cast( +# Sphinx, +# _app(tmp_path, include_external=False, requirement_types=""), +# ), +# None, +# ) +# +# payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) +# assert payload["schema_version"] == "1" +# assert set(payload["metrics_by_type"].keys()) == {"feat_req", "tool_req"} +# +# +# def test_autodiscovery_excludes_tagged_types_not_present_in_needs( +# monkeypatch: pytest.MonkeyPatch, tmp_path: Path +# ) -> None: +# monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) +# +# metamodel_init._write_metrics_json( +# cast( +# Sphinx, +# _app( +# tmp_path, +# include_external=False, +# requirement_types="", +# needs_types=[ +# {"directive": "tool_req", "tags": ["requirement_excl_process"]}, +# {"directive": "feat_req", "tags": ["requirement"]}, +# {"directive": "aou_req", "tags": ["requirement"]}, +# ], +# ), +# ), +# None, +# ) +# +# payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) +# assert set(payload["metrics_by_type"].keys()) == {"feat_req", "tool_req"} +# +# +# def test_write_metrics_json_empty_when_no_types_configured_or_discovered( +# monkeypatch: pytest.MonkeyPatch, tmp_path: Path +# ) -> None: +# monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNonReqNeedsData) +# +# metamodel_init._write_metrics_json( +# cast( +# Sphinx, +# _app( +# tmp_path, +# include_external=False, +# requirement_types="", +# needs_types=[{"directive": "workflow", "tags": []}], +# ), +# ), +# None, +# ) +# +# payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) +# assert payload["schema_version"] == "1" +# assert payload["metrics_by_type"] == {} +# +# +# def test_autodiscovery_without_tagged_requirement_types_is_empty( +# monkeypatch: pytest.MonkeyPatch, tmp_path: Path +# ) -> None: +# monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) +# +# metamodel_init._write_metrics_json( +# cast( +# Sphinx, +# _app( +# tmp_path, +# include_external=False, +# requirement_types="", +# needs_types=[{"directive": "workflow", "tags": []}], +# ), +# ), +# None, +# ) +# +# payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) +# assert payload["metrics_by_type"] == {} +# +# +# def test_autodiscovery_respects_include_external_scope( +# monkeypatch: pytest.MonkeyPatch, tmp_path: Path +# ) -> None: +# monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) +# +# metamodel_init._write_metrics_json( +# cast( +# Sphinx, +# _app( +# tmp_path, +# include_external=True, +# requirement_types="", +# needs_types=[ +# {"directive": "tool_req", "tags": ["requirement_excl_process"]}, +# {"directive": "feat_req", "tags": ["requirement"]}, +# {"directive": "gd_req", "tags": ["requirement"]}, +# ], +# ), +# ), +# None, +# ) +# +# payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) +# assert set(payload["metrics_by_type"].keys()) == {"feat_req", "gd_req", "tool_req"} +# +# +# @pytest.mark.parametrize( +# ("requirement_types", "include_external", "should_exist", "expected_totals"), +# [ +# ("tool_req", False, True, {"tool_req": 1}), +# ("feat_req,tool_req", False, True, {"feat_req": 1, "tool_req": 1}), +# ("", False, True, {"feat_req": 1, "tool_req": 1}), +# (" ", False, True, {"feat_req": 1, "tool_req": 1}), +# ("tool_req", True, True, {"tool_req": 2}), +# ], +# ) +# def test_write_metrics_json_settings_matrix( +# monkeypatch: pytest.MonkeyPatch, +# tmp_path: Path, +# requirement_types: str, +# include_external: bool, +# should_exist: bool, +# expected_totals: dict[str, int], +# ) -> None: +# monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) +# +# metamodel_init._write_metrics_json( +# cast( +# Sphinx, +# _app( +# tmp_path, +# include_external=include_external, +# requirement_types=requirement_types, +# ), +# ), +# None, +# ) +# +# metrics_file = tmp_path / "metrics.json" +# assert metrics_file.exists() is should_exist +# if not should_exist: +# return +# +# payload = json.loads(metrics_file.read_text(encoding="utf-8")) +# by_type = payload["metrics_by_type"] +# assert set(by_type.keys()) == set(expected_totals.keys()) +# +# for req_type, expected_total in expected_totals.items(): +# assert by_type[req_type]["requirements"]["total"] == expected_total diff --git a/src/extensions/score_metrics/traceability_dashboard.py b/src/extensions/score_metrics/traceability_dashboard.py index 2fd592402..a144a72ac 100644 --- a/src/extensions/score_metrics/traceability_dashboard.py +++ b/src/extensions/score_metrics/traceability_dashboard.py @@ -19,13 +19,8 @@ from __future__ import annotations -import json -from pathlib import Path - -from collections.abc import Sequence from typing import Any -from sphinx_needs.need_item import NeedItem from score_metrics.traceability_metrics import CALCULATED_METRICS _DEFAULT_INCLUDE_EXTERNAL = False @@ -35,73 +30,6 @@ # ╙ ╜ -def _get_key_values(results: list[int], argument_paths: list[str]): - """Append integer metric values selected by colon-separated paths. - - Each path is resolved against ``CALCULATED_METRICS`` (for example: - ``"overall_metrics:total"``), converted to ``int``, and appended to - ``results``. - """ - metrics_json = CALCULATED_METRICS - - for raw_path in argument_paths: - path = raw_path.strip() - if not path: - continue - - current: Any = metrics_json - for key in path.split(":"): - current = current[key] - - results.append(int(current)) - - -def get_metrics_with_overall_total_considered( - needs: list[Any], results: list[int], **kwargs: Any -) -> None: - """Append selected metrics and compute remainder from overall total. - - This function appends ``overall_metrics:total`` first, then appends all - metrics referenced by ``kwargs`` values. Finally, it replaces the first - appended value with the remainder after subtracting all other appended - values. - """ - # metrics_path = str(kwargs["arg1"]) + ".json" - # metrics_json = json.loads(Path(metrics_path).read_text(encoding="utf-8")) - metrics_json = CALCULATED_METRICS - results.append(int(metrics_json["overall_metrics"]["total"])) - _get_key_values(results, [str(value) for value in kwargs.values()]) - results[0] -= sum(results[1:]) - print(results) - - -def get_metrics_with_custom_type_total_considered( - needs: list[Any], results: list[int], **kwargs: Any -) -> None: - """Append selected metrics, optionally using a custom total path. - - If the last kwarg value ends with ``":total"``, that path is used as - baseline total and all preceding paths are treated as components; the first - result becomes ``total - sum(components)``. Otherwise, all paths are simply - appended as-is. - """ - # Get the 'total' that was specified as the first value - values = [str(value) for value in kwargs.values()] - if values[-1].endswith(":total"): - _get_key_values(results, [values[-1]]) # baseline total - _get_key_values(results, values[:-1]) # components - results[0] -= sum(results[1:]) - else: - _get_key_values(results, values) - - -def get_just_metrics(needs: list[Any], results: list[int], **kwargs: Any) -> None: - """Append selected metric values without any total/remainder calculation. - - All kwarg values are interpreted as colon-separated metric paths and - appended to ``results`` in insertion order. - """ - _get_key_values(results, [str(value) for value in kwargs.values()]) # def set_default_include_external(include_external: bool) -> None: diff --git a/src/extensions/score_metrics/traceability_metrics.py b/src/extensions/score_metrics/traceability_metrics.py index 15af520f5..27ccb894a 100644 --- a/src/extensions/score_metrics/traceability_metrics.py +++ b/src/extensions/score_metrics/traceability_metrics.py @@ -19,15 +19,14 @@ from __future__ import annotations -import json +from pathlib import Path from typing import Any -from sphinx_needs.need_item import NeedItem -from sphinx_needs.data import NeedsView -from sphinx.application import Sphinx + from score_metamodel import ScoreNeedType -from pathlib import Path -from sphinx_needs.data import SphinxNeedsData from score_metamodel.yaml_parser import load_metamodel_data +from sphinx.application import Sphinx +from sphinx_needs.data import NeedsView, SphinxNeedsData +from sphinx_needs.need_item import NeedItem CALCULATED_METRICS = {} @@ -202,7 +201,6 @@ def calculate_full_need_metrics(app: Sphinx, include_external: bool): "tests": test_metrics, } # Save the metrics in a Global Variable to enable access from other parts. - # Not a great solution but it is needed, as needpie filter functions for example + # Not a great solution but it is needed, as needpie filter functions for example # can not access 'app'. CALCULATED_METRICS = output - diff --git a/src/extensions/score_source_code_linker/tests/test_xml_parser.py b/src/extensions/score_source_code_linker/tests/test_xml_parser.py index c1725d31c..a184e2178 100644 --- a/src/extensions/score_source_code_linker/tests/test_xml_parser.py +++ b/src/extensions/score_source_code_linker/tests/test_xml_parser.py @@ -302,7 +302,9 @@ def test_read_test_xml_file( # No properties at all => Should not be a 'valid' testlink needs2, no_props2, missing_props2 = xml_parser.read_test_xml_file(dir2 / "test.xml") - assert needs2 == [] + assert isinstance(needs2, list) and len(needs2) == 1 + tcneed2 = needs2[0] + assert isinstance(tcneed2, DataOfTestCase) assert no_props2 == ["tc_no_props"] assert missing_props2 == [] @@ -316,7 +318,9 @@ def test_read_test_xml_file( # Missing some properties => Should not be a 'valid' testlink needs4, no_props4, missing_props4 = xml_parser.read_test_xml_file(dir4 / "test.xml") - assert needs4 == [] + assert isinstance(needs4, list) and len(needs4) == 1 + tcneed4 = needs4[0] + assert isinstance(tcneed4, DataOfTestCase) assert no_props4 == [] assert missing_props4 == ["tc_with_missing_props"] diff --git a/src/extensions/score_source_code_linker/xml_parser.py b/src/extensions/score_source_code_linker/xml_parser.py index 48c9277ca..9a8bcac63 100644 --- a/src/extensions/score_source_code_linker/xml_parser.py +++ b/src/extensions/score_source_code_linker/xml_parser.py @@ -237,13 +237,16 @@ def read_test_xml_file(file: Path) -> tuple[list[DataOfTestCase], list[str], lis # I think it should be possible to save the 'from_dict' operation # If the is_valid method would return 'False' anyway. # I just can't think of it right now, leaving this for future me - if properties_element: + if properties_element is not None: case_properties = parse_properties(case_properties, properties_element) case_properties.update(md) - test_case = DataOfTestCase.from_dict(case_properties) - # We will check if the testcase is valid, but still will generate the TestCaseNeed - if not test_case.is_valid(): - missing_prop_tests.append(testname) + + test_case = DataOfTestCase.from_dict(case_properties) + if not test_case.is_valid(): + missing_prop_tests.append(testname) + else: + non_prop_tests.append(testname) + test_case = DataOfTestCase.from_dict(case_properties) test_case_needs.append(test_case) return test_case_needs, non_prop_tests, missing_prop_tests diff --git a/src/extensions/score_sphinx_bundle/__init__.py b/src/extensions/score_sphinx_bundle/__init__.py index 57b2d6892..31d81e91f 100644 --- a/src/extensions/score_sphinx_bundle/__init__.py +++ b/src/extensions/score_sphinx_bundle/__init__.py @@ -31,7 +31,7 @@ "sphinxcontrib.mermaid", "needs_config_writer", "score_sync_toml", - "score_metrics" + "score_metrics", ]