diff --git a/docs/conf.py b/docs/conf.py index 0255915c2..56031d451 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,6 +10,7 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* +import matplotlib project = "Score Docs-as-Code" project_url = "https://eclipse-score.github.io/docs-as-code/" @@ -18,3 +19,4 @@ extensions = [ "score_sphinx_bundle", ] +matplotlib.rcParamsDefault["savefig.bbox"] = "tight" diff --git a/docs/how-to/dashboards_and_quality_gates.rst b/docs/how-to/dashboards_and_quality_gates.rst index ea0797579..0bdaf9465 100644 --- a/docs/how-to/dashboards_and_quality_gates.rst +++ b/docs/how-to/dashboards_and_quality_gates.rst @@ -40,20 +40,35 @@ Typical Setup For details, see :ref:`setup`. -Minimal Configuration Example ------------------------------ +Configuration +------------- + +Default Behavior (No Configuration Needed) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default, ``score_metamodel`` autodiscovers requirement types from the +repository needs present in the current build. Requirement types are identified +from ``needs_types`` entries tagged with ``requirement`` or +``requirement_excl_process``. + +This is the recommended setup for most repositories. -In ``docs/conf.py``: +Optional Override for Requirement Types +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If a repository needs to force a specific set of requirement types, set an +explicit override in ``docs/conf.py``: .. code-block:: python score_metamodel_requirement_types = "feat_req,comp_req,aou_req" - score_metamodel_include_external_needs = False + +When this setting is provided, the explicit list is used instead of +autodiscovery. Use ``score_metamodel_include_external_needs = True`` only in repositories that intentionally aggregate requirements across module dependencies, such as -integration repositories. Use ``False`` for module repositories to gate only on -local traceability. +integration repositories. Building the Dashboard ---------------------- @@ -87,7 +102,7 @@ There are two common modes: **Module repository** -- Set ``score_metamodel_include_external_needs = False``. +- No setting needed. Local-only scope is the default. - Gate only on the needs owned by the repository itself. - Use this for per-module implementation progress and traceability. @@ -136,6 +151,296 @@ For a new consumer repository: 4. Introduce modest thresholds in CI. 5. Raise thresholds over time as the repository matures. +.. + ╓ ╖ + ║ .. The Following part has been generated by Copilot ║ + ╙ ╜ + +Metrics Needpie functions +========================= + +Overview +-------- + +These helpers read values from a nested dictionary named ``CALCULATED_METRICS``. +A metric is selected by a **colon-separated path**, for example: + +- ``overall_metrics:total`` +- ``overall_metrics:with_test_link`` +- ``types:bug:total`` + +All resolved values are converted to ``int`` and appended to a mutable +``results`` list passed by the caller. + +The ``needs`` parameter is accepted for integration compatibility, but it is not +used by the current implementations. + +Path format +----------- + +Paths are split using ``:`` and then resolved step by step. + +Example: + +.. code-block:: text + + path = "overall_metrics:total" + +is resolved like: + +.. code-block:: python + + current = CALCULATED_METRICS + current = current["overall_metrics"] + current = current["total"] + +Function reference +------------------ + +_get_key_values(results, argument_paths) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Purpose +^^^^^^^ + +Internal helper that appends metric values for a list of paths. + +Behavior +^^^^^^^^ + +1. Iterate over each item in ``argument_paths``. +2. Strip whitespace. +3. Skip empty paths. +4. Resolve the path inside ``CALCULATED_METRICS``. +5. Convert to integer and append to ``results``. + +Notes +^^^^^ + +- Modifies ``results`` in place. +- Does not return a new list. +- Raises ``KeyError`` if a path key does not exist. +- Raises ``ValueError`` if a resolved value cannot be converted to ``int``. + +get_metrics_with_overall_total_considered(...) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Purpose +^^^^^^^ + +Compute a remainder against global overall total. + +Behavior +^^^^^^^^ + +1. Append ``CALCULATED_METRICS["overall_metrics"]["total"]`` as first value. +2. Append each metric referenced by ``kwargs.values()``. +3. Replace the first value with: + + ``overall_total - sum(all other appended values)`` + +Typical use case +^^^^^^^^^^^^^^^^ + +Use this when your pie/chart needs an "Other" bucket based on the global total. + +Example +^^^^^^^ + +Assume: + +.. code-block:: python + + CALCULATED_METRICS = { + "overall_metrics": { + "total": 100, + "with_test_link": 30, + "with_review": 20, + } + } + +Call: + +.. code-block:: python + + results = [] + get_metrics_with_overall_total_considered( + needs=[], + results=results, + a="overall_metrics:with_test_link", + b="overall_metrics:with_review", + ) + +Result: + +.. code-block:: python + + # Step 1 append total: [100] + # Step 2 append selected: [100, 30, 20] + # Step 3 remainder: [50, 30, 20] + results == [50, 30, 20] + +get_metrics_with_custom_type_total_considered(...) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Purpose +^^^^^^^ + +Support custom total path if provided as the last kwarg value. + +Behavior +^^^^^^^^ + +If the **last** provided path ends with ``:total``: + +1. Append that last path as baseline total. +2. Append all preceding paths as components. +3. Replace the first value with: + + ``baseline_total - sum(components)`` + +Otherwise: + +- Append all provided paths directly (no subtraction). + +Important ordering rule +^^^^^^^^^^^^^^^^^^^^^^^ + +The special total path must be the **last** kwarg value, for example: + +.. code-block:: python + + get_metrics_with_custom_type_total_considered( + needs=[], + results=results, + part_a="types:feature:done", + part_b="types:feature:in_progress", + total="types:feature:total", # must be last + ) + +Example with custom total +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Assume: + +.. code-block:: python + + CALCULATED_METRICS = { + "types": { + "feature": { + "total": 40, + "done": 10, + "in_progress": 5, + } + } + } + +Call: + +.. code-block:: python + + results = [] + get_metrics_with_custom_type_total_considered( + needs=[], + results=results, + done="types:feature:done", + in_progress="types:feature:in_progress", + total="types:feature:total", + ) + +Result: + +.. code-block:: python + + # baseline total: 40 + # components: 10, 5 + # remainder: 40 - (10 + 5) = 25 + results == [25, 10, 5] + +Example without custom total +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Call: + +.. code-block:: python + + results = [] + get_metrics_with_custom_type_total_considered( + needs=[], + results=results, + done="types:feature:done", + in_progress="types:feature:in_progress", + ) + +Result: + +.. code-block:: python + + # no subtraction mode + results == [10, 5] + +get_just_metrics(...) +~~~~~~~~~~~~~~~~~~~~~ + +Purpose +^^^^^^^ + +Append only the selected metric values. + +Behavior +^^^^^^^^ + +- Interprets each kwarg value as a path. +- Resolves and appends each value. +- No total baseline and no remainder calculation. + +Example +^^^^^^^ + +.. code-block:: python + + results = [] + get_just_metrics( + needs=[], + results=results, + a="overall_metrics:with_test_link", + b="overall_metrics:with_review", + ) + # results might become [30, 20] + +How to use these functions correctly +------------------------------------ + +1. Pass a mutable list in ``results`` (usually ``[]`` initially). +2. Provide metric paths through keyword argument values. +3. Use colon-separated keys (``a:b:c``), not dot notation. +4. For custom-total mode, ensure the total path is the last kwarg value. +5. Expect in-place mutation of ``results``. + +Common pitfalls +--------------- + +- Empty kwargs in custom-total function can lead to index errors + (because ``values[-1]`` is accessed). +- Invalid paths raise ``KeyError``. +- Non-integer values raise ``ValueError`` during ``int(...)`` conversion. +- ``print(results)`` currently causes side effects during execution; remove if + quiet behavior is preferred in production. + +Testing recommendations +----------------------- + +Add unit tests for: + +- Basic path resolution with one and multiple paths. +- Whitespace and empty-path handling. +- Overall total remainder logic. +- Custom total behavior when last path ends with ``:total``. +- Behavior when no custom total is provided. +- Error handling for missing keys and non-integer values. +- Empty ``kwargs`` behavior in custom-total function. + + Related Guides -------------- diff --git a/docs/internals/requirements/implementation_state.rst b/docs/internals/requirements/implementation_state.rst index e46df74fa..95998ae99 100644 --- a/docs/internals/requirements/implementation_state.rst +++ b/docs/internals/requirements/implementation_state.rst @@ -23,105 +23,114 @@ requirements. It focuses on tooling capabilities offered to downstream repositories rather than on product-specific traceability inside those repositories. -Overview --------- - -.. needpie:: Tool Requirements Status - :labels: not implemented, implemented but incomplete traceability, fully linked - :colors: red,yellow, green - :filter-func: src.extensions.score_metamodel.checks.traceability_dashboard.pie_requirements_status(tool_req) - -Jump to evidence tables: - -- :ref:`Tool Requirement Implementation and Links table ` -- :ref:`Process Requirement to Tool Requirement mapping table ` - -How To Read These Levels ------------------------- - -The overview pie combines implementation state and traceability evidence: - -- ``not implemented``: - requirement has ``implemented == NO``. -- ``implemented but incomplete traceability``: - requirement has ``implemented == YES`` or ``implemented == PARTIAL``, - but is missing at least one traceability link (code link and/or test link). -- ``fully linked``: - requirement is implemented and has both ``source_code_link`` and ``testlink``. - -Implementation labels used on this page: - -- ``NO``: requirement is not implemented. -- ``PARTIAL``: requirement is partly implemented. -- ``YES``: requirement is implemented. - -Why multiple pies are shown: - -- ``Requirements with Codelinks`` shows requirement-to-implementation traceability. -- ``Requirements with linked tests`` shows requirement-to-verification traceability. -- ``Requirements fully linked`` is the strict roll-up (both links present). - -These are intentionally separate because they answer different diagnostics: -missing code links, missing test links, or both. - -In Detail ---------- - -.. grid:: 2 - :class-container: score-grid - - .. grid-item-card:: - - .. needpie:: Requirements marked as Implemented - :labels: not implemented, partial, implemented - :colors: red, orange, green - - type == 'tool_req' and implemented == 'NO' - type == 'tool_req' and implemented == 'PARTIAL' - type == 'tool_req' and implemented == 'YES' - - .. grid-item-card:: - - .. needpie:: Requirements with Codelinks - :labels: no codelink, with codelink - :colors: red, green - :filter-func: src.extensions.score_metamodel.checks.traceability_dashboard.pie_requirements_with_code_links(tool_req) - - .. grid-item-card:: - - .. needpie:: Requirements with linked tests - :labels: no test link, with test link - :colors: red, green - :filter-func: src.extensions.score_metamodel.checks.traceability_dashboard.pie_requirements_with_test_links(tool_req) - - .. grid-item-card:: - - .. needpie:: Requirements fully linked (code + tests) - :labels: not fully linked, fully linked - :colors: orange, green - :filter-func: src.extensions.score_metamodel.checks.traceability_dashboard.pie_requirements_fully_linked(tool_req) - - .. grid-item-card:: - - .. needpie:: Process requirements linked by tool requirements - :labels: not linked, linked - :colors: red, green - :filter-func: src.extensions.score_metamodel.checks.traceability_dashboard.pie_process_requirements_linked(tool_req,true) +.. Overview +.. -------- +.. +.. needpie:: Overall Metrics with Total incooperated + :labels: without any link, overall with test link, overall with code, overall fully linked + :colors: red, yellow, blue, green + :filter-func: src.extensions.score_metrics.sphinx_filter.get_metrics_with_overall_total_considered(overall_metrics:with_test_link,overall_metrics:with_code_link,overall_metrics:fully_linked) -Process-to-Tool Mapping ------------------------ -.. _tooling_coverage_table_process_mapping: -.. needtable:: Process requirement -> tool requirement mapping - :types: tool_req - :columns: satisfies as "Process Requirement";id as "Tool Requirement" - :style: table +.. needpie:: Metrics without any total incooperated + :labels: tool req with test link, tool req with code, overall fully linked + :colors: yellow, blue, green + :filter-func: src.extensions.score_metrics.sphinx_filter.get_just_metrics(metrics_by_type:tool_req:with_test_link,metrics_by_type:tool_req:with_code_link,overall_metrics:fully_linked) -.. _tooling_coverage_table_impl_links: -.. needtable:: Tool requirement implementation and links - :types: tool_req - :columns: id as "Tool Requirement";implemented;source_code_link;testlink - :style: table +.. Jump to evidence tables: +.. +.. - :ref:`Tool Requirement Implementation and Links table ` +.. - :ref:`Process Requirement to Tool Requirement mapping table ` +.. +.. How To Read These Levels +.. ------------------------ +.. +.. The overview pie combines implementation state and traceability evidence: +.. +.. - ``not implemented``: +.. requirement has ``implemented == NO``. +.. - ``implemented but incomplete traceability``: +.. requirement has ``implemented == YES`` or ``implemented == PARTIAL``, +.. but is missing at least one traceability link (code link and/or test link). +.. - ``fully linked``: +.. requirement is implemented and has both ``source_code_link`` and ``testlink``. +.. +.. Implementation labels used on this page: +.. +.. - ``NO``: requirement is not implemented. +.. - ``PARTIAL``: requirement is partly implemented. +.. - ``YES``: requirement is implemented. +.. +.. Why multiple pies are shown: +.. +.. - ``Requirements with Codelinks`` shows requirement-to-implementation traceability. +.. - ``Requirements with linked tests`` shows requirement-to-verification traceability. +.. - ``Requirements fully linked`` is the strict roll-up (both links present). +.. +.. These are intentionally separate because they answer different diagnostics: +.. missing code links, missing test links, or both. +.. +.. In Detail +.. --------- +.. +.. .. grid:: 2 +.. :class-container: score-grid +.. +.. .. grid-item-card:: +.. +.. .. needpie:: Requirements marked as Implemented +.. :labels: not implemented, partial, implemented +.. :colors: red, orange, green +.. +.. type == 'tool_req' and implemented == 'NO' +.. type == 'tool_req' and implemented == 'PARTIAL' +.. type == 'tool_req' and implemented == 'YES' +.. +.. .. grid-item-card:: +.. +.. .. needpie:: Requirements with Codelinks +.. :labels: no codelink, with codelink +.. :colors: red, green +.. :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_requirements_with_code_links(tool_req) +.. +.. .. grid-item-card:: +.. +.. .. needpie:: Requirements with linked tests +.. :labels: no test link, with test link +.. :colors: red, green +.. :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_requirements_with_test_links(tool_req) +.. +.. .. grid-item-card:: +.. +.. .. needpie:: Requirements fully linked (code + tests) +.. :labels: not fully linked, fully linked +.. :colors: orange, green +.. :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_requirements_fully_linked(tool_req) +.. +.. .. grid-item-card:: +.. +.. .. needpie:: Process requirements linked by tool requirements +.. :labels: not linked, linked +.. :colors: red, green +.. :filter-func: src.extensions.score_metrics.traceability_dashboard.pie_process_requirements_linked(tool_req,true) +.. +.. +.. Process-to-Tool Mapping +.. ----------------------- +.. +.. .. _tooling_coverage_table_process_mapping: +.. +.. .. needtable:: Process requirement -> tool requirement mapping +.. :types: tool_req +.. :columns: satisfies as "Process Requirement";id as "Tool Requirement" +.. :style: table +.. +.. .. _tooling_coverage_table_impl_links: +.. +.. .. needtable:: Tool requirement implementation and links +.. :types: tool_req +.. :columns: id as "Tool Requirement";implemented;source_code_link;testlink +.. :style: table diff --git a/src/BUILD b/src/BUILD index f12df2fa2..c4515ce09 100644 --- a/src/BUILD +++ b/src/BUILD @@ -41,6 +41,7 @@ filegroup( "//src/extensions/score_source_code_linker:all_sources", "//src/extensions/score_sphinx_bundle:all_sources", "//src/extensions/score_sync_toml:all_sources", + "//src/extensions/score_metrics:all_sources", "//src/helper_lib:all_sources", ], visibility = ["//visibility:public"], diff --git a/src/extensions/score_metamodel/BUILD b/src/extensions/score_metamodel/BUILD index 82fe3eaed..2a182a195 100644 --- a/src/extensions/score_metamodel/BUILD +++ b/src/extensions/score_metamodel/BUILD @@ -56,6 +56,7 @@ py_library( # TODO: Figure out if all requirements are needed or if we can break it down a bit deps = all_requirements + [ "@score_docs_as_code//src/helper_lib", + "@score_docs_as_code//src/extensions/score_metrics:score_metrics", ], ) diff --git a/src/extensions/score_metamodel/__init__.py b/src/extensions/score_metamodel/__init__.py index 7f26c9b83..3f837ed84 100644 --- a/src/extensions/score_metamodel/__init__.py +++ b/src/extensions/score_metamodel/__init__.py @@ -31,9 +31,6 @@ ProhibitedWordCheck as ProhibitedWordCheck, ScoreNeedType as ScoreNeedType, ) -from src.extensions.score_metamodel.traceability_metrics import ( - compute_traceability_summary, -) from src.extensions.score_metamodel.yaml_parser import ( default_options as default_options, load_metamodel_data as load_metamodel_data, @@ -99,54 +96,6 @@ def graph_check(func: graph_check_function): return func -def _write_metrics_json(app: Sphinx, exception: Exception | None) -> None: - """Write a schema-v1 metrics.json alongside needs.json in the build output. - - This is the single source of truth for traceability metrics. It runs - inside the Sphinx build so it has access to all needs (local + external) - and produces the same metrics the dashboard pie charts display. - The traceability_gate reads this file to enforce CI thresholds. - """ - if exception: - return - - all_needs: list[Any] = list(SphinxNeedsData(app.env).get_needs_view().values()) - - raw = str(getattr(app.config, "score_metamodel_requirement_types", "tool_req")) - requirement_types = {t.strip() for t in raw.split(",") if t.strip()} or {"tool_req"} - include_not_implemented = True - include_external: bool = bool( - getattr(app.config, "score_metamodel_include_external_needs", False) - ) - - metrics_by_type: dict[str, Any] = {} - for req_type in sorted(requirement_types): - type_summary = compute_traceability_summary( - all_needs=all_needs, - requirement_types={req_type}, - include_not_implemented=include_not_implemented, - filtered_test_types=set(), - include_external=include_external, - ) - metrics_by_type[req_type] = { - "include_not_implemented": type_summary["include_not_implemented"], - "include_external": type_summary["include_external"], - "requirements": type_summary["requirements"], - "tests": type_summary["tests"], - } - - output: dict[str, Any] = { - "schema_version": "1", - "generated_by": "sphinx_build", - "metrics_by_type": metrics_by_type, - } - - out_path = Path(app.outdir) / "metrics.json" - out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_text(json.dumps(output, indent=2), encoding="utf-8") - logger.info(f"Traceability metrics written to: {out_path}") - - def _run_checks(app: Sphinx, exception: Exception | None) -> None: # Do not run checks if an exception occurred during build if exception: @@ -214,32 +163,6 @@ def is_check_enabled(check: local_check_function | graph_check_function): ) -def _configure_traceability_dashboard(app: Sphinx, config: object) -> None: - """Propagate repo-level traceability settings to dashboard filters.""" - from src.extensions.score_metamodel.checks.traceability_dashboard import ( - set_default_include_external, - ) - - include_external = bool( - getattr(config, "score_metamodel_include_external_needs", False) - ) - set_default_include_external(include_external) - - -def _remove_prefix(word: str, prefixes: list[str]) -> str: - for prefix in prefixes or []: - if isinstance(word, str) and word.startswith(prefix): - return word.removeprefix(prefix) - return word - - -def _get_need_type_for_need(app: Sphinx, need: NeedItem) -> ScoreNeedType: - for nt in app.config.needs_types: - if nt["directive"] == need["type"]: - return nt - raise ValueError(f"Need type {need['type']} not found in needs_types") - - def _resolve_linkable_types( link_name: str, link_value: str, @@ -353,29 +276,6 @@ def setup(app: Sphinx) -> dict[str, str | bool]: ), ) - app.add_config_value( - "score_metamodel_requirement_types", - "tool_req", - rebuild="env", - description=( - "Comma-separated list of need types treated as requirements for " - "traceability metrics (default: tool_req)." - ), - ) - - app.add_config_value( - "score_metamodel_include_external_needs", - False, - rebuild="env", - description=( - "When True, include external requirements in dashboard and CI metrics. " - "Default is False so each repo gates only its own needs." - ), - ) - - _ = app.connect("config-inited", _configure_traceability_dashboard, priority=498) - - _ = app.connect("build-finished", _write_metrics_json) _ = app.connect("build-finished", _run_checks) return { diff --git a/src/extensions/score_metamodel/checks/standards.py b/src/extensions/score_metamodel/checks/standards.py index a04d176ba..da32a3ce2 100644 --- a/src/extensions/score_metamodel/checks/standards.py +++ b/src/extensions/score_metamodel/checks/standards.py @@ -16,13 +16,12 @@ # ╙ ╜ # from sphinx.application import Sphinx -from sphinx_needs.need_item import NeedItem - -from ..sphinx_filters import ( +from score_metrics.sphinx_filters import ( generic_pie_items_by_tag, generic_pie_items_in_relationships, generic_pie_linked_items, ) +from sphinx_needs.need_item import NeedItem # from score_metamodel import ( # CheckLogger, diff --git a/src/extensions/score_metamodel/checks/traceability_dashboard.py b/src/extensions/score_metamodel/checks/traceability_dashboard.py deleted file mode 100644 index a59087928..000000000 --- a/src/extensions/score_metamodel/checks/traceability_dashboard.py +++ /dev/null @@ -1,181 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -# ╓ ╖ -# ║ Some portions generated by Github Copilot ║ -# ╙ ╜ - -"""Needpie filter functions backed by shared traceability metric calculations.""" - -from __future__ import annotations - -from collections.abc import Sequence -from typing import Any - -from sphinx_needs.need_item import NeedItem - -from ..traceability_metrics import compute_traceability_summary, filter_requirements - -_DEFAULT_INCLUDE_EXTERNAL = False - - -def set_default_include_external(include_external: bool) -> None: - """Configure default behaviour for including external requirements.""" - global _DEFAULT_INCLUDE_EXTERNAL - _DEFAULT_INCLUDE_EXTERNAL = bool(include_external) - - -def _include_external(kwargs: dict[str, str | int | float]) -> bool: - """Read include_external override from filter args, else use configured default.""" - raw = kwargs.get("arg2") - if raw is None: - return _DEFAULT_INCLUDE_EXTERNAL - text = str(raw).strip().lower() - return text in {"1", "true", "yes", "on"} - - -def _requirement_types(kwargs: dict[str, str | int | float]) -> set[str]: - raw = str(kwargs.get("arg1", "tool_req")).strip() - values = {value.strip() for value in raw.split(",") if value.strip()} - return values or {"tool_req"} - - -def pie_requirements_status( - needs: Sequence[NeedItem | dict[str, Any]], - results: list[int], - **kwargs: str | int | float, -) -> None: - """Dashboard status split: not implemented, implemented/incomplete, fully linked.""" - req_types = _requirement_types(kwargs) - include_external = _include_external(kwargs) - - all_requirements = filter_requirements( - needs, - requirement_types=req_types, - include_not_implemented=True, - include_external=include_external, - ) - implemented_requirements = filter_requirements( - needs, - requirement_types=req_types, - include_not_implemented=False, - include_external=include_external, - ) - summary = compute_traceability_summary( - all_needs=needs, - requirement_types=req_types, - include_not_implemented=False, - filtered_test_types=set(), - include_external=include_external, - ) - - not_implemented = len(all_requirements) - len(implemented_requirements) - fully_linked = int(summary["requirements"]["fully_linked"]) - implemented_incomplete = len(implemented_requirements) - fully_linked - - results.append(not_implemented) - results.append(implemented_incomplete) - results.append(fully_linked) - - -def pie_requirements_with_code_links( - needs: Sequence[NeedItem | dict[str, Any]], - results: list[int], - **kwargs: str | int | float, -) -> None: - """Dashboard split: requirements with and without source code links.""" - req_types = _requirement_types(kwargs) - include_external = _include_external(kwargs) - summary = compute_traceability_summary( - all_needs=needs, - requirement_types=req_types, - include_not_implemented=True, - filtered_test_types=set(), - include_external=include_external, - ) - - total = int(summary["requirements"]["total"]) - with_code = int(summary["requirements"]["with_code_link"]) - - results.append(total - with_code) - results.append(with_code) - - -def pie_requirements_with_test_links( - needs: Sequence[NeedItem | dict[str, Any]], - results: list[int], - **kwargs: str | int | float, -) -> None: - """Dashboard split: requirements with and without testcase links.""" - req_types = _requirement_types(kwargs) - include_external = _include_external(kwargs) - summary = compute_traceability_summary( - all_needs=needs, - requirement_types=req_types, - include_not_implemented=True, - filtered_test_types=set(), - include_external=include_external, - ) - - total = int(summary["requirements"]["total"]) - with_test = int(summary["requirements"]["with_test_link"]) - - results.append(total - with_test) - results.append(with_test) - - -def pie_requirements_fully_linked( - needs: Sequence[NeedItem | dict[str, Any]], - results: list[int], - **kwargs: str | int | float, -) -> None: - """Dashboard split: requirements fully linked vs incomplete.""" - req_types = _requirement_types(kwargs) - include_external = _include_external(kwargs) - summary = compute_traceability_summary( - all_needs=needs, - requirement_types=req_types, - include_not_implemented=True, - filtered_test_types=set(), - include_external=include_external, - ) - - total = int(summary["requirements"]["total"]) - fully_linked = int(summary["requirements"]["fully_linked"]) - - results.append(total - fully_linked) - results.append(fully_linked) - - -def pie_process_requirements_linked( - needs: Sequence[NeedItem | dict[str, Any]], - results: list[int], - **kwargs: str | int | float, -) -> None: - """Dashboard split: process requirements linked vs not linked.""" - req_types = _requirement_types(kwargs) - include_external = _include_external(kwargs) - summary = compute_traceability_summary( - all_needs=needs, - requirement_types=req_types, - include_not_implemented=True, - filtered_test_types=set(), - include_external=include_external, - ) - - process_requirements = summary["process_requirements"] - total = int(process_requirements["total"]) - linked = int(process_requirements["linked"]) - - results.append(total - linked) - results.append(linked) diff --git a/src/extensions/score_metamodel/tests/test_sphinx_filters.py b/src/extensions/score_metamodel/tests/test_sphinx_filters.py deleted file mode 100644 index c065a01b7..000000000 --- a/src/extensions/score_metamodel/tests/test_sphinx_filters.py +++ /dev/null @@ -1,74 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -from typing import cast - -from sphinx_needs.need_item import NeedItem - -from src.extensions.score_metamodel.sphinx_filters import ( - generic_pie_items_by_tag, - generic_pie_linked_items, -) - - -def test_generic_pie_linked_items_matches_source_by_id_prefix() -> None: - needs = cast( - list[NeedItem], - [ - {"id": "std_req__iso26262__001", "type": "std_req"}, - # Type intentionally does not match selector prefix, id does. - { - "id": "gd_guidl__xyz", - "type": "guideline", - "complies": ["std_req__iso26262__001"], - }, - ], - ) - - results: list[int] = [] - generic_pie_linked_items( - needs, - results, - arg1="std_req__iso26262__", - arg2="gd_", - arg3="complies", - ) - - assert results == [1, 0] - - -def test_generic_pie_items_by_tag_matches_source_by_id_prefix() -> None: - needs = cast( - list[NeedItem], - [ - {"id": "REQ_A", "type": "tool_req", "tags": ["aspice40_man5"]}, - {"id": "REQ_B", "type": "tool_req", "tags": ["aspice40_man5"]}, - # Type intentionally does not match selector prefix, id does. - { - "id": "gd_req__abc", - "type": "process_requirement", - "complies": ["REQ_A"], - }, - ], - ) - - results: list[int] = [] - generic_pie_items_by_tag( - needs, - results, - arg1="aspice40_man5", - arg2="gd_", - arg3="complies", - ) - - assert results == [1, 1] diff --git a/src/extensions/score_metamodel/tests/test_traceability_dashboard.py b/src/extensions/score_metamodel/tests/test_traceability_dashboard.py deleted file mode 100644 index 2e06b3b7d..000000000 --- a/src/extensions/score_metamodel/tests/test_traceability_dashboard.py +++ /dev/null @@ -1,218 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -"""Tests that dashboard filters follow local/external settings.""" - -from collections.abc import Sequence -from typing import Any - -import pytest - -from src.extensions.score_metamodel.checks import traceability_dashboard -from src.extensions.score_metamodel.checks.traceability_dashboard import ( - pie_process_requirements_linked, - pie_requirements_fully_linked, - pie_requirements_with_code_links, - pie_requirements_with_test_links, - set_default_include_external, -) -from src.extensions.score_metamodel.traceability_metrics import ( - compute_traceability_summary, -) - - -def _needs() -> list[dict[str, object]]: - return [ - { - "id": "LOCAL_REQ", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "", - "testlink": "", - "is_external": False, - }, - { - "id": "LOCAL_SYS_REQ", - "type": "sys_req", - "implemented": "YES", - "source_code_link": "", - "testlink": "T_LOCAL", - "is_external": False, - }, - { - "id": "EXT_REQ", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "src/ext.py:10", - "testlink": "T_EXT", - "is_external": True, - }, - ] - - -def test_dashboard_defaults_to_local_only() -> None: - set_default_include_external(False) - - results: list[int] = [] - pie_requirements_with_code_links(_needs(), results, arg1="tool_req") - - summary = compute_traceability_summary( - all_needs=_needs(), - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=False, - ) - - assert results == [1, 0] - assert results == [ - summary["requirements"]["total"] - summary["requirements"]["with_code_link"], - summary["requirements"]["with_code_link"], - ] - - -def test_dashboard_can_include_external_via_default_flag() -> None: - set_default_include_external(True) - - results: list[int] = [] - pie_requirements_with_code_links(_needs(), results, arg1="tool_req") - - summary = compute_traceability_summary( - all_needs=_needs(), - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=True, - ) - - assert results == [1, 1] - assert results == [ - summary["requirements"]["total"] - summary["requirements"]["with_code_link"], - summary["requirements"]["with_code_link"], - ] - - -def test_dashboard_filter_arg_can_override_default() -> None: - set_default_include_external(True) - - results: list[int] = [] - pie_requirements_with_code_links(_needs(), results, arg1="tool_req", arg2="false") - - assert results == [1, 0] - - -def test_requirements_with_test_links_default_local_only() -> None: - set_default_include_external(False) - - results: list[int] = [] - pie_requirements_with_test_links(_needs(), results, arg1="tool_req") - - summary = compute_traceability_summary( - all_needs=_needs(), - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=False, - ) - - assert results == [1, 0] - assert results == [ - summary["requirements"]["total"] - summary["requirements"]["with_test_link"], - summary["requirements"]["with_test_link"], - ] - - -def test_requirements_with_test_links_can_override_include_external() -> None: - set_default_include_external(False) - - results: list[int] = [] - pie_requirements_with_test_links(_needs(), results, arg1="tool_req", arg2="true") - - assert results == [1, 1] - - -def test_requirements_with_test_links_parses_multiple_types() -> None: - set_default_include_external(False) - - results: list[int] = [] - pie_requirements_with_test_links(_needs(), results, arg1="tool_req,sys_req") - - assert results == [1, 1] - - -def test_requirements_fully_linked_uses_shared_summary() -> None: - set_default_include_external(False) - - results: list[int] = [] - pie_requirements_fully_linked(_needs(), results, arg1="tool_req") - - summary = compute_traceability_summary( - all_needs=_needs(), - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=False, - ) - - assert results == [1, 0] - assert results == [ - summary["requirements"]["total"] - summary["requirements"]["fully_linked"], - summary["requirements"]["fully_linked"], - ] - - -def test_requirements_fully_linked_can_include_external() -> None: - set_default_include_external(True) - - results: list[int] = [] - pie_requirements_fully_linked(_needs(), results, arg1="tool_req") - - assert results == [1, 1] - - -def test_process_requirements_linked_uses_stream_a_process_requirement_totals( - monkeypatch: pytest.MonkeyPatch, -) -> None: - captured: dict[str, object] = {} - - def _fake_summary( - all_needs: Sequence[dict[str, Any]], - requirement_types: set[str], - include_not_implemented: bool, - filtered_test_types: set[str], - include_external: bool, - ) -> dict[str, dict[str, int]]: - captured["all_needs"] = all_needs - captured["requirement_types"] = requirement_types - captured["include_not_implemented"] = include_not_implemented - captured["filtered_test_types"] = filtered_test_types - captured["include_external"] = include_external - return { - "requirements": {"total": 99, "linked": 0}, - "process_requirements": {"total": 4, "linked": 3}, - } - - monkeypatch.setattr( - traceability_dashboard, "compute_traceability_summary", _fake_summary - ) - - results: list[int] = [] - pie_process_requirements_linked( - _needs(), results, arg1="tool_req,sys_req", arg2="true" - ) - - assert results == [1, 3] - assert captured["requirement_types"] == {"tool_req", "sys_req"} - assert captured["include_not_implemented"] is True - assert captured["filtered_test_types"] == set() - assert captured["include_external"] is True diff --git a/src/extensions/score_metamodel/tests/test_traceability_metrics.py b/src/extensions/score_metamodel/tests/test_traceability_metrics.py deleted file mode 100644 index 850f064f6..000000000 --- a/src/extensions/score_metamodel/tests/test_traceability_metrics.py +++ /dev/null @@ -1,206 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -"""Unit tests for traceability_metrics include_external handling.""" - -from src.extensions.score_metamodel.traceability_metrics import ( - compute_traceability_summary, - filter_requirements, -) - - -def _needs() -> list[dict[str, object]]: - return [ - { - "id": "LOCAL_REQ", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "src/local.py:1", - "testlink": "tests/test_local.py::test_ok", - "is_external": False, - }, - { - "id": "EXT_REQ", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "src/external.py:9", - "testlink": "tests/test_external.py::test_ok", - "is_external": True, - }, - { - "id": "TC_1", - "type": "testcase", - "partially_verifies": "LOCAL_REQ", - "fully_verifies": "", - "is_external": False, - }, - ] - - -def test_filter_requirements_defaults_to_local_only() -> None: - filtered = filter_requirements( - _needs(), - requirement_types={"tool_req"}, - include_not_implemented=True, - ) - - assert [need["id"] for need in filtered] == ["LOCAL_REQ"] - - -def test_filter_requirements_can_include_external_needs() -> None: - filtered = filter_requirements( - _needs(), - requirement_types={"tool_req"}, - include_not_implemented=True, - include_external=True, - ) - - assert sorted(need["id"] for need in filtered) == ["EXT_REQ", "LOCAL_REQ"] - - -def test_compute_traceability_summary_propagates_include_external() -> None: - summary_local = compute_traceability_summary( - all_needs=_needs(), - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=False, - ) - summary_all = compute_traceability_summary( - all_needs=_needs(), - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=True, - ) - - assert summary_local["include_external"] is False - assert summary_local["requirements"]["total"] == 1 - assert summary_all["include_external"] is True - assert summary_all["requirements"]["total"] == 2 - - -def test_compute_traceability_summary_process_requirements_summary() -> None: - summary = compute_traceability_summary( - all_needs=[ - { - "id": "TOOL_REQ_1", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "src/req.py:10", - "testlink": "tests/test_req.py::test_ok", - "satisfies": "PR_LOCAL_1,OTHER_REQ", - "is_external": False, - }, - { - "id": "TOOL_REQ_2", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "src/req.py:20", - "testlink": "tests/test_req.py::test_ok_2", - "satisfies": ["PR_LOCAL_1", "PR_LOCAL_2"], - "is_external": False, - }, - { - "id": "PR_LOCAL_1", - "type": "process_req", - "is_external": False, - }, - { - "id": "PR_LOCAL_2", - "type": "gd_req", - "is_external": False, - }, - { - "id": "PR_LOCAL_3", - "type": "gd_req", - "is_external": False, - }, - ], - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=False, - ) - - process_requirements = summary["process_requirements"] - - assert process_requirements["total"] == 3 - assert process_requirements["linked_by_tool_requirements"] == 2 - assert process_requirements["linked_by_tool_requirements_pct"] == (2 / 3) * 100 - assert process_requirements["unlinked_ids"] == ["PR_LOCAL_3"] - - -def test_compute_traceability_summary_process_requirements_respects_include_external() -> ( - None -): - all_needs = [ - { - "id": "TOOL_REQ_LOCAL", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "src/local.py:1", - "testlink": "tests/test_local.py::test_ok", - "satisfies": "PR_LOCAL", - "is_external": False, - }, - { - "id": "TOOL_REQ_EXTERNAL", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "src/external.py:1", - "testlink": "tests/test_external.py::test_ok", - "satisfies": "PR_EXTERNAL", - "is_external": True, - }, - { - "id": "PR_LOCAL", - "type": "gd_req", - "is_external": False, - }, - { - "id": "PR_EXTERNAL", - "type": "gd_req", - "is_external": True, - }, - ] - - summary_local = compute_traceability_summary( - all_needs=all_needs, - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=False, - ) - summary_all = compute_traceability_summary( - all_needs=all_needs, - requirement_types={"tool_req"}, - include_not_implemented=True, - filtered_test_types=set(), - include_external=True, - ) - - assert summary_local["process_requirements"] == { - "total": 1, - "linked": 1, - "linked_by_tool_requirements": 1, - "linked_by_tool_requirements_pct": 100.0, - "unlinked_ids": [], - } - assert summary_all["process_requirements"] == { - "total": 2, - "linked": 2, - "linked_by_tool_requirements": 2, - "linked_by_tool_requirements_pct": 100.0, - "unlinked_ids": [], - } diff --git a/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py b/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py deleted file mode 100644 index 764659874..000000000 --- a/src/extensions/score_metamodel/tests/test_traceability_metrics_json_generation.py +++ /dev/null @@ -1,96 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -"""Tests for Sphinx-side metrics.json generation defaults.""" - -import json -from pathlib import Path -from types import SimpleNamespace -from typing import cast - -import pytest -from sphinx.application import Sphinx - -import src.extensions.score_metamodel.__init__ as metamodel_init - - -class _FakeNeedsData: - def __init__(self, env: object): - self._env = env - - def get_needs_view(self) -> dict[str, dict[str, object]]: - return { - "LOCAL_REQ": { - "id": "LOCAL_REQ", - "type": "tool_req", - "implemented": "YES", - "source_code_link": "", - "testlink": "", - "is_external": False, - }, - "EXT_REQ": { - "id": "EXT_REQ", - "type": "tool_req", - "implemented": "NO", - "source_code_link": "src/ext.py:1", - "testlink": "", - "is_external": True, - }, - } - - -def _app(tmp_path: Path, include_external: bool) -> SimpleNamespace: - return SimpleNamespace( - env=object(), - outdir=str(tmp_path), - config=SimpleNamespace( - score_metamodel_requirement_types="tool_req", - score_metamodel_include_external_needs=include_external, - ), - ) - - -def test_write_metrics_json_defaults_to_local_only( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) - - metamodel_init._write_metrics_json( - cast(Sphinx, _app(tmp_path, include_external=False)), - None, - ) - - payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) - metrics = payload["metrics_by_type"]["tool_req"] - - assert payload["schema_version"] == "1" - assert metrics["include_not_implemented"] is True - assert metrics["include_external"] is False - assert metrics["requirements"]["total"] == 1 - - -def test_write_metrics_json_can_include_external( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) - - metamodel_init._write_metrics_json( - cast(Sphinx, _app(tmp_path, include_external=True)), - None, - ) - - payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) - metrics = payload["metrics_by_type"]["tool_req"] - - assert metrics["include_external"] is True - assert metrics["requirements"]["total"] == 2 diff --git a/src/extensions/score_metamodel/traceability_metrics.py b/src/extensions/score_metamodel/traceability_metrics.py deleted file mode 100644 index 08e1723e9..000000000 --- a/src/extensions/score_metamodel/traceability_metrics.py +++ /dev/null @@ -1,258 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -# ╓ ╖ -# ║ Some portions generated by Github Copilot ║ -# ╙ ╜ - -"""Shared traceability metric calculations for CI checks and dashboards.""" - -from __future__ import annotations - -from collections.abc import Sequence -from typing import Any - - -def is_non_empty(value: Any) -> bool: - """Return True if value should be treated as present for traceability checks.""" - if isinstance(value, str): - return bool(value.strip()) - return bool(value) - - -def parse_need_id_list(value: Any) -> list[str]: - """Normalize need-id lists encoded as CSV strings or string lists.""" - if value is None: - return [] - if isinstance(value, str): - return [item.strip() for item in value.split(",") if item.strip()] - if isinstance(value, list): - out: list[str] = [] - for item in value: - if isinstance(item, str) and item.strip(): - out.append(item.strip()) - return out - return [] - - -def safe_percent(numerator: int, denominator: int) -> float: - """Return percentage in range [0, 100], treating empty denominator as 100%.""" - if denominator == 0: - return 100.0 - return (numerator / denominator) * 100.0 - - -def filter_requirements( - all_needs: Sequence[Any], - requirement_types: set[str], - include_not_implemented: bool, - include_external: bool = False, -) -> list[Any]: - """Extract requirements by type, implementation state, and origin.""" - requirements: list[dict[str, Any]] = [] - for need in all_needs: - need_type = str(need.get("type", "")).strip() - if need_type not in requirement_types: - continue - if not include_external and need.get("is_external", False): - continue - if not include_not_implemented: - implemented = str(need.get("implemented", "")).upper().strip() - if implemented not in {"YES", "PARTIAL"}: - continue - requirements.append(need) - return requirements - - -def calculate_requirement_metrics( - requirements: Sequence[Any], -) -> dict[str, Any]: - """Calculate requirement traceability statistics for links and completeness.""" - total = len(requirements) - with_code = sum( - 1 for need in requirements if is_non_empty(need.get("source_code_link")) - ) - with_test = sum(1 for need in requirements if is_non_empty(need.get("testlink"))) - fully_linked = sum( - 1 - for need in requirements - if is_non_empty(need.get("source_code_link")) - and is_non_empty(need.get("testlink")) - ) - - missing_code_ids = [ - str(need.get("id", "")) - for need in requirements - if not is_non_empty(need.get("source_code_link")) and need.get("id") - ] - missing_test_ids = [ - str(need.get("id", "")) - for need in requirements - if not is_non_empty(need.get("testlink")) and need.get("id") - ] - not_fully_linked_ids = [ - str(need.get("id", "")) - for need in requirements - if ( - ( - not is_non_empty(need.get("source_code_link")) - or not is_non_empty(need.get("testlink")) - ) - and need.get("id") - ) - ] - - return { - "total": total, - "with_code_link": with_code, - "with_test_link": with_test, - "fully_linked": fully_linked, - "with_code_link_pct": safe_percent(with_code, total), - "with_test_link_pct": safe_percent(with_test, total), - "fully_linked_pct": safe_percent(fully_linked, total), - "missing_code_link_ids": sorted(missing_code_ids), - "missing_test_link_ids": sorted(missing_test_ids), - "not_fully_linked_ids": sorted(not_fully_linked_ids), - } - - -def calculate_test_metrics( - all_needs: Sequence[Any], - requirement_ids: set[str], - filtered_test_types: set[str], -) -> dict[str, Any]: - """Calculate testcase linkage and broken testcase-reference statistics.""" - testcases = [ - need for need in all_needs if str(need.get("type", "")).strip() == "testcase" - ] - if filtered_test_types: - testcases = [ - need - for need in testcases - if str(need.get("test_type", need.get("TestType", ""))).strip() - in filtered_test_types - ] - - tests_total = len(testcases) - tests_linked = 0 - broken_references: list[dict[str, str]] = [] - - for test_need in testcases: - test_id = str(test_need.get("id", "")) - partially = parse_need_id_list( - test_need.get("partially_verifies", test_need.get("PartiallyVerifies")) - ) - fully = parse_need_id_list( - test_need.get("fully_verifies", test_need.get("FullyVerifies")) - ) - refs = partially + fully - if refs: - tests_linked += 1 - for ref in refs: - if ref not in requirement_ids: - broken_references.append({"testcase": test_id, "missing_need": ref}) - - return { - "total": tests_total, - "filtered_test_types": sorted(filtered_test_types), - "linked_to_requirements": tests_linked, - "linked_to_requirements_pct": safe_percent(tests_linked, tests_total), - "broken_references": broken_references, - } - - -def calculate_process_requirement_metrics( - all_needs: Sequence[Any], - include_not_implemented: bool, - include_external: bool, -) -> dict[str, Any]: - """Calculate process-requirement coverage via tool_req ``satisfies`` links.""" - process_requirements = [ - need - for need in all_needs - if str(need.get("type", "")).strip() in {"gd_req", "process_req"} - and (include_external or not need.get("is_external", False)) - ] - process_requirement_ids = { - str(need.get("id", "")).strip() - for need in process_requirements - if need.get("id") - } - - tool_requirements = filter_requirements( - all_needs, - requirement_types={"tool_req"}, - include_not_implemented=include_not_implemented, - include_external=include_external, - ) - - linked_process_requirement_ids: set[str] = set() - for need in tool_requirements: - satisfies_ids = parse_need_id_list(need.get("satisfies", need.get("Satisfies"))) - for ref_id in satisfies_ids: - if ref_id in process_requirement_ids: - linked_process_requirement_ids.add(ref_id) - - total = len(process_requirement_ids) - linked_by_tool_requirements = len(linked_process_requirement_ids) - unlinked_ids = sorted(process_requirement_ids - linked_process_requirement_ids) - - return { - "total": total, - "linked": linked_by_tool_requirements, - "linked_by_tool_requirements": linked_by_tool_requirements, - "linked_by_tool_requirements_pct": safe_percent( - linked_by_tool_requirements, total - ), - "unlinked_ids": unlinked_ids, - } - - -def compute_traceability_summary( - all_needs: Sequence[Any], - requirement_types: set[str], - include_not_implemented: bool, - filtered_test_types: set[str], - include_external: bool = False, -) -> dict[str, Any]: - """Return full CI/dashboard summary using one shared metric implementation.""" - requirements = filter_requirements( - all_needs, - requirement_types=requirement_types, - include_not_implemented=include_not_implemented, - include_external=include_external, - ) - requirement_ids = { - str(need.get("id", "")).strip() for need in requirements if need.get("id") - } - - req_metrics = calculate_requirement_metrics(requirements) - test_metrics = calculate_test_metrics( - all_needs, - requirement_ids=requirement_ids, - filtered_test_types=filtered_test_types, - ) - process_requirement_metrics = calculate_process_requirement_metrics( - all_needs, - include_not_implemented=include_not_implemented, - include_external=include_external, - ) - - return { - "requirement_types": sorted(requirement_types), - "include_not_implemented": include_not_implemented, - "include_external": include_external, - "requirements": req_metrics, - "tests": test_metrics, - "process_requirements": process_requirement_metrics, - } diff --git a/src/extensions/score_metrics/BUILD b/src/extensions/score_metrics/BUILD new file mode 100644 index 000000000..59ca9d7b3 --- /dev/null +++ b/src/extensions/score_metrics/BUILD @@ -0,0 +1,55 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +load("@aspect_rules_py//py:defs.bzl", "py_library") +load("@docs_as_code_hub_env//:requirements.bzl", "all_requirements") +load("//:score_pytest.bzl", "score_pytest") + +filegroup( + name = "all_sources", + srcs = glob( + [ + "*.py", + ], + ), + visibility = ["//visibility:public"], +) + +filegroup( + name = "tests", + srcs = glob( + ["tests/*.py"], + ), +) + +py_library( + name = "score_metrics", + srcs = [":all_sources"], + imports = ["."], + visibility = ["//visibility:public"], + # TODO: Figure out if all requirements are needed or if we can break it down a bit + deps = all_requirements + [ + "@score_docs_as_code//src/helper_lib", + ], +) + +score_pytest( + name = "score_metrics_test", + size = "small", + srcs = glob(["tests/*.py"]), + pytest_config = "//:pyproject.toml", + # All requirements already in the library so no need to have it double + deps = [ + ":score_metrics", + "@score_docs_as_code//src/extensions/score_metamodel", + ], +) diff --git a/src/extensions/score_metrics/__init__.py b/src/extensions/score_metrics/__init__.py new file mode 100644 index 000000000..fbc76b5c8 --- /dev/null +++ b/src/extensions/score_metrics/__init__.py @@ -0,0 +1,93 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +import json +import os +from pathlib import Path +from typing import Any + +from score_metrics.traceability_metrics import calculate_full_need_metrics +from sphinx.application import Sphinx +from sphinx.environment import BuildEnvironment +from sphinx_needs import logging + +from src.extensions.score_metrics.traceability_metrics import CALCULATED_METRICS + +logger = logging.get_logger(__name__) + + +# def _configure_traceability_dashboard(app: Sphinx, config: object) -> None: +# """Propagate repo-level traceability settings to dashboard filters.""" +# from src.extensions.score_metrics.traceability_dashboard import ( +# set_default_include_external, +# ) +# +# include_external = bool( +# getattr(config, "score_metamodel_include_external_needs", False) +# ) +# set_default_include_external(include_external) +# +def calculate_need_metrics(app: Sphinx, env: BuildEnvironment) -> None: + """ + This is the single source of truth for traceability metrics. It runs + inside the Sphinx build so it has access to all needs (local + external) + and produces the same metrics the dashboard pie charts display. + The traceability_gate reads this file to enforce CI thresholds. + """ + include_external: bool = bool( + getattr(app.config, "score_metamodel_include_external_needs", False) + ) + calculate_full_need_metrics(app=app, include_external=include_external) + + +def _write_metrics_json(app: Sphinx, exception: Any | None) -> None: + """ + Write a schema-v1 metrics.json alongside needs.json in the build output. + """ + + out_path = Path(app.outdir) / "metrics.json" + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(CALCULATED_METRICS, indent=2), encoding="utf-8") + print(f"Traceability metrics written to: {out_path}") + + +def setup(app: Sphinx) -> dict[str, str | bool]: + app.add_config_value( + "score_metamodel_requirement_types", + "", + rebuild="env", + description=( + "Comma-separated list of need types treated as requirements for " + "traceability metrics. If empty, requirement types are autodiscovered " + "from needs_types tags (requirement, requirement_excl_process)." + ), + ) + + app.add_config_value( + "score_metamodel_include_external_needs", + False, + rebuild="env", + description=( + "When True, include external requirements in dashboard and CI metrics. " + "Default is False so each repo gates only its own needs." + ), + ) + + # _ = app.connect("config-inited", _configure_traceability_dashboard, priority=498) + _ = app.connect("env-updated", calculate_need_metrics, priority=600) + _ = app.connect("build-finished", _write_metrics_json, priority=550) + + return { + "version": "0.1", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/src/extensions/score_metamodel/sphinx_filters.py b/src/extensions/score_metrics/sphinx_filters.py similarity index 66% rename from src/extensions/score_metamodel/sphinx_filters.py rename to src/extensions/score_metrics/sphinx_filters.py index 8fd5fca19..e2fe08a4d 100644 --- a/src/extensions/score_metamodel/sphinx_filters.py +++ b/src/extensions/score_metrics/sphinx_filters.py @@ -38,8 +38,10 @@ def func(needs: list[NeedItem], results: list[int], **kwargs) -> None: ... """ from __future__ import annotations +from typing import Any from sphinx_needs.need_item import NeedItem +from score_metrics.traceability_metrics import CALCULATED_METRICS def _matches_source_selector(need: NeedItem, selector: str) -> bool: @@ -55,7 +57,7 @@ def _matches_source_selector(need: NeedItem, selector: str) -> bool: def generic_pie_linked_items( - needs: list[NeedItem], results: list[int], **kwargs: str | int | float + needs: list[NeedItem], results: list[int], **kwargs: Any ) -> None: """Count target IDs by whether they are linked by selected source needs. @@ -63,6 +65,7 @@ def generic_pie_linked_items( selector prefix, matched against source ``type`` and ``id``), and ``arg3`` (link field name, default ``complies``). """ + results.clear() id_prefix = str(kwargs.get("arg1", "")) source_selector = str(kwargs.get("arg2", "")) link_field = str(kwargs.get("arg3", "complies")) @@ -89,7 +92,7 @@ def generic_pie_linked_items( def generic_pie_items_by_tag( - needs: list[NeedItem], results: list[int], **kwargs: str | int | float + needs: list[NeedItem], results: list[int], **kwargs: Any ) -> None: """Count tagged items split by whether selected source needs link them. @@ -97,6 +100,7 @@ def generic_pie_items_by_tag( matched against source ``type`` and ``id``), and ``arg3`` (link field name, default ``complies``). """ + results.clear() tag = str(kwargs.get("arg1", "")) source_selector = str(kwargs.get("arg2", "")) link_field = str(kwargs.get("arg3", "complies")) @@ -123,7 +127,7 @@ def generic_pie_items_by_tag( def generic_pie_items_in_relationships( - needs: list[NeedItem], results: list[int], **kwargs: str | int | float + needs: list[NeedItem], results: list[int], **kwargs: Any ) -> None: """Count items of a given type by how many container items reference them. @@ -142,6 +146,7 @@ def generic_pie_items_in_relationships( Appends to *results*: ``[not_referenced_count, referenced_once_count, referenced_multiple_count]`` """ + results.clear() container_type = str(kwargs.get("arg1", "")) field = str(kwargs.get("arg2", "")) item_type = str(kwargs.get("arg3", "")) @@ -164,3 +169,73 @@ def generic_pie_items_in_relationships( results.append(not_referenced) results.append(referenced_once) results.append(referenced_multiple) + + +def _get_key_values(results: list[int], argument_paths: list[str]): + """Append integer metric values selected by colon-separated paths. + + Each path is resolved against ``CALCULATED_METRICS`` (for example: + ``"overall_metrics:total"``), converted to ``int``, and appended to + ``results``. + """ + metrics_json = CALCULATED_METRICS + for raw_path in argument_paths: + path = raw_path.strip() + if not path: + continue + + current: Any = metrics_json + for key in path.split(":"): + current = current[key] + + results.append(int(current)) + + +def get_metrics_with_overall_total_considered( + needs: list[Any], results: list[int], **kwargs: Any +) -> None: + """Append selected metrics and compute remainder from overall total. + + This function appends ``overall_metrics:total`` first, then appends all + metrics referenced by ``kwargs`` values. Finally, it replaces the first + appended value with the remainder after subtracting all other appended + values. + """ + results.clear() + metrics_json = CALCULATED_METRICS + results.append(int(metrics_json["overall_metrics"]["total"])) + _get_key_values(results, [str(value) for value in kwargs.values()]) + results[0] -= sum(results[1:]) + print(results) + + +def get_metrics_with_custom_type_total_considered( + needs: list[Any], results: list[int], **kwargs: Any +) -> None: + """Append selected metrics, optionally using a custom total path. + + If the last kwarg value ends with ``":total"``, that path is used as + baseline total and all preceding paths are treated as components; the first + result becomes ``total - sum(components)``. Otherwise, all paths are simply + appended as-is. + """ + # Get the 'total' that was specified as the first value + + results.clear() + values = [str(value) for value in kwargs.values()] + if values[-1].endswith(":total"): + _get_key_values(results, [values[-1]]) # baseline total + _get_key_values(results, values[:-1]) # components + results[0] -= sum(results[1:]) + else: + _get_key_values(results, values) + + +def get_just_metrics(needs: list[Any], results: list[int], **kwargs: Any) -> None: + """Append selected metric values without any total/remainder calculation. + + All kwarg values are interpreted as colon-separated metric paths and + appended to ``results`` in insertion order. + """ + results.clear() + _get_key_values(results, [str(value) for value in kwargs.values()]) diff --git a/src/extensions/score_metrics/tests/test_sphinx_filters.py b/src/extensions/score_metrics/tests/test_sphinx_filters.py new file mode 100644 index 000000000..3b08c3522 --- /dev/null +++ b/src/extensions/score_metrics/tests/test_sphinx_filters.py @@ -0,0 +1,205 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +from typing import cast +import pytest + +# noqa + +import score_metrics.sphinx_filters as sphinx_filters +from score_metrics.sphinx_filters import ( + generic_pie_items_by_tag, + generic_pie_linked_items, +) +from sphinx_needs.need_item import NeedItem + + +def test_generic_pie_linked_items_matches_source_by_id_prefix() -> None: + needs = cast( + list[NeedItem], + [ + {"id": "std_req__iso26262__001", "type": "std_req"}, + # Type intentionally does not match selector prefix, id does. + { + "id": "gd_guidl__xyz", + "type": "guideline", + "complies": ["std_req__iso26262__001"], + }, + ], + ) + + results: list[int] = [] + generic_pie_linked_items( + needs, results, arg1="std_req__iso26262__", arg2="gd_", arg3="complies" + ) + + assert results == [1, 0] + + +def test_generic_pie_items_by_tag_matches_source_by_id_prefix() -> None: + needs = cast( + list[NeedItem], + [ + {"id": "REQ_A", "type": "tool_req", "tags": ["aspice40_man5"]}, + {"id": "REQ_B", "type": "tool_req", "tags": ["aspice40_man5"]}, + # Type intentionally does not match selector prefix, id does. + { + "id": "gd_req__abc", + "type": "process_requirement", + "complies": ["REQ_A"], + }, + ], + ) + + results: list[int] = [] + generic_pie_items_by_tag( + needs, results, arg1="aspice40_man5", arg2="gd_", arg3="complies" + ) + + assert results == [1, 1] + + +EXAMPLE_METRICS = { + "schema_version": "1", + "generated_by": "sphinx_build", + "overall_metrics": { + "total": 61, + "with_code_link": 46, + "with_test_link": 3, + "fully_linked": 2, + "with_code_link_pct": 75.40983606557377, + "with_test_link_pct": 4.918032786885246, + "fully_linked_pct": 3.278688524590164, + }, + "metrics_by_type": { + "tool_req": { + "total": 61, + "with_code_link": 46, + "with_test_link": 3, + "fully_linked": 2, + } + }, + "tests": { + "total": 208, + "linked_to_requirements": 16, + "linked_to_requirements_pct": 7.6923076923076925, + "broken_references": [], + }, +} + + +@pytest.fixture(autouse=True) +def reset_global_metrics(): + """Reset global CALCULATED_METRICS before and after each test.""" + sphinx_filters.CALCULATED_METRICS = {} + yield + sphinx_filters.CALCULATED_METRICS = {} + + +def test_get_key_values_raises_key_error_when_global_is_empty() -> None: + """It raises KeyError if CALCULATED_METRICS is still empty.""" + results: list[int] = [] + with pytest.raises(KeyError): + sphinx_filters._get_key_values(results, ["overall_metrics:total"]) + + +def test_get_key_values_appends_values_when_metrics_loaded() -> None: + """It appends resolved integer values once metrics data is loaded.""" + sphinx_filters.CALCULATED_METRICS = EXAMPLE_METRICS + results: list[int] = [] + + sphinx_filters._get_key_values( + results, + [ + "overall_metrics:total", + "overall_metrics:with_code_link", + "tests:linked_to_requirements", + ], + ) + + assert results == [61, 46, 16] + + +def test_get_metrics_with_overall_total_considered_when_metrics_loaded() -> None: + """It computes remainder from overall total and selected metrics.""" + sphinx_filters.CALCULATED_METRICS = EXAMPLE_METRICS + results: list[int] = [] + + sphinx_filters.get_metrics_with_overall_total_considered( + needs=[], + results=results, + code="overall_metrics:with_code_link", + test="overall_metrics:with_test_link", + fully="overall_metrics:fully_linked", + ) + + assert results == [10, 46, 3, 2] + + +def test_get_metrics_with_custom_type_total_considered_with_total_suffix() -> None: + """It uses trailing ':total' as baseline and computes remainder.""" + sphinx_filters.CALCULATED_METRICS = EXAMPLE_METRICS + results: list[int] = [] + + sphinx_filters.get_metrics_with_custom_type_total_considered( + needs=[], + results=results, + code="metrics_by_type:tool_req:with_code_link", + test="metrics_by_type:tool_req:with_test_link", + total="metrics_by_type:tool_req:total", + ) + + assert results == [12, 46, 3] + + +def test_get_metrics_with_custom_type_total_considered_without_total_suffix() -> None: + """It appends values directly when trailing ':total' is not provided.""" + sphinx_filters.CALCULATED_METRICS = EXAMPLE_METRICS + results: list[int] = [] + + sphinx_filters.get_metrics_with_custom_type_total_considered( + needs=[], + results=results, + code="metrics_by_type:tool_req:with_code_link", + test="metrics_by_type:tool_req:with_test_link", + ) + + assert results == [46, 3] + + +def test_get_just_metrics_appends_values_when_metrics_loaded() -> None: + """It appends selected values without remainder logic.""" + sphinx_filters.CALCULATED_METRICS = EXAMPLE_METRICS + results: list[int] = [] + + sphinx_filters.get_just_metrics( + needs=[], + results=results, + total="overall_metrics:total", + linked_tests="tests:linked_to_requirements", + ) + + assert results == [61, 16] + + +def test_get_metrics_with_custom_type_total_considered_empty_kwargs_raises_index_error() -> ( + None +): + """Current behavior: empty kwargs raises IndexError.""" + sphinx_filters.CALCULATED_METRICS = EXAMPLE_METRICS + results: list[int] = [] + + with pytest.raises(IndexError): + sphinx_filters.get_metrics_with_custom_type_total_considered( + needs=[], results=results + ) diff --git a/src/extensions/score_metrics/tests/test_traceability_dashboard.py b/src/extensions/score_metrics/tests/test_traceability_dashboard.py new file mode 100644 index 000000000..adb3a47b2 --- /dev/null +++ b/src/extensions/score_metrics/tests/test_traceability_dashboard.py @@ -0,0 +1,214 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Tests that dashboard filters follow local/external settings.""" + + +# from src.extensions.score_metamodel.checks import traceability_dashboard +# from src.extensions.score_metrics.traceability_dashboard import ( +# pie_process_requirements_linked, +# pie_requirements_fully_linked, +# pie_requirements_with_code_links, +# pie_requirements_with_test_links, +# set_default_include_external, +# ) +# from src.extensions.score_metrics.traceability_metrics import ( +# compute_traceability_summary, +# ) +# +# +# def _needs() -> list[dict[str, object]]: +# return [ +# { +# "id": "LOCAL_REQ", +# "type": "tool_req", +# "implemented": "YES", +# "source_code_link": "", +# "testlink": "", +# "is_external": False, +# }, +# { +# "id": "LOCAL_SYS_REQ", +# "type": "sys_req", +# "implemented": "YES", +# "source_code_link": "", +# "testlink": "T_LOCAL", +# "is_external": False, +# }, +# { +# "id": "EXT_REQ", +# "type": "tool_req", +# "implemented": "YES", +# "source_code_link": "src/ext.py:10", +# "testlink": "T_EXT", +# "is_external": True, +# }, +# ] +# +# +# def test_dashboard_defaults_to_local_only() -> None: +# set_default_include_external(False) +# +# results: list[int] = [] +# pie_requirements_with_code_links(_needs(), results, arg1="tool_req") +# +# summary = compute_traceability_summary( +# all_needs=_needs(), +# requirement_types={"tool_req"}, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=False, +# ) +# +# assert results == [1, 0] +# assert results == [ +# summary["requirements"]["total"] - summary["requirements"]["with_code_link"], +# summary["requirements"]["with_code_link"], +# ] +# +# +# def test_dashboard_can_include_external_via_default_flag() -> None: +# set_default_include_external(True) +# +# results: list[int] = [] +# pie_requirements_with_code_links(_needs(), results, arg1="tool_req") +# +# summary = compute_traceability_summary( +# all_needs=_needs(), +# requirement_types={"tool_req"}, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=True, +# ) +# +# assert results == [1, 1] +# assert results == [ +# summary["requirements"]["total"] - summary["requirements"]["with_code_link"], +# summary["requirements"]["with_code_link"], +# ] +# +# +# def test_dashboard_filter_arg_can_override_default() -> None: +# set_default_include_external(True) +# +# results: list[int] = [] +# pie_requirements_with_code_links(_needs(), results, arg1="tool_req", arg2="false") +# +# assert results == [1, 0] +# +# +# def test_requirements_with_test_links_default_local_only() -> None: +# set_default_include_external(False) +# +# results: list[int] = [] +# pie_requirements_with_test_links(_needs(), results, arg1="tool_req") +# +# summary = compute_traceability_summary( +# all_needs=_needs(), +# requirement_types={"tool_req"}, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=False, +# ) +# +# assert results == [1, 0] +# assert results == [ +# summary["requirements"]["total"] - summary["requirements"]["with_test_link"], +# summary["requirements"]["with_test_link"], +# ] +# +# +# def test_requirements_with_test_links_can_override_include_external() -> None: +# set_default_include_external(False) +# +# results: list[int] = [] +# pie_requirements_with_test_links(_needs(), results, arg1="tool_req", arg2="true") +# +# assert results == [1, 1] +# +# +# def test_requirements_with_test_links_parses_multiple_types() -> None: +# set_default_include_external(False) +# +# results: list[int] = [] +# pie_requirements_with_test_links(_needs(), results, arg1="tool_req,sys_req") +# +# assert results == [1, 1] +# +# +# def test_requirements_fully_linked_uses_shared_summary() -> None: +# set_default_include_external(False) +# +# results: list[int] = [] +# pie_requirements_fully_linked(_needs(), results, arg1="tool_req") +# +# summary = compute_traceability_summary( +# all_needs=_needs(), +# requirement_types={"tool_req"}, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=False, +# ) +# +# assert results == [1, 0] +# assert results == [ +# summary["requirements"]["total"] - summary["requirements"]["fully_linked"], +# summary["requirements"]["fully_linked"], +# ] +# +# +# def test_requirements_fully_linked_can_include_external() -> None: +# set_default_include_external(True) +# +# results: list[int] = [] +# pie_requirements_fully_linked(_needs(), results, arg1="tool_req") +# +# assert results == [1, 1] +# +# +# def test_process_requirements_linked_uses_stream_a_process_requirement_totals( +# monkeypatch: pytest.MonkeyPatch, +# ) -> None: +# captured: dict[str, object] = {} +# +# def _fake_summary( +# all_needs: Sequence[dict[str, Any]], +# requirement_types: set[str], +# include_not_implemented: bool, +# filtered_test_types: set[str], +# include_external: bool, +# ) -> dict[str, dict[str, int]]: +# captured["all_needs"] = all_needs +# captured["requirement_types"] = requirement_types +# captured["include_not_implemented"] = include_not_implemented +# captured["filtered_test_types"] = filtered_test_types +# captured["include_external"] = include_external +# return { +# "requirements": {"total": 99, "linked": 0}, +# "process_requirements": {"total": 4, "linked": 3}, +# } +# +# monkeypatch.setattr( +# traceability_dashboard, "compute_traceability_summary", _fake_summary +# ) +# +# results: list[int] = [] +# pie_process_requirements_linked( +# _needs(), results, arg1="tool_req,sys_req", arg2="true" +# ) +# +# assert results == [1, 3] +# assert captured["requirement_types"] == {"tool_req", "sys_req"} +# assert captured["include_not_implemented"] is True +# assert captured["filtered_test_types"] == set() +# assert captured["include_external"] is True diff --git a/src/extensions/score_metrics/tests/test_traceability_metrics.py b/src/extensions/score_metrics/tests/test_traceability_metrics.py new file mode 100644 index 000000000..e9b4cb4d7 --- /dev/null +++ b/src/extensions/score_metrics/tests/test_traceability_metrics.py @@ -0,0 +1,201 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Unit tests for traceability_metrics include_external handling.""" + + +def _needs() -> list[dict[str, object]]: + return [ + { + "id": "LOCAL_REQ", + "type": "tool_req", + "implemented": "YES", + "source_code_link": "src/local.py:1", + "testlink": "tests/test_local.py::test_ok", + "is_external": False, + }, + { + "id": "EXT_REQ", + "type": "tool_req", + "implemented": "YES", + "source_code_link": "src/external.py:9", + "testlink": "tests/test_external.py::test_ok", + "is_external": True, + }, + { + "id": "TC_1", + "type": "testcase", + "partially_verifies": "LOCAL_REQ", + "fully_verifies": "", + "is_external": False, + }, + ] + + +# def test_filter_requirements_defaults_to_local_only() -> None: +# filtered = filter_requirements( +# _needs(), +# requirement_types={"tool_req"}, +# include_not_implemented=True, +# ) +# +# assert [need["id"] for need in filtered] == ["LOCAL_REQ"] +# +# +# def test_filter_requirements_can_include_external_needs() -> None: +# filtered = filter_requirements( +# _needs(), +# requirement_types={"tool_req"}, +# include_not_implemented=True, +# include_external=True, +# ) +# +# assert sorted(need["id"] for need in filtered) == ["EXT_REQ", "LOCAL_REQ"] +# +# +# def test_compute_traceability_summary_propagates_include_external() -> None: +# summary_local = compute_traceability_summary( +# all_needs=_needs(), +# requirement_types={"tool_req"}, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=False, +# ) +# summary_all = compute_traceability_summary( +# all_needs=_needs(), +# requirement_types={"tool_req"}, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=True, +# ) +# +# assert summary_local["include_external"] is False +# assert summary_local["requirements"]["total"] == 1 +# assert summary_all["include_external"] is True +# assert summary_all["requirements"]["total"] == 2 +# +# +# def test_compute_traceability_summary_process_requirements_summary() -> None: +# summary = compute_traceability_summary( +# all_needs=[ +# { +# "id": "TOOL_REQ_1", +# "type": "tool_req", +# "implemented": "YES", +# "source_code_link": "src/req.py:10", +# "testlink": "tests/test_req.py::test_ok", +# "satisfies": "PR_LOCAL_1,OTHER_REQ", +# "is_external": False, +# }, +# { +# "id": "TOOL_REQ_2", +# "type": "tool_req", +# "implemented": "YES", +# "source_code_link": "src/req.py:20", +# "testlink": "tests/test_req.py::test_ok_2", +# "satisfies": ["PR_LOCAL_1", "PR_LOCAL_2"], +# "is_external": False, +# }, +# { +# "id": "PR_LOCAL_1", +# "type": "process_req", +# "is_external": False, +# }, +# { +# "id": "PR_LOCAL_2", +# "type": "gd_req", +# "is_external": False, +# }, +# { +# "id": "PR_LOCAL_3", +# "type": "gd_req", +# "is_external": False, +# }, +# ], +# requirement_types={"tool_req"}, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=False, +# ) +# +# process_requirements = summary["process_requirements"] +# +# assert process_requirements["total"] == 3 +# assert process_requirements["linked_by_tool_requirements"] == 2 +# assert process_requirements["linked_by_tool_requirements_pct"] == (2 / 3) * 100 +# assert process_requirements["unlinked_ids"] == ["PR_LOCAL_3"] +# +# +# def test_compute_traceability_summary_process_requirements_respects_include_external() -> ( +# None +# ): +# all_needs = [ +# { +# "id": "TOOL_REQ_LOCAL", +# "type": "tool_req", +# "implemented": "YES", +# "source_code_link": "src/local.py:1", +# "testlink": "tests/test_local.py::test_ok", +# "satisfies": "PR_LOCAL", +# "is_external": False, +# }, +# { +# "id": "TOOL_REQ_EXTERNAL", +# "type": "tool_req", +# "implemented": "YES", +# "source_code_link": "src/external.py:1", +# "testlink": "tests/test_external.py::test_ok", +# "satisfies": "PR_EXTERNAL", +# "is_external": True, +# }, +# { +# "id": "PR_LOCAL", +# "type": "gd_req", +# "is_external": False, +# }, +# { +# "id": "PR_EXTERNAL", +# "type": "gd_req", +# "is_external": True, +# }, +# ] +# +# summary_local = compute_traceability_summary( +# all_needs=all_needs, +# requirement_types={"tool_req"}, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=False, +# ) +# summary_all = compute_traceability_summary( +# all_needs=all_needs, +# requirement_types={"tool_req"}, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=True, +# ) +# +# assert summary_local["process_requirements"] == { +# "total": 1, +# "linked": 1, +# "linked_by_tool_requirements": 1, +# "linked_by_tool_requirements_pct": 100.0, +# "unlinked_ids": [], +# } +# assert summary_all["process_requirements"] == { +# "total": 2, +# "linked": 2, +# "linked_by_tool_requirements": 2, +# "linked_by_tool_requirements_pct": 100.0, +# "unlinked_ids": [], +# } diff --git a/src/extensions/score_metrics/tests/test_traceability_metrics_json_generation.py b/src/extensions/score_metrics/tests/test_traceability_metrics_json_generation.py new file mode 100644 index 000000000..2e2e355de --- /dev/null +++ b/src/extensions/score_metrics/tests/test_traceability_metrics_json_generation.py @@ -0,0 +1,337 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Tests for Sphinx-side metrics.json generation defaults.""" + +import json +from pathlib import Path +from types import SimpleNamespace +from typing import cast + +import pytest +from sphinx.application import Sphinx + +# import src.extensions.score_metamodel.__init__ as metamodel_init +# +# +# class _FakeNeedsData: +# def __init__(self, env: object): +# self._env = env +# +# def get_needs_view(self) -> dict[str, dict[str, object]]: +# return { +# "LOCAL_REQ": { +# "id": "LOCAL_REQ", +# "type": "tool_req", +# "implemented": "YES", +# "source_code_link": "", +# "testlink": "", +# "is_external": False, +# }, +# "LOCAL_FEAT": { +# "id": "LOCAL_FEAT", +# "type": "feat_req", +# "implemented": "YES", +# "source_code_link": "", +# "testlink": "", +# "is_external": False, +# }, +# "EXT_REQ": { +# "id": "EXT_REQ", +# "type": "tool_req", +# "implemented": "NO", +# "source_code_link": "src/ext.py:1", +# "testlink": "", +# "is_external": True, +# }, +# "EXT_FEAT": { +# "id": "EXT_FEAT", +# "type": "feat_req", +# "implemented": "NO", +# "source_code_link": "src/ext_feat.py:1", +# "testlink": "", +# "is_external": True, +# }, +# "EXT_GD": { +# "id": "EXT_GD", +# "type": "gd_req", +# "implemented": "NO", +# "source_code_link": "src/ext_gd.py:1", +# "testlink": "", +# "is_external": True, +# }, +# } +# +# +# class _FakeNonReqNeedsData: +# def __init__(self, env: object): +# self._env = env +# +# def get_needs_view(self) -> dict[str, dict[str, object]]: +# return { +# "LOCAL_COMP": { +# "id": "LOCAL_COMP", +# "type": "comp", +# "implemented": "YES", +# "source_code_link": "", +# "testlink": "", +# "is_external": False, +# }, +# "LOCAL_DOC": { +# "id": "LOCAL_DOC", +# "type": "document", +# "implemented": "YES", +# "source_code_link": "", +# "testlink": "", +# "is_external": False, +# }, +# } +# +# +# def _app( +# tmp_path: Path, +# include_external: bool, +# requirement_types: str = "", +# needs_types: list[dict[str, object]] | None = None, +# ) -> SimpleNamespace: +# discovered_types = needs_types or [ +# {"directive": "tool_req", "tags": ["requirement_excl_process"]}, +# {"directive": "feat_req", "tags": ["requirement"]}, +# {"directive": "workflow", "tags": []}, +# ] +# return SimpleNamespace( +# env=object(), +# outdir=str(tmp_path), +# config=SimpleNamespace( +# score_metamodel_requirement_types=requirement_types, +# score_metamodel_include_external_needs=include_external, +# needs_types=discovered_types, +# ), +# ) +# +# +# def test_write_metrics_json_defaults_to_local_only( +# monkeypatch: pytest.MonkeyPatch, tmp_path: Path +# ) -> None: +# monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) +# +# metamodel_init._write_metrics_json( +# cast(Sphinx, _app(tmp_path, include_external=False)), +# None, +# ) +# +# payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) +# metrics = payload["metrics_by_type"]["tool_req"] +# +# assert payload["schema_version"] == "1" +# assert metrics["include_not_implemented"] is True +# assert metrics["include_external"] is False +# assert metrics["requirements"]["total"] == 1 +# +# +# def test_write_metrics_json_can_include_external( +# monkeypatch: pytest.MonkeyPatch, tmp_path: Path +# ) -> None: +# monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) +# +# metamodel_init._write_metrics_json( +# cast(Sphinx, _app(tmp_path, include_external=True)), +# None, +# ) +# +# payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) +# metrics = payload["metrics_by_type"]["tool_req"] +# +# assert metrics["include_external"] is True +# assert metrics["requirements"]["total"] == 2 +# +# +# def test_explicit_requirement_types_disable_autodiscovery( +# monkeypatch: pytest.MonkeyPatch, tmp_path: Path +# ) -> None: +# monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) +# +# metamodel_init._write_metrics_json( +# cast( +# Sphinx, +# _app( +# tmp_path, +# include_external=False, +# requirement_types="tool_req", +# ), +# ), +# None, +# ) +# +# payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) +# assert set(payload["metrics_by_type"].keys()) == {"tool_req"} +# +# +# def test_write_metrics_json_autodiscovers_when_types_unset( +# monkeypatch: pytest.MonkeyPatch, tmp_path: Path +# ) -> None: +# monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) +# +# metamodel_init._write_metrics_json( +# cast( +# Sphinx, +# _app(tmp_path, include_external=False, requirement_types=""), +# ), +# None, +# ) +# +# payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) +# assert payload["schema_version"] == "1" +# assert set(payload["metrics_by_type"].keys()) == {"feat_req", "tool_req"} +# +# +# def test_autodiscovery_excludes_tagged_types_not_present_in_needs( +# monkeypatch: pytest.MonkeyPatch, tmp_path: Path +# ) -> None: +# monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) +# +# metamodel_init._write_metrics_json( +# cast( +# Sphinx, +# _app( +# tmp_path, +# include_external=False, +# requirement_types="", +# needs_types=[ +# {"directive": "tool_req", "tags": ["requirement_excl_process"]}, +# {"directive": "feat_req", "tags": ["requirement"]}, +# {"directive": "aou_req", "tags": ["requirement"]}, +# ], +# ), +# ), +# None, +# ) +# +# payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) +# assert set(payload["metrics_by_type"].keys()) == {"feat_req", "tool_req"} +# +# +# def test_write_metrics_json_empty_when_no_types_configured_or_discovered( +# monkeypatch: pytest.MonkeyPatch, tmp_path: Path +# ) -> None: +# monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNonReqNeedsData) +# +# metamodel_init._write_metrics_json( +# cast( +# Sphinx, +# _app( +# tmp_path, +# include_external=False, +# requirement_types="", +# needs_types=[{"directive": "workflow", "tags": []}], +# ), +# ), +# None, +# ) +# +# payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) +# assert payload["schema_version"] == "1" +# assert payload["metrics_by_type"] == {} +# +# +# def test_autodiscovery_without_tagged_requirement_types_is_empty( +# monkeypatch: pytest.MonkeyPatch, tmp_path: Path +# ) -> None: +# monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) +# +# metamodel_init._write_metrics_json( +# cast( +# Sphinx, +# _app( +# tmp_path, +# include_external=False, +# requirement_types="", +# needs_types=[{"directive": "workflow", "tags": []}], +# ), +# ), +# None, +# ) +# +# payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) +# assert payload["metrics_by_type"] == {} +# +# +# def test_autodiscovery_respects_include_external_scope( +# monkeypatch: pytest.MonkeyPatch, tmp_path: Path +# ) -> None: +# monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) +# +# metamodel_init._write_metrics_json( +# cast( +# Sphinx, +# _app( +# tmp_path, +# include_external=True, +# requirement_types="", +# needs_types=[ +# {"directive": "tool_req", "tags": ["requirement_excl_process"]}, +# {"directive": "feat_req", "tags": ["requirement"]}, +# {"directive": "gd_req", "tags": ["requirement"]}, +# ], +# ), +# ), +# None, +# ) +# +# payload = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8")) +# assert set(payload["metrics_by_type"].keys()) == {"feat_req", "gd_req", "tool_req"} +# +# +# @pytest.mark.parametrize( +# ("requirement_types", "include_external", "should_exist", "expected_totals"), +# [ +# ("tool_req", False, True, {"tool_req": 1}), +# ("feat_req,tool_req", False, True, {"feat_req": 1, "tool_req": 1}), +# ("", False, True, {"feat_req": 1, "tool_req": 1}), +# (" ", False, True, {"feat_req": 1, "tool_req": 1}), +# ("tool_req", True, True, {"tool_req": 2}), +# ], +# ) +# def test_write_metrics_json_settings_matrix( +# monkeypatch: pytest.MonkeyPatch, +# tmp_path: Path, +# requirement_types: str, +# include_external: bool, +# should_exist: bool, +# expected_totals: dict[str, int], +# ) -> None: +# monkeypatch.setattr(metamodel_init, "SphinxNeedsData", _FakeNeedsData) +# +# metamodel_init._write_metrics_json( +# cast( +# Sphinx, +# _app( +# tmp_path, +# include_external=include_external, +# requirement_types=requirement_types, +# ), +# ), +# None, +# ) +# +# metrics_file = tmp_path / "metrics.json" +# assert metrics_file.exists() is should_exist +# if not should_exist: +# return +# +# payload = json.loads(metrics_file.read_text(encoding="utf-8")) +# by_type = payload["metrics_by_type"] +# assert set(by_type.keys()) == set(expected_totals.keys()) +# +# for req_type, expected_total in expected_totals.items(): +# assert by_type[req_type]["requirements"]["total"] == expected_total diff --git a/src/extensions/score_metrics/traceability_dashboard.py b/src/extensions/score_metrics/traceability_dashboard.py new file mode 100644 index 000000000..a144a72ac --- /dev/null +++ b/src/extensions/score_metrics/traceability_dashboard.py @@ -0,0 +1,184 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +# ╓ ╖ +# ║ Some portions generated by Github Copilot ║ +# ╙ ╜ + +"""Needpie filter functions backed by shared traceability metric calculations.""" + +from __future__ import annotations + +from typing import Any + +from score_metrics.traceability_metrics import CALCULATED_METRICS + +_DEFAULT_INCLUDE_EXTERNAL = False + +# ╓ ╖ +# ║ Docstrings of functions generated by Copilot ║ +# ╙ ╜ + + + + +# def set_default_include_external(include_external: bool) -> None: +# """Configure default behaviour for including external requirements.""" +# global _DEFAULT_INCLUDE_EXTERNAL +# _DEFAULT_INCLUDE_EXTERNAL = bool(include_external) + + +# def _include_external(kwargs: dict[str, str | int | float]) -> bool: +# """Read include_external override from filter args, else use configured default.""" +# raw = kwargs.get("arg2") +# if raw is None: +# return _DEFAULT_INCLUDE_EXTERNAL +# text = str(raw).strip().lower() +# return text in {"1", "true", "yes", "on"} +# +# +# def _requirement_types(kwargs: dict[str, str | int | float]) -> set[str]: +# raw = str(kwargs.get("arg1", "tool_req")).strip() +# values = {value.strip() for value in raw.split(",") if value.strip()} +# return values or {"tool_req"} +# +# +# def pie_requirements_status( +# needs: Sequence[NeedItem | dict[str, Any]], +# results: list[int], +# **kwargs: str | int | float, +# ) -> None: +# """Dashboard status split: not implemented, implemented/incomplete, fully linked.""" +# req_types = _requirement_types(kwargs) +# include_external = _include_external(kwargs) +# +# all_requirements = filter_requirements( +# needs, +# requirement_types=req_types, +# include_not_implemented=True, +# include_external=include_external, +# ) +# implemented_requirements = filter_requirements( +# needs, +# requirement_types=req_types, +# include_not_implemented=False, +# include_external=include_external, +# ) +# summary = compute_traceability_summary( +# all_needs=needs, +# requirement_types=req_types, +# include_not_implemented=False, +# filtered_test_types=set(), +# include_external=include_external, +# ) +# +# not_implemented = len(all_requirements) - len(implemented_requirements) +# fully_linked = int(summary["requirements"]["fully_linked"]) +# implemented_incomplete = len(implemented_requirements) - fully_linked +# +# results.append(not_implemented) +# results.append(implemented_incomplete) +# results.append(fully_linked) +# +# +# def pie_requirements_with_code_links( +# needs: Sequence[NeedItem | dict[str, Any]], +# results: list[int], +# **kwargs: str | int | float, +# ) -> None: +# """Dashboard split: requirements with and without source code links.""" +# req_types = _requirement_types(kwargs) +# include_external = _include_external(kwargs) +# summary = compute_traceability_summary( +# all_needs=needs, +# requirement_types=req_types, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=include_external, +# ) +# +# total = int(summary["requirements"]["total"]) +# with_code = int(summary["requirements"]["with_code_link"]) +# +# results.append(total - with_code) +# results.append(with_code) +# +# +# def pie_requirements_with_test_links( +# needs: Sequence[NeedItem | dict[str, Any]], +# results: list[int], +# **kwargs: str | int | float, +# ) -> None: +# """Dashboard split: requirements with and without testcase links.""" +# req_types = _requirement_types(kwargs) +# include_external = _include_external(kwargs) +# summary = compute_traceability_summary( +# all_needs=needs, +# requirement_types=req_types, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=include_external, +# ) +# +# total = int(summary["requirements"]["total"]) +# with_test = int(summary["requirements"]["with_test_link"]) +# +# results.append(total - with_test) +# results.append(with_test) +# +# +# def pie_requirements_fully_linked( +# needs: Sequence[NeedItem | dict[str, Any]], +# results: list[int], +# **kwargs: str | int | float, +# ) -> None: +# """Dashboard split: requirements fully linked vs incomplete.""" +# req_types = _requirement_types(kwargs) +# include_external = _include_external(kwargs) +# summary = compute_traceability_summary( +# all_needs=needs, +# requirement_types=req_types, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=include_external, +# ) +# +# total = int(summary["requirements"]["total"]) +# fully_linked = int(summary["requirements"]["fully_linked"]) +# +# results.append(total - fully_linked) +# results.append(fully_linked) +# +# +# def pie_process_requirements_linked( +# needs: Sequence[NeedItem | dict[str, Any]], +# results: list[int], +# **kwargs: str | int | float, +# ) -> None: +# """Dashboard split: process requirements linked vs not linked.""" +# req_types = _requirement_types(kwargs) +# include_external = _include_external(kwargs) +# summary = compute_traceability_summary( +# all_needs=needs, +# requirement_types=req_types, +# include_not_implemented=True, +# filtered_test_types=set(), +# include_external=include_external, +# ) +# +# process_requirements = summary["process_requirements"] +# total = int(process_requirements["total"]) +# linked = int(process_requirements["linked"]) +# +# results.append(total - linked) +# results.append(linked) diff --git a/src/extensions/score_metrics/traceability_metrics.py b/src/extensions/score_metrics/traceability_metrics.py new file mode 100644 index 000000000..27ccb894a --- /dev/null +++ b/src/extensions/score_metrics/traceability_metrics.py @@ -0,0 +1,206 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +# ╓ ╖ +# ║ Some portions generated by Github Copilot ║ +# ╙ ╜ + +"""Shared traceability metric calculations for CI checks and dashboards.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from score_metamodel import ScoreNeedType +from score_metamodel.yaml_parser import load_metamodel_data +from sphinx.application import Sphinx +from sphinx_needs.data import NeedsView, SphinxNeedsData +from sphinx_needs.need_item import NeedItem + +CALCULATED_METRICS = {} + + +def get_need_types_by_tags(needs: list[ScoreNeedType], tags: set[str]) -> list[str]: + """ + Takes a list of 'ScoreNeedTypes' and filters out any that have one or more of the + specified tags given. + """ + found_need_types: list[str] = [] + for need_type in needs: + found_tag_set = set(need_type["tags"]) + if tags.intersection(found_tag_set): + found_need_types.append(need_type["directive"]) + return found_need_types + + +def is_non_empty(value: Any) -> bool: + """Return True if value should be treated as present for traceability checks.""" + if isinstance(value, str): + return bool(value.strip()) + return bool(value) + + +def safe_percent(numerator: int, denominator: int) -> float: + """Return percentage in range [0, 100], treating empty denominator as 100%.""" + if denominator == 0: + return 100.0 + return (numerator / denominator) * 100.0 + + +def calculate_requirement_metrics( + current_requirement_needs: list[NeedItem], +) -> dict[str, Any]: + """Calculate requirement traceability statistics for links and completeness.""" + total = len(current_requirement_needs) + reqs_with_code_link = 0 + reqs_with_test_link = 0 + reqs_fully_linked = 0 + missing_code_links_need_ids: set[str] = set() + missing_test_links_need_ids: set[str] = set() + missing_both_links_need_ids: set[str] = set() + + for need in current_requirement_needs: + # Sourcecode link check + if is_non_empty(need.get("source_code_link")): + reqs_with_code_link += 1 + else: + missing_code_links_need_ids.add(need.get("id")) + + # Testlink check + if is_non_empty(need.get("testlink")): + reqs_with_test_link += 1 + else: + missing_test_links_need_ids.add(need.get("id")) + # Negative check (both missing) + if is_non_empty(need.get("testlink")) and is_non_empty( + need.get("source_code_link") + ): + reqs_fully_linked += 1 + + missing_both_links_need_ids = missing_code_links_need_ids.intersection( + missing_test_links_need_ids + ) + + return { + "total": total, + "with_code_link": reqs_with_code_link, + "with_test_link": reqs_with_test_link, + "fully_linked": reqs_fully_linked, + "with_code_link_pct": safe_percent(reqs_with_code_link, total), + "with_test_link_pct": safe_percent(reqs_with_test_link, total), + "fully_linked_pct": safe_percent(reqs_fully_linked, total), + "missing_code_link_ids": sorted(missing_code_links_need_ids), + "missing_test_link_ids": sorted(missing_test_links_need_ids), + "not_fully_linked_ids": sorted(missing_both_links_need_ids), + } + + +def calculate_test_metrics( + test_needs: list[NeedItem], all_needs: NeedsView +) -> dict[str, Any]: + """Calculate testcase linkage and broken testcase-reference statistics.""" + tests_total = len(test_needs) + tests_linked = 0 + broken_references: list[dict[str, str]] = [] + + for test_need in test_needs: + test_id = str(test_need.get("id")) + partially: list[str] = test_need.get("partially_verifies") + fully: list[str] = test_need.get("fully_verifies") + refs = set(partially + fully) + if refs: + tests_linked += 1 + for ref in refs: + if ref not in all_needs: + broken_references.append({"testcase": test_id, "missing_need": ref}) + + return { + "total": tests_total, + "linked_to_requirements": tests_linked, + "linked_to_requirements_pct": safe_percent(tests_linked, tests_total), + "broken_references": broken_references, + } + + +def calculate_full_need_metrics(app: Sphinx, include_external: bool): + """ + Calculate all tracked metrics for requirements and tests. + Will save the result in a global variable 'CALCULATED_METRICS' + """ + # ───────────────[ Getting configuration values ]─────────────── + global CALCULATED_METRICS + all_needs: NeedsView = SphinxNeedsData(app.env).get_needs_view() + + raw_metamodel_path = app.config.score_metamodel_yaml + override_path = Path(raw_metamodel_path) if raw_metamodel_path else None + metamodel = load_metamodel_data(override_path) + + # We either get the types that should be considered from the configuration + # If none are specified the need types declared in the Metamodel with + # the tags 'requirement' are taken + raw = getattr(app.config, "score_metamodel_requirement_types", "").strip() + filter_reqs = [t.strip() for t in raw.split(",") if t.strip()] + if not filter_reqs: + filter_reqs = get_need_types_by_tags(metamodel.needs_types, {"reqiurement"}) + # ──────────────────[ Calculate Test Metrics ]────────────────── + + test_needs = list(all_needs.filter_types(["testcase"]).values()) + test_metrics = calculate_test_metrics(test_needs, all_needs) + + metrics_by_type: dict[str, Any] = {} + + # Metrics accumulated over all requirements types + overall_metrics: dict[str, Any] = { + "total": 0, + "with_code_link": 0, + "with_test_link": 0, + "fully_linked": 0, + } + + # ─────[ Calculating Metrics for each requirement_type ]─── + for req_type in sorted(filter_reqs): + needs_of_req_type = all_needs.filter_types([req_type]).filter_is_external( + include_external + ) + # We do not care if there is no requirements of this type. + if not list(needs_of_req_type.values()): + continue + req_metrics = calculate_requirement_metrics(list(needs_of_req_type.values())) + overall_metrics["total"] += req_metrics["total"] + overall_metrics["with_code_link"] += req_metrics["with_code_link"] + overall_metrics["with_test_link"] += req_metrics["with_test_link"] + overall_metrics["fully_linked"] += req_metrics["fully_linked"] + metrics_by_type[req_type] = req_metrics + # Calculating % of each category for the overall metrics + overall_metrics["with_code_link_pct"] = safe_percent( + overall_metrics["with_code_link"], overall_metrics["total"] + ) + overall_metrics["with_test_link_pct"] = safe_percent( + overall_metrics["with_test_link"], overall_metrics["total"] + ) + overall_metrics["fully_linked_pct"] = safe_percent( + overall_metrics["fully_linked"], overall_metrics["total"] + ) + + output: dict[str, Any] = { + "schema_version": "1", + "generated_by": "sphinx_build", + "overall_metrics": overall_metrics, + "metrics_by_type": metrics_by_type, + "tests": test_metrics, + } + # Save the metrics in a Global Variable to enable access from other parts. + # Not a great solution but it is needed, as needpie filter functions for example + # can not access 'app'. + CALCULATED_METRICS = output diff --git a/src/extensions/score_source_code_linker/tests/test_xml_parser.py b/src/extensions/score_source_code_linker/tests/test_xml_parser.py index c1725d31c..a184e2178 100644 --- a/src/extensions/score_source_code_linker/tests/test_xml_parser.py +++ b/src/extensions/score_source_code_linker/tests/test_xml_parser.py @@ -302,7 +302,9 @@ def test_read_test_xml_file( # No properties at all => Should not be a 'valid' testlink needs2, no_props2, missing_props2 = xml_parser.read_test_xml_file(dir2 / "test.xml") - assert needs2 == [] + assert isinstance(needs2, list) and len(needs2) == 1 + tcneed2 = needs2[0] + assert isinstance(tcneed2, DataOfTestCase) assert no_props2 == ["tc_no_props"] assert missing_props2 == [] @@ -316,7 +318,9 @@ def test_read_test_xml_file( # Missing some properties => Should not be a 'valid' testlink needs4, no_props4, missing_props4 = xml_parser.read_test_xml_file(dir4 / "test.xml") - assert needs4 == [] + assert isinstance(needs4, list) and len(needs4) == 1 + tcneed4 = needs4[0] + assert isinstance(tcneed4, DataOfTestCase) assert no_props4 == [] assert missing_props4 == ["tc_with_missing_props"] diff --git a/src/extensions/score_source_code_linker/xml_parser.py b/src/extensions/score_source_code_linker/xml_parser.py index 302d87469..9a8bcac63 100644 --- a/src/extensions/score_source_code_linker/xml_parser.py +++ b/src/extensions/score_source_code_linker/xml_parser.py @@ -219,9 +219,10 @@ def read_test_xml_file(file: Path) -> tuple[list[DataOfTestCase], list[str], lis properties_element = testcase.find("properties") # HINT: This list is hard coded here, might not be ideal to have that in the # long run. - if properties_element is None: - non_prop_tests.append(testname) - continue + # Even if we have no properties we still want to create test_case needs + # if properties_element is None: + # non_prop_tests.append(testname) + # continue # ╓ ╖ # ║ Disabled Temporarily ║ @@ -236,12 +237,16 @@ def read_test_xml_file(file: Path) -> tuple[list[DataOfTestCase], list[str], lis # I think it should be possible to save the 'from_dict' operation # If the is_valid method would return 'False' anyway. # I just can't think of it right now, leaving this for future me - case_properties = parse_properties(case_properties, properties_element) - case_properties.update(md) - test_case = DataOfTestCase.from_dict(case_properties) - if not test_case.is_valid(): - missing_prop_tests.append(testname) - continue + if properties_element is not None: + case_properties = parse_properties(case_properties, properties_element) + case_properties.update(md) + + test_case = DataOfTestCase.from_dict(case_properties) + if not test_case.is_valid(): + missing_prop_tests.append(testname) + else: + non_prop_tests.append(testname) + test_case = DataOfTestCase.from_dict(case_properties) test_case_needs.append(test_case) return test_case_needs, non_prop_tests, missing_prop_tests @@ -332,23 +337,21 @@ def short_hash(input_str: str, length: int = 5) -> str: def construct_and_add_need(app: Sphinx, tn: DataOfTestCase): - # Asserting worldview to a peace Language Server - # And ensure non crashing due to non string concatenation - # Everything but 'result_text', - # and either 'Fully' or 'PartiallyVerifies' should not be None here - assert tn.file is not None + # We will now allow file to be empty in case of non fully fleshed out testcases + file = tn.file if tn.file is not None else "" assert tn.name is not None + name = tn.name external_url = "" if tn.repo_name is None or tn.hash is None or tn.url is None: logger.info( "Creating testcase need with fallback URL due to incomplete repo metadata: " - f"name={tn.name}, file={tn.file}, repo_name={tn.repo_name}, " + f"name={name}, file={file}, repo_name={tn.repo_name}, " f"hash={tn.hash}, url={tn.url}", type="score_source_code_linker", ) line = tn.line if tn.line is not None else 1 external_url = ( - f"https://github.com/placeholder/placeholder/blob/unknown/{tn.file}#L{line}" + f"https://github.com/placeholder/placeholder/blob/unknown/{file}#L{line}" ) else: # Have to build metadata here for the gh link func @@ -359,10 +362,10 @@ def construct_and_add_need(app: Sphinx, tn: DataOfTestCase): _ = add_external_need( app=app, need_type="testcase", - title=tn.name, + title=name, tags="TEST", - id=f"testcase__{tn.name}_{short_hash(tn.file + tn.name)}", - name=tn.name, + id=f"testcase__{name}_{short_hash(file + name)}", + name=name, external_url=external_url, fully_verifies=tn.FullyVerifies if tn.FullyVerifies is not None else "", partially_verifies=tn.PartiallyVerifies @@ -370,7 +373,7 @@ def construct_and_add_need(app: Sphinx, tn: DataOfTestCase): else "", test_type=tn.TestType, derivation_technique=tn.DerivationTechnique, - file=tn.file, + file=file, line=tn.line, result=tn.result, # We just want the 'failed' or whatever result_text=tn.result_text if tn.result_text else "", diff --git a/src/extensions/score_sphinx_bundle/BUILD b/src/extensions/score_sphinx_bundle/BUILD index 832e35a61..c34d2907e 100644 --- a/src/extensions/score_sphinx_bundle/BUILD +++ b/src/extensions/score_sphinx_bundle/BUILD @@ -29,6 +29,7 @@ py_library( "@score_docs_as_code//src/extensions/score_layout", "@score_docs_as_code//src/extensions/score_metamodel", "@score_docs_as_code//src/extensions/score_source_code_linker", + "@score_docs_as_code//src/extensions/score_metrics", "@score_docs_as_code//src/extensions/score_sync_toml", "@score_docs_as_code//src/helper_lib", ], diff --git a/src/extensions/score_sphinx_bundle/__init__.py b/src/extensions/score_sphinx_bundle/__init__.py index 6ae04008b..31d81e91f 100644 --- a/src/extensions/score_sphinx_bundle/__init__.py +++ b/src/extensions/score_sphinx_bundle/__init__.py @@ -31,6 +31,7 @@ "sphinxcontrib.mermaid", "needs_config_writer", "score_sync_toml", + "score_metrics", ]