diff --git a/src/cache.py b/src/cache.py index 5b67bf8..94d8ea7 100644 --- a/src/cache.py +++ b/src/cache.py @@ -105,7 +105,8 @@ def put( "cached_at": time.time(), } try: - path.write_text(json.dumps(entry)) # lgtm [py/clear-text-storage-sensitive-data] redacted above + # lgtm[py/clear-text-storage-sensitive-data] Credential-shaped data is rejected above. + path.write_text(json.dumps(entry)) except OSError: pass # Cache write failure is non-fatal diff --git a/src/ghas_alerts.py b/src/ghas_alerts.py index 51f7aa7..3e113d4 100644 --- a/src/ghas_alerts.py +++ b/src/ghas_alerts.py @@ -28,6 +28,7 @@ import requests from src.cache import ResponseCache +from src.http_link_header import next_link_from_header logger = logging.getLogger(__name__) @@ -85,11 +86,8 @@ def _paginate( if next_link: next_url = next_link else: - import re link_header = resp.headers.get("Link", "") - match = re.search(r'<([^>]+)>;\s*rel="next"', link_header) - if match: - next_url = match.group(1) + next_url = next_link_from_header(link_header) return results diff --git a/src/github_client.py b/src/github_client.py index dd8fd33..d77931d 100644 --- a/src/github_client.py +++ b/src/github_client.py @@ -2,7 +2,6 @@ import base64 import logging -import re import sys import time from collections.abc import Callable @@ -13,6 +12,7 @@ from urllib3.util import Retry from src.cache import ResponseCache +from src.http_link_header import next_link_from_header from src.models import RepoMetadata logger = logging.getLogger(__name__) @@ -130,12 +130,10 @@ def _paginate(self, url: str, params: dict | None = None) -> list[dict]: # After the first request, params are baked into the next URL params = {} - # Parse next link — prefer response.links, fall back to regex + # Parse next link — prefer requests' parser, then a linear fallback. next_link = response.links.get("next", {}).get("url") if not next_link: - link_header = response.headers.get("Link", "") - match = re.search(r'<([^>]+)>;\s*rel="next"', link_header) - next_link = match.group(1) if match else None + next_link = next_link_from_header(response.headers.get("Link", "")) url = next_link # type: ignore[assignment] return results diff --git a/src/http_link_header.py b/src/http_link_header.py new file mode 100644 index 0000000..851a127 --- /dev/null +++ b/src/http_link_header.py @@ -0,0 +1,27 @@ +from __future__ import annotations + + +def next_link_from_header(link_header: str) -> str | None: + """Return the first ``rel=next`` target from a GitHub Link header. + + ``requests.Response.links`` remains the primary parser. This bounded, + linear fallback accepts GitHub's standard Link shape and fails closed for + malformed entries. + """ + for raw_entry in link_header.split(","): + entry = raw_entry.strip() + if not entry.startswith("<"): + continue + target_end = entry.find(">") + if target_end <= 1: + continue + target = entry[1:target_end] + for raw_parameter in entry[target_end + 1 :].split(";"): + name, separator, value = raw_parameter.partition("=") + if ( + separator + and name.strip().lower() == "rel" + and value.strip().strip("\"'").lower() == "next" + ): + return target + return None diff --git a/src/operator_control_center_artifacts.py b/src/operator_control_center_artifacts.py index c8d4632..e4c7139 100644 --- a/src/operator_control_center_artifacts.py +++ b/src/operator_control_center_artifacts.py @@ -91,6 +91,10 @@ def write_control_center_artifacts( "json_path": str(weekly_json), "markdown_path": str(weekly_md), } - json_path.write_text(json.dumps(payload, indent=2)) # codeql[py/clear-text-storage-sensitive-data] guarded above - md_path.write_text(render_control_center_markdown(snapshot, username, generated_at.isoformat())) # codeql[py/clear-text-storage-sensitive-data] guarded above + # lgtm[py/clear-text-storage-sensitive-data] Credential-shaped data is rejected above. + json_path.write_text(json.dumps(payload, indent=2)) + # lgtm[py/clear-text-storage-sensitive-data] Credential-shaped data is rejected above. + md_path.write_text( + render_control_center_markdown(snapshot, username, generated_at.isoformat()) + ) return json_path, md_path, weekly_json, weekly_md, payload diff --git a/tests/test_github_client.py b/tests/test_github_client.py index a859a1c..85c3419 100644 --- a/tests/test_github_client.py +++ b/tests/test_github_client.py @@ -6,6 +6,7 @@ import requests from src.github_client import REST_API_VERSION, GitHubClient +from src.http_link_header import next_link_from_header class _MemoryCache: @@ -30,6 +31,49 @@ def test_rest_session_sets_explicit_api_version(self): client = GitHubClient() assert client.session.headers["X-GitHub-Api-Version"] == REST_API_VERSION + def test_pagination_uses_linear_link_header_fallback(self, monkeypatch): + client = GitHubClient() + + class _Page: + def __init__(self, payload, link_header=""): + self._payload = payload + self.links = {} + self.headers = {"Link": link_header} + + def json(self): + return self._payload + + pages = iter( + [ + _Page( + [{"page": 1}], + '; rel="next", ' + '; rel="last"', + ), + _Page([{"page": 2}]), + ] + ) + requested = [] + + def _request(url, params=None): + requested.append((url, params)) + return next(pages) + + monkeypatch.setattr(client, "_request", _request) + + assert client._paginate( + "https://api.github.test/items", {"per_page": 100} + ) == [{"page": 1}, {"page": 2}] + assert requested == [ + ("https://api.github.test/items", {"per_page": 100}), + ("https://api.github.test/items?page=2", {}), + ] + + def test_link_header_fallback_fails_closed_on_large_malformed_input(self): + malformed = "<" + "<=" * 100_000 + + assert next_link_from_header(malformed) is None + def test_repo_list_cache_key_includes_owner_private_scope(self, monkeypatch): cache = _MemoryCache() client = GitHubClient(token="secret", cache=cache)