Retry transient GitHub config fetch timeouts

src/sentry_config.py

@@ -15,6 +15,12 @@
 SENTRY_CONFIG_API_URL = (
     "https://api.github.com/repos/{owner}/.sentry/contents/sentry_config.ini"
 )
+GITHUB_REQUEST_TIMEOUT = 5
+GITHUB_REQUEST_ATTEMPTS = 3
+GITHUB_RETRYABLE_EXCEPTIONS = (
+    requests.exceptions.ConnectionError,
+    requests.exceptions.Timeout,
+)
 
 
 def fetch_dsn_for_github_org(org: str, token: str) -> str:
@@ -27,7 +33,7 @@
         api_url = SENTRY_CONFIG_API_URL.replace("{owner}", org)
 
         # - Get meta about sentry_config.ini file
-        resp = requests.get(api_url, headers=headers)
+        resp = _fetch_sentry_config(api_url, headers)
         resp.raise_for_status()
         meta = resp.json()
 
@@ -46,3 +52,20 @@
     except Exception as e:
         logger.exception(e)
         raise e
+
+
+def _fetch_sentry_config(api_url, headers):
+    for attempt in range(1, GITHUB_REQUEST_ATTEMPTS + 1):
+        try:
+            return requests.get(
+                api_url,
+                headers=headers,
+                timeout=GITHUB_REQUEST_TIMEOUT,
+            )
+        except GITHUB_RETRYABLE_EXCEPTIONS as e:
+            if attempt == GITHUB_REQUEST_ATTEMPTS:
+                raise
+            logger.warning(
+                "Retrying GitHub Sentry config fetch after transient error: %s",
+                e,
+            )

tests/test_sentry_config_file.py

@@ -2,8 +2,11 @@
 
 from unittest import TestCase
 
+import pytest
+import requests
 import responses
 
+from src.sentry_config import GITHUB_REQUEST_ATTEMPTS
 from src.sentry_config import fetch_dsn_for_github_org
 from src.sentry_config import SENTRY_CONFIG_API_URL as api_url
 
@@ -47,6 +50,40 @@ def setUp(self) -> None:
     def test_fetch_parse_sentry_config_file(self) -> None:
         assert fetch_dsn_for_github_org(org, token) == expected_dsn
 
+    @responses.activate
+    def test_fetch_retries_transient_timeout(self) -> None:
+        retry_org = "example-org"
+        retry_url = api_url.replace("{owner}", retry_org)
+        responses.add(
+            method="GET",
+            url=retry_url,
+            body=requests.exceptions.ConnectTimeout("connection timed out"),
+        )
+        responses.add(
+            method="GET",
+            url=retry_url,
+            json=sentry_config_file_meta,
+            status=200,
+        )
+
+        assert fetch_dsn_for_github_org(retry_org, token) == expected_dsn
+        assert len(responses.calls) == 2
+
+    @responses.activate
+    def test_fetch_raises_after_retry_exhaustion(self) -> None:
+        retry_org = "example-org"
+        retry_url = api_url.replace("{owner}", retry_org)
+        for _ in range(GITHUB_REQUEST_ATTEMPTS):
+            responses.add(
+                method="GET",
+                url=retry_url,
+                body=requests.exceptions.ConnectTimeout("connection timed out"),
+            )
+
+        with pytest.raises(requests.exceptions.ConnectTimeout):
+            fetch_dsn_for_github_org(retry_org, token)
+        assert len(responses.calls) == GITHUB_REQUEST_ATTEMPTS
+
     def test_fetch_private_repo(self) -> None:
         pass