From a287edb2489a40e287520c279b25f1c552252103 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Thu, 9 Jul 2026 13:55:34 -0300 Subject: [PATCH] fix(httpx): try passing headers to httpx requests to prevent 403 from hetzner --- pysus/api/ducklake/catalog/adapters.py | 17 ++++++++++++++--- pysus/api/ducklake/functional.py | 10 ++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/pysus/api/ducklake/catalog/adapters.py b/pysus/api/ducklake/catalog/adapters.py index 98d3a36d..62edc56e 100644 --- a/pysus/api/ducklake/catalog/adapters.py +++ b/pysus/api/ducklake/catalog/adapters.py @@ -158,7 +158,8 @@ async def _download_catalog( force: bool = False, callback: Callable[[int, int], None] | None = None, ) -> None: - url = f"https://{types.S3_ENDPOINT}/{types.S3_BUCKET}/{remote_path}" + remote = str(remote_path).replace("\\", "/") + url = f"https://{types.S3_ENDPOINT}/{types.S3_BUCKET}/{remote}" if local_path.exists() and not force: try: @@ -170,7 +171,17 @@ async def _download_catalog( remote_size = 0 if local_size != -1: - async with httpx.AsyncClient(follow_redirects=True) as client: + headers = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/120.0.0.0 Safari/537.36" + ), + "Referer": "https://github.com/AlertaDengue/PySUS", + } + async with httpx.AsyncClient( + headers=headers, follow_redirects=True + ) as client: try: head = await client.head(url) @@ -189,7 +200,7 @@ async def _download_catalog( try: await download_http( - remote_path=remote_path, + remote_path=remote, local_path=local_path, callback=callback, ) diff --git a/pysus/api/ducklake/functional.py b/pysus/api/ducklake/functional.py index 3d61b4f1..4666522d 100644 --- a/pysus/api/ducklake/functional.py +++ b/pysus/api/ducklake/functional.py @@ -14,15 +14,25 @@ async def download_http( local_path: Path, callback: Callable[[int, int], None] | None = None, ) -> None: + remote_path = str(remote_path).replace("\\", "/") url = f"https://{types.S3_ENDPOINT}/{types.S3_BUCKET}/{remote_path}" max_retries = 5 timeout = httpx.Timeout(15.0, read=60.0, write=20.0, connect=15.0) limits = httpx.Limits(max_keepalive_connections=5, max_connections=10) + headers = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/120.0.0.0 Safari/537.36" + ), + "Referer": "https://github.com/AlertaDengue/PySUS", + } for attempt in range(max_retries): try: async with httpx.AsyncClient( + headers=headers, follow_redirects=True, verify=False, limits=limits,