From 4c5cb1c7575ac21388ce32aa1118ef2f1bccf583 Mon Sep 17 00:00:00 2001 From: Anas Khan <83116240+anxkhn@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:24:07 +0530 Subject: [PATCH] fix: treat a 5xx robots.txt response as complete disallow RobotsTxtFile.load only special-cased 4xx responses (unavailable -> allow all). Because the HTTP clients do not raise on 5xx, a server-error response fell through and its body was parsed as robots rules. An error page such as an HTML 500 is then read by Protego as allow-all, so a transient 5xx on robots.txt silently permitted crawling every URL. Per RFC 9309 section 2.3.1.4, a robots.txt that is unreachable due to server errors (5xx) is undefined and the crawler must assume complete disallow. Handle 5xx explicitly with a disallow-all directive using the existing is_status_code_server_error helper, leaving the 4xx, 2xx/3xx, and network-error paths unchanged. Signed-off-by: Anas Khan <83116240+anxkhn@users.noreply.github.com> --- src/crawlee/_utils/robots.py | 16 ++++++++++------ tests/unit/_utils/test_robots.py | 10 ++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/crawlee/_utils/robots.py b/src/crawlee/_utils/robots.py index 2c0d70b1c0..960f92b61e 100644 --- a/src/crawlee/_utils/robots.py +++ b/src/crawlee/_utils/robots.py @@ -8,7 +8,7 @@ from crawlee._utils.sitemap import Sitemap from crawlee._utils.urls import filter_url -from crawlee._utils.web import is_status_code_client_error +from crawlee._utils.web import is_status_code_client_error, is_status_code_server_error if TYPE_CHECKING: from typing_extensions import Self @@ -57,11 +57,15 @@ async def load(cls, url: str, http_client: HttpClient, proxy_info: ProxyInfo | N try: response = await http_client.send_request(url, proxy_info=proxy_info) - body = ( - b'User-agent: *\nAllow: /' - if is_status_code_client_error(response.status_code) - else await response.read() - ) + status_code = response.status_code + if is_status_code_client_error(status_code): + # A 4xx response means the file is unavailable, so crawling is unrestricted (RFC 9309, section 2.3.1.3). + body = b'User-agent: *\nAllow: /' + elif is_status_code_server_error(status_code): + # A 5xx response means the file is unreachable, so assume complete disallow (RFC 9309, section 2.3.1.4). + body = b'User-agent: *\nDisallow: /' + else: + body = await response.read() robots = Protego.parse(body.decode('utf-8')) except Exception as e: diff --git a/tests/unit/_utils/test_robots.py b/tests/unit/_utils/test_robots.py index 5f4ad28f22..a099e46f80 100644 --- a/tests/unit/_utils/test_robots.py +++ b/tests/unit/_utils/test_robots.py @@ -25,6 +25,16 @@ async def test_allow_disallow_robots_txt(server_url: URL, http_client: HttpClien assert not robots.is_allowed(str(server_url / 'deny_all/page.html')) +async def test_server_error_disallows_all(server_url: URL, http_client: HttpClient) -> None: + """A 5xx robots.txt response is treated as complete disallow (RFC 9309, section 2.3.1.4). + + The response body must not be parsed as rules; otherwise an error page would be misread as allow-all. + """ + robots = await RobotsTxtFile.load(str(server_url / 'status/500'), http_client) + assert not robots.is_allowed(str(server_url / 'admin/page.html')) + assert not robots.is_allowed(str(server_url / 'something/page.html')) + + async def test_extract_sitemaps_urls(server_url: URL, http_client: HttpClient) -> None: """Cross-host sitemap entries are dropped under the `'same-hostname'` enqueue strategy.""" robots = await RobotsTxtFile.find(str(server_url), http_client)