From a8432352ba82a3fdd8e6e41fa8f31504485fe84e Mon Sep 17 00:00:00 2001 From: Anas Khan <83116240+anxkhn@users.noreply.github.com> Date: Fri, 3 Jul 2026 14:09:25 +0530 Subject: [PATCH 1/3] fix(request): stop Request.from_url mutating caller user_data and writing empty __crawlee Request.from_url took a live reference to the caller's user_data dict and its nested `__crawlee` block, then mutated them in place and replaced the `__crawlee` key with a CrawleeRequestData model. As a result, reusing a user_data dict leaked `enqueueStrategy`/`maxRetries` into the caller's nested dict, and passing the same dict object twice raised `TypeError: 'CrawleeRequestData' object does not support item assignment` because the key was no longer a plain dict. Copy both the outer user_data dict and the nested `__crawlee` block before mutating them so the caller's input is left untouched. Also, the `if crawlee_data:` guard meant to skip writing `__crawlee` when there is no crawlee data was always true, because an empty pydantic model is truthy. This wrote an empty `{'__crawlee': {}}` for requests with no crawlee options. Compare against `CrawleeRequestData()` so the write is skipped when empty. Add regression tests covering input mutation, reusing the same dict, and the empty-guard case. Signed-off-by: Anas Khan <83116240+anxkhn@users.noreply.github.com> --- src/crawlee/_request.py | 6 +++--- tests/unit/test_request.py | 41 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_request.py diff --git a/src/crawlee/_request.py b/src/crawlee/_request.py index 0d9598db8b..820031aaf5 100644 --- a/src/crawlee/_request.py +++ b/src/crawlee/_request.py @@ -310,8 +310,8 @@ def from_url( if always_enqueue: unique_key = f'{crypto_random_object_id()}|{unique_key}' - user_data_dict = kwargs.pop('user_data', {}) or {} - crawlee_data_dict = user_data_dict.get('__crawlee', {}) + user_data_dict = dict(kwargs.pop('user_data', {}) or {}) + crawlee_data_dict = dict(user_data_dict.get('__crawlee') or {}) if max_retries is not None: crawlee_data_dict['maxRetries'] = max_retries @@ -321,7 +321,7 @@ def from_url( crawlee_data = CrawleeRequestData(**crawlee_data_dict) - if crawlee_data: + if crawlee_data != CrawleeRequestData(): user_data_dict['__crawlee'] = crawlee_data request = cls( diff --git a/tests/unit/test_request.py b/tests/unit/test_request.py new file mode 100644 index 0000000000..099086c4ae --- /dev/null +++ b/tests/unit/test_request.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from crawlee import Request + + +def test_from_url_does_not_mutate_caller_user_data() -> None: + user_data = {'__crawlee': {'crawlDepth': 2}} + original_crawlee_data = user_data['__crawlee'] + + Request.from_url('https://example.com', user_data=user_data, enqueue_strategy='same-domain') + + # The caller's dict and its nested `__crawlee` block must be left untouched. + assert user_data['__crawlee'] is original_crawlee_data + assert user_data['__crawlee'] == {'crawlDepth': 2} + + +def test_from_url_can_reuse_same_user_data_dict() -> None: + user_data = {'__crawlee': {'crawlDepth': 2}} + + # Passing the same dict object twice used to raise because the first call replaced the + # nested `__crawlee` value with a model that does not support item assignment. + Request.from_url('https://example.com', user_data=user_data, enqueue_strategy='same-domain') + Request.from_url('https://example.com', user_data=user_data, enqueue_strategy='same-domain') + + +def test_from_url_omits_crawlee_data_when_empty() -> None: + request = Request.from_url('https://example.com') + + # No crawlee-specific options were supplied, so `__crawlee` must not be written to `user_data`. + assert '__crawlee' not in request.model_dump()['user_data'] + + +def test_from_url_keeps_crawlee_data_when_supplied() -> None: + request = Request.from_url('https://example.com', max_retries=3, enqueue_strategy='same-hostname') + + assert request.model_dump()['user_data']['__crawlee'] == { + 'maxRetries': 3, + 'enqueueStrategy': 'same-hostname', + } + assert request.max_retries == 3 + assert request.enqueue_strategy == 'same-hostname' From 2a0f5569e3fc6cedf7416577fea85a9372e2101a Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 15 Jul 2026 13:38:27 +0200 Subject: [PATCH 2/3] refactor(request): skip building CrawleeRequestData when no crawlee data --- src/crawlee/_request.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/crawlee/_request.py b/src/crawlee/_request.py index 820031aaf5..326e8ddce9 100644 --- a/src/crawlee/_request.py +++ b/src/crawlee/_request.py @@ -319,10 +319,8 @@ def from_url( if enqueue_strategy is not None: crawlee_data_dict['enqueueStrategy'] = enqueue_strategy - crawlee_data = CrawleeRequestData(**crawlee_data_dict) - - if crawlee_data != CrawleeRequestData(): - user_data_dict['__crawlee'] = crawlee_data + if crawlee_data_dict: + user_data_dict['__crawlee'] = CrawleeRequestData(**crawlee_data_dict) request = cls( url=url, From dbbba7da4cd9c9a963411441a87376d204f7d5b6 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 15 Jul 2026 14:21:25 +0200 Subject: [PATCH 3/3] test(request): add docstrings to from_url regression tests --- tests/unit/test_request.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_request.py b/tests/unit/test_request.py index 099086c4ae..615c1096b3 100644 --- a/tests/unit/test_request.py +++ b/tests/unit/test_request.py @@ -4,33 +4,34 @@ def test_from_url_does_not_mutate_caller_user_data() -> None: + """Verify `from_url` leaves the caller's `user_data` dict and its nested `__crawlee` block untouched.""" user_data = {'__crawlee': {'crawlDepth': 2}} original_crawlee_data = user_data['__crawlee'] Request.from_url('https://example.com', user_data=user_data, enqueue_strategy='same-domain') - # The caller's dict and its nested `__crawlee` block must be left untouched. assert user_data['__crawlee'] is original_crawlee_data assert user_data['__crawlee'] == {'crawlDepth': 2} def test_from_url_can_reuse_same_user_data_dict() -> None: + """Verify the same `user_data` dict object can be passed to `from_url` twice without raising.""" user_data = {'__crawlee': {'crawlDepth': 2}} - # Passing the same dict object twice used to raise because the first call replaced the - # nested `__crawlee` value with a model that does not support item assignment. + # The first call used to replace the nested `__crawlee` value with a model that rejects item assignment. Request.from_url('https://example.com', user_data=user_data, enqueue_strategy='same-domain') Request.from_url('https://example.com', user_data=user_data, enqueue_strategy='same-domain') def test_from_url_omits_crawlee_data_when_empty() -> None: + """Verify `from_url` does not write a `__crawlee` block when no crawlee-specific options are supplied.""" request = Request.from_url('https://example.com') - # No crawlee-specific options were supplied, so `__crawlee` must not be written to `user_data`. assert '__crawlee' not in request.model_dump()['user_data'] def test_from_url_keeps_crawlee_data_when_supplied() -> None: + """Verify `from_url` serializes supplied `max_retries` and `enqueue_strategy` into the `__crawlee` block.""" request = Request.from_url('https://example.com', max_retries=3, enqueue_strategy='same-hostname') assert request.model_dump()['user_data']['__crawlee'] == {