test: Reduce flakiness in tests

tests/unit/browsers/test_browser_pool.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-from datetime import timedelta
 from typing import TYPE_CHECKING
 from unittest.mock import AsyncMock
 
@@ -40,6 +39,7 @@ async def test_default_plugin_new_page_creation(server_url: URL) -> None:
         await page_2.page.close()
 
 
+@pytest.mark.run_alone
 async def test_multiple_plugins_new_page_creation(server_url: URL) -> None:
     plugin_chromium = PlaywrightBrowserPlugin(browser_type='chromium')
     plugin_firefox = PlaywrightBrowserPlugin(browser_type='firefox')
@@ -72,10 +72,7 @@ async def test_multiple_plugins_new_page_creation(server_url: URL) -> None:
         assert browser_pool.total_pages_count == 3
 
 
-@pytest.mark.flaky(
-    reruns=3,
-    reason='Test is flaky on Windows and MacOS, see https://github.com/apify/crawlee-python/issues/1660.',
-)
+@pytest.mark.run_alone
 async def test_new_page_with_each_plugin(server_url: URL) -> None:
     plugin_chromium = PlaywrightBrowserPlugin(browser_type='chromium')
     plugin_firefox = PlaywrightBrowserPlugin(browser_type='firefox')
@@ -108,7 +105,8 @@ async def test_with_default_plugin_constructor(server_url: URL) -> None:
     # even with a generous operation timeout when several workers compete for the runner). Run it alone and
     # keep a generous operation timeout so that Firefox has enough time to launch on slow CI.
     async with BrowserPool.with_default_plugin(
-        headless=True, browser_type='firefox', operation_timeout=timedelta(seconds=60)
+        headless=True,
+        browser_type='firefox',
     ) as browser_pool:
         assert len(browser_pool.plugins) == 1
         assert isinstance(browser_pool.plugins[0], PlaywrightBrowserPlugin)

tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py

@@ -45,6 +45,9 @@
 _H2_TEXT = 'Only in browser'
 _H3_CHANGED_TEXT = 'Changed by JS'
 _INJECTED_JS_DELAY_MS = 100
+# Generous wait for the JS-injected element. Resolves as soon as it appears, so it stays robust on
+# slow CI without slowing the happy path.
+_ELEMENT_WAIT_TIMEOUT = timedelta(seconds=5)
 _PAGE_CONTENT_STATIC = f"""
 <h1>{_H1_TEXT}</h1>
 <h3>Initial text</h3>
@@ -636,10 +639,6 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
     mocked_browser_handler.assert_called_once_with()
 
 
-@pytest.mark.flaky(
-    reruns=3,
-    reason='Test is flaky on Windows and MacOS, see https://github.com/apify/crawlee-python/issues/1650.',
-)
 async def test_adaptive_context_query_selector_beautiful_soup(test_urls: list[str]) -> None:
     """Test that `context.query_selector_one` works regardless of the crawl type for BeautifulSoup variant.
 
@@ -663,9 +662,9 @@ async def test_adaptive_context_query_selector_beautiful_soup(test_urls: list[st
 
     @crawler.router.default_handler
     async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
-        h1 = await context.query_selector_one('h1', timedelta(milliseconds=_INJECTED_JS_DELAY_MS * 2))
+        h1 = await context.query_selector_one('h1', _ELEMENT_WAIT_TIMEOUT)
         mocked_h1_handler(h1)
-        h2 = await context.query_selector_one('h2', timedelta(milliseconds=_INJECTED_JS_DELAY_MS * 2))
+        h2 = await context.query_selector_one('h2', _ELEMENT_WAIT_TIMEOUT)
         mocked_h2_handler(h2)
 
     await crawler.run(test_urls[:1])
@@ -682,10 +681,6 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
     mocked_h2_handler.assert_has_calls([call(expected_h2_tag)])
 
 
-@pytest.mark.flaky(
-    reruns=3,
-    reason='Test is flaky on Windows and MacOS, see https://github.com/apify/crawlee-python/issues/1650.',
-)
 async def test_adaptive_context_query_selector_parsel(test_urls: list[str]) -> None:
     """Test that `context.query_selector_one` works regardless of the crawl type for Parsel variant.
 
@@ -711,9 +706,9 @@ async def test_adaptive_context_query_selector_parsel(test_urls: list[str]) -> N
 
     @crawler.router.default_handler
     async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
-        if h1 := await context.query_selector_one('h1', timedelta(milliseconds=_INJECTED_JS_DELAY_MS * 2)):
+        if h1 := await context.query_selector_one('h1', _ELEMENT_WAIT_TIMEOUT):
             mocked_h1_handler(type(h1), h1.get())
-        if h2 := await context.query_selector_one('h2', timedelta(milliseconds=_INJECTED_JS_DELAY_MS * 2)):
+        if h2 := await context.query_selector_one('h2', _ELEMENT_WAIT_TIMEOUT):
             mocked_h2_handler(type(h2), h2.get())
 
     await crawler.run(test_urls[:1])
@@ -746,7 +741,7 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
 
         # Reparse whole page after h2 appears
         parsed_content_after_h2_appeared = await context.parse_with_static_parser(
-            selector='h2', timeout=timedelta(milliseconds=_INJECTED_JS_DELAY_MS * 2)
+            selector='h2', timeout=_ELEMENT_WAIT_TIMEOUT
         )
         mocked_h2_handler(parsed_content_after_h2_appeared.css('h2')[0].get())
 

tests/unit/crawlers/_basic/test_basic_crawler.py

@@ -1529,10 +1529,6 @@ async def handler(context: BasicCrawlingContext) -> None:
     assert mocked_handler_after_sleep.call_count == 1
 
 
-@pytest.mark.flaky(
-    reruns=3,
-    reason='Test is flaky on Windows and MacOS, see https://github.com/apify/crawlee-python/issues/1649.',
-)
 @pytest.mark.parametrize(
     ('keep_alive', 'max_requests_per_crawl', 'expected_handled_requests_count'),
     [
@@ -1565,16 +1561,38 @@ async def handler(context: BasicCrawlingContext) -> None:
         if context.request == additional_urls[-1]:
             crawler.stop()
 
-    crawler_run_task = asyncio.create_task(crawler.run())
+    # Deterministically wait until the crawler has drained the initial queue.
+    idle_signal = asyncio.Event()
+    original_is_finished = crawler._autoscaled_pool._is_finished_function
 
-    # Give some time to crawler to finish(or be in keep_alive state) and add new request.
-    # TODO: Replace sleep time by waiting for specific crawler state.
-    # https://github.com/apify/crawlee-python/issues/925
-    await asyncio.sleep(1)
-    assert crawler_run_task.done() != keep_alive
-    add_request_task = asyncio.create_task(crawler.add_requests(additional_urls))
+    async def signal_idle_reached(*args: object, **kwargs: object) -> bool:
+        try:
+            return await original_is_finished(*args, **kwargs)
+        finally:
+            idle_signal.set()
 
-    await asyncio.gather(crawler_run_task, add_request_task)
+    with patch.object(
+        crawler._autoscaled_pool,
+        '_is_finished_function',
+        side_effect=signal_idle_reached,
+    ):
+        crawler_run_task = asyncio.create_task(crawler.run())
+        try:
+            await asyncio.wait_for(idle_signal.wait(), timeout=30)
+
+            if not keep_alive:
+                # Without keep_alive the crawler finishes on its own once the queue is drained.
+                await asyncio.wait_for(crawler_run_task, timeout=30)
+
+            assert crawler_run_task.done() != keep_alive
+            add_request_task = asyncio.create_task(crawler.add_requests(additional_urls))
+
+            await asyncio.gather(crawler_run_task, add_request_task)
+        finally:
+            # Never leave the crawler running in the background if an assertion above fails.
+            if not crawler_run_task.done():
+                crawler.stop()
+                await asyncio.gather(crawler_run_task, return_exceptions=True)
 
     mocked_handler.assert_has_calls(expected_handler_calls)
 
@@ -2252,7 +2270,7 @@ async def test_global_and_local_event_manager_in_crawler_run() -> None:
     @crawler.router.default_handler
     async def handler(context: BasicCrawlingContext) -> None:
         global_event_manager = service_locator.get_event_manager()
-        handler_call(local_event_manager.active, global_event_manager.active)
+        await handler_call(local_event_manager.active, global_event_manager.active)
 
     await crawler.run(['https://a.placeholder.com'])
 

tests/unit/crawlers/_playwright/test_playwright_crawler.py

@@ -266,7 +266,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None:
     assert 'headless' not in headers['user-agent'].lower()
 
 
-@pytest.mark.flaky(reruns=3, reason='Test is flaky.')
+@pytest.mark.run_alone
 async def test_firefox_headless_headers(header_network: dict, server_url: URL) -> None:
     browser_type: BrowserType = 'firefox'
     crawler = PlaywrightCrawler(headless=True, browser_type=browser_type)