Tests: Retry apiCheckSubscriberExists

.github/workflows/tests.yml

@@ -180,6 +180,19 @@ jobs:
           php-version: ${{ matrix.php-versions }}
           coverage: xdebug
 
+      # Workaround for PHP 8.3 (and potentially other versions): the GitHub
+      # Actions ubuntu-latest runner ships with a pre-installed PHP that leaves
+      # an empty regular file at /run/php/php<version>-fpm.sock. setup-php
+      # installs a different patch version on top, but PHP-FPM never replaces
+      # the stale file with a real Unix socket, so nginx returns 502.
+      # Removing the stale file and restarting PHP-FPM forces a clean socket.
+      - name: Restart PHP-FPM
+        run: |
+          sudo rm -f /run/php/php${{ matrix.php-versions }}-fpm.sock
+          sudo systemctl restart php${{ matrix.php-versions }}-fpm
+          sleep 1
+          ls -la /run/php/
+
       # Configure nginx to use the PHP version and WOrdPress installation at /var/www/html
       - name: Configure nginx site
         run: |
@@ -305,7 +318,7 @@ jobs:
       # Run Codeception End to End Tests.
       - name: Run tests/${{ matrix.test-groups }}
         working-directory: ${{ env.PLUGIN_DIR }}
-        run: php vendor/bin/codecept run tests/${{ matrix.test-groups }} --fail-fast
+        run: php vendor/bin/codecept run tests/${{ matrix.test-groups }} --env headless --fail-fast
 
       # Artifacts are data generated by this workflow that we want to access, such as log files, screenshots, HTML output.
       # The if: failure() directive means that this will run when the workflow fails e.g. if a test fails, which is needed

tests/EndToEnd.suite.yml

@@ -68,4 +68,27 @@ modules:
             wpRootFolder: '%WORDPRESS_ROOT_DIR%'
             dbUrl: '%WORDPRESS_DB_URL%'
             domain: '%WORDPRESS_DOMAIN%'
-
+
+# Environments
+#
+# Run with `vendor/bin/codecept run EndToEnd --env headless` to run Chrome in
+# headless mode (used by GitHub Actions). Without `--env headless`, Chrome runs
+# in headed mode so a developer can watch tests execute locally.
+env:
+    headless:
+        modules:
+            config:
+                lucatume\WPBrowser\Module\WPWebDriver:
+                    capabilities:
+                      "goog:chromeOptions":
+                        args:
+                          - "--headless"
+                          - "--disable-gpu"
+                          - "--disable-dev-shm-usage"
+                          - "--disable-software-rasterizer"
+                          - "--proxy-server='direct://'"
+                          - "--proxy-bypass-list=*"
+                          - "--no-sandbox"
+                          - "--user-agent=%TEST_SITE_HTTP_USER_AGENT%"
+                        prefs:
+                          download.default_directory: '%WORDPRESS_ROOT_DIR%'

tests/Support/Helper/KitAPI.php

@@ -53,20 +53,32 @@ public function apiEncodeState($returnTo, $clientID)
 	 */
 	public function apiCheckSubscriberExists($I, $emailAddress, $firstName = false, $customFields = false)
 	{
-		// Run request.
-		$results = $this->apiRequest(
-			'subscribers',
-			'GET',
-			[
-				'email_address'       => $emailAddress,
-				'include_total_count' => true,
+		// Wait for the API to update.
+		$I->wait(3);
 
-				// Some test email addresses might bounce, so we want to check all subscriber states.
-				'status'              => 'all',
-			]
+		// Retry the API request as sometimes there's a lag before the subscriber is queryable via the API.
+		$results = $this->retryUntil(
+			function () use ($emailAddress) {
+				$results = $this->apiRequest(
+					'subscribers',
+					'GET',
+					[
+						'email_address'       => $emailAddress,
+						'include_total_count' => true,
+
+						// Check all subscriber states.
+						'status'              => 'all',
+					]
+				);
+
+				// Return the results only if a subscriber was found, so
+				// retryUntil() will keep trying otherwise.
+				return ( $results['pagination']['total_count'] > 0 ) ? $results : false;
+			}
 		);
 
 		// Check at least one subscriber was returned and it matches the email address.
+		$I->assertNotFalse($results);
 		$I->assertGreaterThan(0, $results['pagination']['total_count']);
 		$I->assertEquals($emailAddress, $results['subscribers'][0]['email_address']);
 
@@ -288,4 +300,39 @@ public function apiRequest($endpoint, $method = 'GET', $params = array())
 		// Return JSON decoded response.
 		return json_decode($result->getBody()->getContents(), true);
 	}
+
+	/**
+	 * Repeatedly invokes the given callback until it returns a truthy value, or
+	 * the maximum number of attempts is reached.
+	 *
+	 * Use this to wrap API checks that can be flaky due to ingestion lag at
+	 * Kit's end (e.g. a subscriber created via a form submission isn't always
+	 * immediately queryable via the `subscribers` endpoint).
+	 *
+	 * @since   1.9.4
+	 *
+	 * @param   callable $callback   Callback to invoke. Should return the value
+	 *                                to use, or false/null to indicate the
+	 *                                check has not yet succeeded.
+	 * @param   int      $attempts   Maximum number of attempts.
+	 * @param   int      $delay      Seconds to wait between attempts.
+	 * @return  mixed                The truthy value returned by $callback, or
+	 *                                false if all attempts are exhausted.
+	 */
+	private function retryUntil(callable $callback, $attempts = 4, $delay = 3)
+	{
+		for ($i = 0; $i < $attempts; $i++) {
+			$result = $callback();
+			if ($result) {
+				return $result;
+			}
+
+			// Don't sleep after the final attempt.
+			if ($i < $attempts - 1) {
+				sleep($delay);
+			}
+		}
+
+		return false;
+	}
 }