diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml
index 0e63359da..fa21e1371 100644
--- a/.github/workflows/android.yml
+++ b/.github/workflows/android.yml
@@ -43,8 +43,10 @@ jobs:
yes | "$SDKMANAGER" --licenses >/dev/null
"$SDKMANAGER" "platforms;android-36" "build-tools;36.0.0"
- - name: Package npm-bundled Android snapshot helper
- run: pnpm package:android-snapshot-helper:npm
+ - name: Package npm-bundled Android helpers
+ run: |
+ pnpm package:android-snapshot-helper:npm
+ pnpm package:android-multitouch-helper:npm
- name: Run Android smoke checks
uses: reactivecircus/android-emulator-runner@b530d96654c385303d652368551fb075bc2f0b6b # v2.35.0
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ada17d156..11cb07349 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -104,8 +104,8 @@ jobs:
- name: Run typecheck
run: pnpm typecheck
- android-snapshot-helper:
- name: Android Snapshot Helper Package
+ android-helpers:
+ name: Android Helper Packages
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
@@ -134,8 +134,10 @@ jobs:
javac --version
java --version
- - name: Package npm-bundled Android snapshot helper
- run: pnpm package:android-snapshot-helper:npm
+ - name: Package npm-bundled Android helpers
+ run: |
+ pnpm package:android-snapshot-helper:npm
+ pnpm package:android-multitouch-helper:npm
integration:
name: Integration Tests
diff --git a/.gitignore b/.gitignore
index 6f25df8b2..1dc2da634 100644
--- a/.gitignore
+++ b/.gitignore
@@ -33,3 +33,5 @@ xcuserdata/
.skillgym-results/
android-snapshot-helper/build/
android-snapshot-helper/dist/
+android-multitouch-helper/build/
+android-multitouch-helper/dist/
diff --git a/android-multitouch-helper/AndroidManifest.xml b/android-multitouch-helper/AndroidManifest.xml
new file mode 100644
index 000000000..d246e117a
--- /dev/null
+++ b/android-multitouch-helper/AndroidManifest.xml
@@ -0,0 +1,17 @@
+
+
+
+
+
+
+
diff --git a/android-multitouch-helper/README.md b/android-multitouch-helper/README.md
new file mode 100644
index 000000000..c23fe15bc
--- /dev/null
+++ b/android-multitouch-helper/README.md
@@ -0,0 +1,41 @@
+# Android MultiTouch Helper
+
+Small instrumentation APK used to inject Android two-pointer gestures through
+`UiAutomation.injectInputEvent`. The helper accepts a compact base64 JSON payload so local ADB,
+remote ADB tunnels, and remote providers that allow `adb install -t` plus `am instrument` can use
+the same contract.
+
+The helper is separate from `android-snapshot-helper` because the payload and output protocol are
+gesture-specific. The install/version/cache lifecycle should stay aligned with the snapshot helper.
+
+## Build
+
+```sh
+VERSION="$(node -p 'require("./package.json").version')"
+sh ./scripts/build-android-multitouch-helper.sh "$VERSION" .tmp/android-multitouch-helper
+```
+
+## Run
+
+```sh
+PAYLOAD="$(printf '%s' '{"kind":"transform","x":672,"y":1500,"dx":80,"dy":-40,"scale":1.8,"degrees":35,"durationMs":700}' | base64)"
+adb install -r -t ".tmp/android-multitouch-helper/agent-device-android-multitouch-helper-$VERSION.apk"
+adb shell am instrument -w \
+ -e payloadBase64 "$PAYLOAD" \
+ com.callstack.agentdevice.multitouchhelper/.MultiTouchInstrumentation
+```
+
+## Output Contract
+
+The APK emits instrumentation result records using
+`agentDeviceProtocol=android-multitouch-helper-v1`.
+
+Successful results include:
+
+- `ok=true`
+- `helperApiVersion=1`
+- `kind` (`pinch`, `rotate`, or `transform`)
+- `injectedEvents`
+- `elapsedMs`
+
+Failures return `ok=false`, `errorType`, and `message`.
diff --git a/android-multitouch-helper/src/main/java/com/callstack/agentdevice/multitouchhelper/MultiTouchInstrumentation.java b/android-multitouch-helper/src/main/java/com/callstack/agentdevice/multitouchhelper/MultiTouchInstrumentation.java
new file mode 100644
index 000000000..31fe96d83
--- /dev/null
+++ b/android-multitouch-helper/src/main/java/com/callstack/agentdevice/multitouchhelper/MultiTouchInstrumentation.java
@@ -0,0 +1,276 @@
+package com.callstack.agentdevice.multitouchhelper;
+
+import android.app.Instrumentation;
+import android.app.UiAutomation;
+import android.os.Bundle;
+import android.os.SystemClock;
+import android.util.Base64;
+import android.view.InputDevice;
+import android.view.MotionEvent;
+import java.nio.charset.StandardCharsets;
+import org.json.JSONObject;
+
+public final class MultiTouchInstrumentation extends Instrumentation {
+ private static final String PROTOCOL = "android-multitouch-helper-v1";
+ private static final String HELPER_API_VERSION = "1";
+ private static final int DEFAULT_RADIUS = 160;
+ private static final int MIN_RADIUS = 24;
+ private static final int MAX_RADIUS = 1200;
+ private static final int MIN_DURATION_MS = 16;
+ private static final int MAX_DURATION_MS = 10_000;
+ private Bundle arguments;
+
+ @Override
+ public void onCreate(Bundle arguments) {
+ super.onCreate(arguments);
+ this.arguments = arguments;
+ start();
+ }
+
+ @Override
+ public void onStart() {
+ super.onStart();
+ Bundle result = new Bundle();
+ result.putString("agentDeviceProtocol", PROTOCOL);
+ result.putString("helperApiVersion", HELPER_API_VERSION);
+ try {
+ long startedAtMs = System.currentTimeMillis();
+ GestureSpec spec = readSpec(arguments);
+ int injectedEvents = injectGesture(spec);
+ result.putString("ok", "true");
+ result.putString("kind", spec.kind);
+ result.putString("injectedEvents", Integer.toString(injectedEvents));
+ result.putString("elapsedMs", Long.toString(System.currentTimeMillis() - startedAtMs));
+ finish(0, result);
+ } catch (Throwable error) {
+ result.putString("ok", "false");
+ result.putString("errorType", error.getClass().getName());
+ result.putString(
+ "message",
+ error.getMessage() == null ? error.getClass().getName() : error.getMessage());
+ finish(1, result);
+ }
+ }
+
+ private GestureSpec readSpec(Bundle arguments) throws Exception {
+ String payloadBase64 = arguments.getString("payloadBase64", "");
+ if (payloadBase64.isEmpty()) {
+ throw new IllegalArgumentException("Missing payloadBase64");
+ }
+ String json =
+ new String(Base64.decode(payloadBase64, Base64.DEFAULT), StandardCharsets.UTF_8);
+ JSONObject payload = new JSONObject(json);
+ String protocol = payload.optString("protocol", PROTOCOL);
+ if (!PROTOCOL.equals(protocol)) {
+ throw new IllegalArgumentException("Unsupported protocol: " + protocol);
+ }
+ String kind = payload.getString("kind");
+ if (!"pinch".equals(kind) && !"rotate".equals(kind) && !"transform".equals(kind)) {
+ throw new IllegalArgumentException("Unsupported kind: " + kind);
+ }
+ int x = payload.getInt("x");
+ int y = payload.getInt("y");
+ int dx = payload.optInt("dx", 0);
+ int dy = payload.optInt("dy", 0);
+ int durationMs = clamp(payload.optInt("durationMs", 300), MIN_DURATION_MS, MAX_DURATION_MS);
+ int radius = clamp(payload.optInt("radius", DEFAULT_RADIUS), MIN_RADIUS, MAX_RADIUS);
+ double scale = payload.optDouble("scale", 1.0d);
+ double degrees = payload.optDouble("degrees", 0.0d);
+ if (("pinch".equals(kind) || "transform".equals(kind)) && (!isFinite(scale) || scale <= 0)) {
+ throw new IllegalArgumentException("Scale must be > 0");
+ }
+ if (("rotate".equals(kind) || "transform".equals(kind)) && !isFinite(degrees)) {
+ throw new IllegalArgumentException("Degrees must be finite");
+ }
+ return new GestureSpec(kind, x, y, dx, dy, durationMs, scale, degrees, radius);
+ }
+
+ private int injectGesture(GestureSpec spec) {
+ UiAutomation automation = getUiAutomation();
+ long downTime = SystemClock.uptimeMillis();
+ long eventTime = downTime;
+ PointerPair start = pointerPairAt(spec, 0);
+ PointerPair end = pointerPairAt(spec, 1);
+ int count = 0;
+
+ inject(
+ automation,
+ motionEvent(downTime, eventTime, MotionEvent.ACTION_DOWN, start.firstOnly()));
+ count += 1;
+ eventTime += 8;
+ inject(
+ automation,
+ motionEvent(
+ downTime,
+ eventTime,
+ MotionEvent.ACTION_POINTER_DOWN | (1 << MotionEvent.ACTION_POINTER_INDEX_SHIFT),
+ start));
+ count += 1;
+
+ int frameCount = Math.max(3, Math.round(spec.durationMs / 16.0f));
+ for (int index = 1; index < frameCount; index += 1) {
+ double t = (double) index / (double) frameCount;
+ PointerPair frame = pointerPairAt(spec, t);
+ eventTime = downTime + Math.round(spec.durationMs * t);
+ inject(automation, motionEvent(downTime, eventTime, MotionEvent.ACTION_MOVE, frame));
+ count += 1;
+ }
+
+ eventTime = downTime + spec.durationMs;
+ inject(
+ automation,
+ motionEvent(
+ downTime,
+ eventTime,
+ MotionEvent.ACTION_POINTER_UP | (1 << MotionEvent.ACTION_POINTER_INDEX_SHIFT),
+ end));
+ count += 1;
+ inject(
+ automation,
+ motionEvent(downTime, eventTime + 8, MotionEvent.ACTION_UP, end.firstOnly()));
+ count += 1;
+ return count;
+ }
+
+ private static void inject(UiAutomation automation, MotionEvent event) {
+ try {
+ if (!automation.injectInputEvent(event, true)) {
+ throw new IllegalStateException("injectInputEvent returned false");
+ }
+ } finally {
+ event.recycle();
+ }
+ }
+
+ private static MotionEvent motionEvent(long downTime, long eventTime, int action, PointerPair pair) {
+ MotionEvent.PointerProperties[] properties =
+ new MotionEvent.PointerProperties[pair.pointerCount];
+ MotionEvent.PointerCoords[] coords = new MotionEvent.PointerCoords[pair.pointerCount];
+ for (int index = 0; index < pair.pointerCount; index += 1) {
+ properties[index] = new MotionEvent.PointerProperties();
+ properties[index].id = index;
+ properties[index].toolType = MotionEvent.TOOL_TYPE_FINGER;
+ coords[index] = new MotionEvent.PointerCoords();
+ coords[index].x = pair.x[index];
+ coords[index].y = pair.y[index];
+ coords[index].pressure = 1.0f;
+ coords[index].size = 1.0f;
+ }
+ MotionEvent event =
+ MotionEvent.obtain(
+ downTime,
+ eventTime,
+ action,
+ pair.pointerCount,
+ properties,
+ coords,
+ 0,
+ 0,
+ 1.0f,
+ 1.0f,
+ 0,
+ 0,
+ InputDevice.SOURCE_TOUCHSCREEN,
+ 0);
+ event.setSource(InputDevice.SOURCE_TOUCHSCREEN);
+ return event;
+ }
+
+ private static PointerPair pointerPairAt(GestureSpec spec, double t) {
+ if ("pinch".equals(spec.kind)) {
+ double startRadius = spec.radius / Math.max(spec.scale, 1.0d);
+ double endRadius = spec.radius;
+ if (spec.scale < 1.0d) {
+ startRadius = spec.radius;
+ endRadius = spec.radius * spec.scale;
+ }
+ double radius = startRadius + (endRadius - startRadius) * t;
+ return new PointerPair(
+ new float[] {(float) (spec.x - radius), (float) (spec.x + radius)},
+ new float[] {(float) spec.y, (float) spec.y});
+ }
+ double centerX = spec.x;
+ double centerY = spec.y;
+ double radius = spec.radius;
+ if ("transform".equals(spec.kind)) {
+ centerX = spec.x + spec.dx * t;
+ centerY = spec.y + spec.dy * t;
+ double startRadius = spec.radius / Math.max(spec.scale, 1.0d);
+ double endRadius = spec.radius;
+ if (spec.scale < 1.0d) {
+ startRadius = spec.radius;
+ endRadius = spec.radius * spec.scale;
+ }
+ radius = startRadius + (endRadius - startRadius) * t;
+ }
+ double angle = Math.toRadians(-90 + spec.degrees * t);
+ return new PointerPair(
+ new float[] {
+ (float) (centerX + Math.cos(angle) * radius),
+ (float) (centerX - Math.cos(angle) * radius)
+ },
+ new float[] {
+ (float) (centerY + Math.sin(angle) * radius),
+ (float) (centerY - Math.sin(angle) * radius)
+ });
+ }
+
+ private static int clamp(int value, int min, int max) {
+ return Math.min(Math.max(value, min), max);
+ }
+
+ private static boolean isFinite(double value) {
+ return !Double.isNaN(value) && !Double.isInfinite(value);
+ }
+
+ private static final class GestureSpec {
+ final String kind;
+ final int x;
+ final int y;
+ final int dx;
+ final int dy;
+ final int durationMs;
+ final double scale;
+ final double degrees;
+ final int radius;
+
+ GestureSpec(
+ String kind,
+ int x,
+ int y,
+ int dx,
+ int dy,
+ int durationMs,
+ double scale,
+ double degrees,
+ int radius) {
+ this.kind = kind;
+ this.x = x;
+ this.y = y;
+ this.dx = dx;
+ this.dy = dy;
+ this.durationMs = durationMs;
+ this.scale = scale;
+ this.degrees = degrees;
+ this.radius = radius;
+ }
+ }
+
+ private static final class PointerPair {
+ final int pointerCount;
+ final float[] x;
+ final float[] y;
+
+ PointerPair(float[] x, float[] y) {
+ this.pointerCount = x.length;
+ this.x = x;
+ this.y = y;
+ }
+
+ PointerPair firstOnly() {
+ return new PointerPair(
+ new float[] {x[0]},
+ new float[] {y[0]});
+ }
+ }
+}
diff --git a/examples/test-app/README.md b/examples/test-app/README.md
index 8b7990ac2..dc167d119 100644
--- a/examples/test-app/README.md
+++ b/examples/test-app/README.md
@@ -88,6 +88,11 @@ pnpm test-app:replay:android
These run the `.ad` replay suite in `examples/test-app/replays`.
+`gesture-lab.ad` verifies `gesture pan`, `gesture fling`, `gesture pinch`, and
+`gesture rotate` against the gesture metrics rendered by the Home screen on iOS
+and Android. Android and iOS simulator sessions also support `gesture transform`
+for a combined pan/zoom/rotate gesture.
+
To target a specific iOS simulator or an installed Expo development build, run the
underlying command directly so global flags stay before replay inputs:
diff --git a/examples/test-app/app/_layout.tsx b/examples/test-app/app/_layout.tsx
index e59901b74..478ef2a36 100644
--- a/examples/test-app/app/_layout.tsx
+++ b/examples/test-app/app/_layout.tsx
@@ -1,6 +1,8 @@
import { ThemeProvider } from '@react-navigation/native';
import { Stack } from 'expo-router';
import { StatusBar } from 'expo-status-bar';
+import { StyleSheet } from 'react-native';
+import { GestureHandlerRootView } from 'react-native-gesture-handler';
import { SafeAreaProvider } from 'react-native-safe-area-context';
import { ToastViewport } from '../src/components';
@@ -30,10 +32,18 @@ function RootLayoutContent() {
export default function RootLayout() {
return (
-
-
-
-
-
+
+
+
+
+
+
+
);
}
+
+const styles = StyleSheet.create({
+ root: {
+ flex: 1,
+ },
+});
diff --git a/examples/test-app/package.json b/examples/test-app/package.json
index e6309f66b..1f7916bcf 100644
--- a/examples/test-app/package.json
+++ b/examples/test-app/package.json
@@ -19,6 +19,7 @@
"expo-status-bar": "~55.0.5",
"react": "19.2.0",
"react-native": "0.83.4",
+ "react-native-gesture-handler": "^2.31.2",
"react-native-safe-area-context": "~5.6.2",
"react-native-screens": "~4.23.0"
},
diff --git a/examples/test-app/pnpm-lock.yaml b/examples/test-app/pnpm-lock.yaml
index 2ae4b943b..9b0e7f8d7 100644
--- a/examples/test-app/pnpm-lock.yaml
+++ b/examples/test-app/pnpm-lock.yaml
@@ -4,11 +4,6 @@ settings:
autoInstallPeers: true
excludeLinksFromLockfile: false
-overrides:
- '@xmldom/xmldom': 0.8.13
- postcss: 8.5.12
- uuid: 14.0.0
-
importers:
.:
@@ -30,7 +25,7 @@ importers:
version: 55.0.11(expo@55.0.12)(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)(typescript@5.9.3)
expo-router:
specifier: ~55.0.11
- version: 55.0.11(@expo/log-box@55.0.10)(@expo/metro-runtime@55.0.9)(@types/react@19.2.14)(expo-constants@55.0.12)(expo-font@55.0.6)(expo-linking@55.0.11)(expo@55.0.12)(react-dom@19.2.4(react@19.2.0))(react-native-safe-area-context@5.6.2(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0))(react-native-screens@4.23.0(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0))(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)
+ version: 55.0.11(@expo/log-box@55.0.10)(@expo/metro-runtime@55.0.9)(@types/react@19.2.14)(expo-constants@55.0.12)(expo-font@55.0.6)(expo-linking@55.0.11)(expo@55.0.12)(react-dom@19.2.4(react@19.2.0))(react-native-gesture-handler@2.31.2(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0))(react-native-safe-area-context@5.6.2(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0))(react-native-screens@4.23.0(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0))(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)
expo-status-bar:
specifier: ~55.0.5
version: 55.0.5(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)
@@ -40,6 +35,9 @@ importers:
react-native:
specifier: 0.83.4
version: 0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0)
+ react-native-gesture-handler:
+ specifier: ^2.31.2
+ version: 2.31.2(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)
react-native-safe-area-context:
specifier: ~5.6.2
version: 5.6.2(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)
@@ -546,6 +544,10 @@ packages:
resolution: {integrity: sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==}
engines: {node: '>=6.9.0'}
+ '@egjs/hammerjs@2.0.17':
+ resolution: {integrity: sha512-XQsZgjm2EcVUiZQf11UBJQfmZeEmOW8DpI1gsFeln6w0ae0ii4dMQEQ0kjl6DspdWX1aGY1/loyXnP0JS06e/A==}
+ engines: {node: '>=0.8.0'}
+
'@expo-google-fonts/material-symbols@0.4.29':
resolution: {integrity: sha512-3WBhUiK6V91KFlAqWkBRk/lLIHyo0EXg2Paifp+J0LLwlAw7pW2/xf42A5U2g3JzAfzvYHYylfD5Ch4sjvVmqg==}
@@ -1122,6 +1124,9 @@ packages:
'@types/graceful-fs@4.1.9':
resolution: {integrity: sha512-olP3sd1qOEe5dXTSaFvQG+02VdRXcdytWLAZsAq1PecU8uqQAhkrnbli7DagjtXKW/Bl7YJbUsa8MPcuc8LHEQ==}
+ '@types/hammerjs@2.0.46':
+ resolution: {integrity: sha512-ynRvcq6wvqexJ9brDMS4BnBLzmr0e14d6ZJTEShTBWKymQiHwlAyGu0ZPEFI2Fh1U53F7tN9ufClWM5KvqkKOw==}
+
'@types/istanbul-lib-coverage@2.0.6':
resolution: {integrity: sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==}
@@ -1134,6 +1139,9 @@ packages:
'@types/node@25.5.2':
resolution: {integrity: sha512-tO4ZIRKNC+MDWV4qKVZe3Ql/woTnmHDr5JD8UI5hn2pwBrHEwOEMZK7WlNb5RKB6EoJ02gwmQS9OrjuFnZYdpg==}
+ '@types/react-test-renderer@19.1.0':
+ resolution: {integrity: sha512-XD0WZrHqjNrxA/MaR9O22w/RNidWR9YZmBdRGI7wcnWGrv/3dA8wKCJ8m63Sn+tLJhcjmuhOi629N66W6kgWzQ==}
+
'@types/react@19.2.14':
resolution: {integrity: sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==}
@@ -1148,6 +1156,7 @@ packages:
'@ungap/structured-clone@1.3.0':
resolution: {integrity: sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==}
+ deprecated: Potential CWE-502 - Update to 1.3.1 or higher
'@xmldom/xmldom@0.8.13':
resolution: {integrity: sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw==}
@@ -1830,6 +1839,9 @@ packages:
hermes-parser@0.33.3:
resolution: {integrity: sha512-Yg3HgaG4CqgyowtYjX/FsnPAuZdHOqSMtnbpylbptsQ9nwwSKsy6uRWcGO5RK0EqiX12q8HvDWKgeAVajRO5DA==}
+ hoist-non-react-statics@3.3.2:
+ resolution: {integrity: sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==}
+
hosted-git-info@7.0.2:
resolution: {integrity: sha512-puUZAUKT5m8Zzvs72XWy3HtvVbTWljRE66cP60bxJzAqf2DgICo7lYTY2IHUmLnNpjYvw5bvmoHvPc0QO2a62w==}
engines: {node: ^16.14.0 || >=18.0.0}
@@ -2339,8 +2351,8 @@ packages:
resolution: {integrity: sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w==}
engines: {node: '>=4.0.0'}
- postcss@8.5.12:
- resolution: {integrity: sha512-W62t/Se6rA0Az3DfCL0AqJwXuKwBeYg6nOaIgzP+xZ7N5BFCI7DYi1qs6ygUYT6rvfi6t9k65UMLJC+PHZpDAA==}
+ postcss@8.4.49:
+ resolution: {integrity: sha512-OCVPnIObs4N29kxTjzLfUryOkvZEq+pf8jTF0lg8E7uETuWHA+v7j3c/xJmiqpX450191LlmZfUKkXxkTry7nA==}
engines: {node: ^10 || ^12 || >=14}
pretty-format@29.7.0:
@@ -2390,12 +2402,21 @@ packages:
peerDependencies:
react: '>=17.0.0'
+ react-is@16.13.1:
+ resolution: {integrity: sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==}
+
react-is@18.3.1:
resolution: {integrity: sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==}
react-is@19.2.4:
resolution: {integrity: sha512-W+EWGn2v0ApPKgKKCy/7s7WHXkboGcsrXE+2joLyVxkbyVQfO3MUEaUQDHoSmb8TFFrSKYa9mw64WZHNHSDzYA==}
+ react-native-gesture-handler@2.31.2:
+ resolution: {integrity: sha512-rw5q74i2AfS7YGYdbxQDhOU7xqgY6WRM1132/CCm3erqjblhECZDZFHIm0tteHoC9ih24wogVBVVzcTBQtZ+5A==}
+ peerDependencies:
+ react: '*'
+ react-native: '*'
+
react-native-is-edge-to-edge@1.3.1:
resolution: {integrity: sha512-NIXU/iT5+ORyCc7p0z2nnlkouYKX425vuU1OEm6bMMtWWR9yvb+Xg5AZmImTKoF9abxCPqrKC3rOZsKzUYgYZA==}
peerDependencies:
@@ -2787,8 +2808,9 @@ packages:
resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==}
engines: {node: '>= 0.4.0'}
- uuid@14.0.0:
- resolution: {integrity: sha512-Qo+uWgilfSmAhXCMav1uYFynlQO7fMFiMVZsQqZRMIXp0O7rR7qjkj+cPvBHLgBqi960QCoo/PH2/6ZtVqKvrg==}
+ uuid@7.0.3:
+ resolution: {integrity: sha512-DPSke0pXhTZgoF/d+WSt2QaKMCFSfx7QegxEWT+JOuHF5aWrKEn0G+ztjuJg/gG8/ItK+rbPCD/yNv8yyih6Cg==}
+ deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).
hasBin: true
validate-npm-package-name@5.0.1:
@@ -3498,6 +3520,10 @@ snapshots:
'@babel/helper-string-parser': 7.27.1
'@babel/helper-validator-identifier': 7.28.5
+ '@egjs/hammerjs@2.0.17':
+ dependencies:
+ '@types/hammerjs': 2.0.46
+
'@expo-google-fonts/material-symbols@0.4.29': {}
'@expo/cli@55.0.22(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-constants@55.0.12)(expo-font@55.0.6)(expo-router@55.0.11)(expo@55.0.12)(react-dom@19.2.4(react@19.2.0))(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)(typescript@5.9.3)':
@@ -3561,7 +3587,7 @@ snapshots:
ws: 8.20.0
zod: 3.25.76
optionalDependencies:
- expo-router: 55.0.11(@expo/log-box@55.0.10)(@expo/metro-runtime@55.0.9)(@types/react@19.2.14)(expo-constants@55.0.12)(expo-font@55.0.6)(expo-linking@55.0.11)(expo@55.0.12)(react-dom@19.2.4(react@19.2.0))(react-native-safe-area-context@5.6.2(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0))(react-native-screens@4.23.0(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0))(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)
+ expo-router: 55.0.11(@expo/log-box@55.0.10)(@expo/metro-runtime@55.0.9)(@types/react@19.2.14)(expo-constants@55.0.12)(expo-font@55.0.6)(expo-linking@55.0.11)(expo@55.0.12)(react-dom@19.2.4(react@19.2.0))(react-native-gesture-handler@2.31.2(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0))(react-native-safe-area-context@5.6.2(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0))(react-native-screens@4.23.0(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0))(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)
react-native: 0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0)
transitivePeerDependencies:
- '@expo/dom-webview'
@@ -3712,7 +3738,7 @@ snapshots:
jsc-safe-url: 0.2.4
lightningcss: 1.32.0
picomatch: 4.0.4
- postcss: 8.5.12
+ postcss: 8.4.49
resolve-from: 5.0.0
optionalDependencies:
expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.4(react@19.2.0))(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)(typescript@5.9.3)
@@ -3814,7 +3840,7 @@ snapshots:
react: 19.2.0
optionalDependencies:
'@expo/metro-runtime': 55.0.9(@expo/dom-webview@55.0.5)(expo@55.0.12)(react-dom@19.2.4(react@19.2.0))(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)
- expo-router: 55.0.11(@expo/log-box@55.0.10)(@expo/metro-runtime@55.0.9)(@types/react@19.2.14)(expo-constants@55.0.12)(expo-font@55.0.6)(expo-linking@55.0.11)(expo@55.0.12)(react-dom@19.2.4(react@19.2.0))(react-native-safe-area-context@5.6.2(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0))(react-native-screens@4.23.0(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0))(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)
+ expo-router: 55.0.11(@expo/log-box@55.0.10)(@expo/metro-runtime@55.0.9)(@types/react@19.2.14)(expo-constants@55.0.12)(expo-font@55.0.6)(expo-linking@55.0.11)(expo@55.0.12)(react-dom@19.2.4(react@19.2.0))(react-native-gesture-handler@2.31.2(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0))(react-native-safe-area-context@5.6.2(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0))(react-native-screens@4.23.0(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0))(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)
react-dom: 19.2.4(react@19.2.0)
transitivePeerDependencies:
- supports-color
@@ -4340,6 +4366,8 @@ snapshots:
dependencies:
'@types/node': 25.5.2
+ '@types/hammerjs@2.0.46': {}
+
'@types/istanbul-lib-coverage@2.0.6': {}
'@types/istanbul-lib-report@3.0.3':
@@ -4354,6 +4382,10 @@ snapshots:
dependencies:
undici-types: 7.18.2
+ '@types/react-test-renderer@19.1.0':
+ dependencies:
+ '@types/react': 19.2.14
+
'@types/react@19.2.14':
dependencies:
csstype: 3.2.3
@@ -4880,7 +4912,7 @@ snapshots:
react: 19.2.0
react-native: 0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0)
- expo-router@55.0.11(@expo/log-box@55.0.10)(@expo/metro-runtime@55.0.9)(@types/react@19.2.14)(expo-constants@55.0.12)(expo-font@55.0.6)(expo-linking@55.0.11)(expo@55.0.12)(react-dom@19.2.4(react@19.2.0))(react-native-safe-area-context@5.6.2(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0))(react-native-screens@4.23.0(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0))(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0):
+ expo-router@55.0.11(@expo/log-box@55.0.10)(@expo/metro-runtime@55.0.9)(@types/react@19.2.14)(expo-constants@55.0.12)(expo-font@55.0.6)(expo-linking@55.0.11)(expo@55.0.12)(react-dom@19.2.4(react@19.2.0))(react-native-gesture-handler@2.31.2(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0))(react-native-safe-area-context@5.6.2(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0))(react-native-screens@4.23.0(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0))(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0):
dependencies:
'@expo/log-box': 55.0.10(@expo/dom-webview@55.0.5)(expo@55.0.12)(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)
'@expo/metro-runtime': 55.0.9(@expo/dom-webview@55.0.5)(expo@55.0.12)(react-dom@19.2.4(react@19.2.0))(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)
@@ -4918,6 +4950,7 @@ snapshots:
vaul: 1.1.2(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.0))(react@19.2.0)
optionalDependencies:
react-dom: 19.2.4(react@19.2.0)
+ react-native-gesture-handler: 2.31.2(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)
transitivePeerDependencies:
- '@react-native-masked-view/masked-view'
- '@types/react'
@@ -5088,6 +5121,10 @@ snapshots:
dependencies:
hermes-estree: 0.33.3
+ hoist-non-react-statics@3.3.2:
+ dependencies:
+ react-is: 16.13.1
+
hosted-git-info@7.0.2:
dependencies:
lru-cache: 10.4.3
@@ -5668,7 +5705,7 @@ snapshots:
pngjs@3.4.0: {}
- postcss@8.5.12:
+ postcss@8.4.49:
dependencies:
nanoid: 3.3.11
picocolors: 1.1.1
@@ -5725,10 +5762,21 @@ snapshots:
dependencies:
react: 19.2.0
+ react-is@16.13.1: {}
+
react-is@18.3.1: {}
react-is@19.2.4: {}
+ react-native-gesture-handler@2.31.2(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0):
+ dependencies:
+ '@egjs/hammerjs': 2.0.17
+ '@types/react-test-renderer': 19.1.0
+ hoist-non-react-statics: 3.3.2
+ invariant: 2.2.4
+ react: 19.2.0
+ react-native: 0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0)
+
react-native-is-edge-to-edge@1.3.1(react-native@0.83.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0):
dependencies:
react: 19.2.0
@@ -6098,7 +6146,7 @@ snapshots:
utils-merge@1.0.1: {}
- uuid@14.0.0: {}
+ uuid@7.0.3: {}
validate-npm-package-name@5.0.1: {}
@@ -6153,7 +6201,7 @@ snapshots:
xcode@3.0.1:
dependencies:
simple-plist: 1.3.1
- uuid: 14.0.0
+ uuid: 7.0.3
xml2js@0.6.0:
dependencies:
diff --git a/examples/test-app/replays/gesture-lab.ad b/examples/test-app/replays/gesture-lab.ad
new file mode 100644
index 000000000..a47066ee1
--- /dev/null
+++ b/examples/test-app/replays/gesture-lab.ad
@@ -0,0 +1,39 @@
+context platform=ios timeout=60000
+
+env APP_TARGET="Agent Device Tester"
+env APP_URL=""
+
+open "${APP_TARGET}" --relaunch --launch-url "${APP_URL}"
+wait "Gesture lab" 30000
+
+gesture fling left 195 443 180
+wait "fling 1" 5000
+
+gesture fling right 195 443 180
+wait "fling 2" 5000
+
+gesture fling up 195 443 80 80
+wait "fling 3" 5000
+
+gesture fling down 195 443 80 80
+wait "fling 4" 5000
+
+gesture pan 195 443 -80 0
+wait "x -" 5000
+
+gesture pan 195 443 160 0
+wait "x 72" 5000
+
+gesture pan 195 443 0 -80
+wait "y -" 5000
+
+gesture pan 195 443 0 160
+wait "y 56" 5000
+
+gesture pinch 1.25 195 443
+wait "pinch changed yes" 5000
+
+gesture rotate 35 195 443
+wait "rotate changed yes" 5000
+
+close
diff --git a/examples/test-app/src/screens/GestureLab.tsx b/examples/test-app/src/screens/GestureLab.tsx
new file mode 100644
index 000000000..4fcaf6c71
--- /dev/null
+++ b/examples/test-app/src/screens/GestureLab.tsx
@@ -0,0 +1,251 @@
+import { useRef, useState } from 'react';
+import { Image, StyleSheet, Text, View } from 'react-native';
+import {
+ Directions,
+ FlingGestureHandler,
+ PanGestureHandler,
+ PinchGestureHandler,
+ RotationGestureHandler,
+ State,
+ type FlingGestureHandlerStateChangeEvent,
+ type PanGestureHandlerGestureEvent,
+ type PanGestureHandlerStateChangeEvent,
+ type PinchGestureHandlerGestureEvent,
+ type PinchGestureHandlerStateChangeEvent,
+ type RotationGestureHandlerGestureEvent,
+ type RotationGestureHandlerStateChangeEvent,
+} from 'react-native-gesture-handler';
+
+import { SectionCard } from '../components';
+import { useAppColors, type AppColors } from '../theme';
+
+const gestureImageUri = 'https://reactnative.dev/img/logo-share.png';
+
+type TransformState = {
+ offsetX: number;
+ offsetY: number;
+ rotation: number;
+ scale: number;
+};
+
+type GestureCounts = {
+ fling: number;
+};
+
+const initialTransform: TransformState = {
+ offsetX: 0,
+ offsetY: 0,
+ rotation: 0,
+ scale: 1,
+};
+
+const initialCounts: GestureCounts = {
+ fling: 0,
+};
+const minScale = 0.5;
+const maxScale = 2;
+
+export function GestureLab() {
+ const colors = useAppColors();
+ const styles = createStyles(colors);
+ const [transform, setTransform] = useState(initialTransform);
+ const [counts, setCounts] = useState(initialCounts);
+ const transformRef = useRef(initialTransform);
+ const gestureStartRef = useRef(initialTransform);
+ const flingDownRef = useRef(null);
+ const flingLeftRef = useRef(null);
+ const flingRightRef = useRef(null);
+ const flingUpRef = useRef(null);
+ const panRef = useRef(null);
+ const pinchRef = useRef(null);
+ const rotationRef = useRef(null);
+ const flingRefs = [flingLeftRef, flingRightRef, flingUpRef, flingDownRef];
+
+ function updateTransform(nextTransform: TransformState) {
+ transformRef.current = nextTransform;
+ setTransform(nextTransform);
+ }
+
+ function beginTransformGesture() {
+ gestureStartRef.current = transformRef.current;
+ }
+
+ function handlePan(event: PanGestureHandlerGestureEvent) {
+ const start = gestureStartRef.current;
+ updateTransform({
+ ...transformRef.current,
+ offsetX: clamp(start.offsetX + event.nativeEvent.translationX, -72, 72),
+ offsetY: clamp(start.offsetY + event.nativeEvent.translationY, -56, 56),
+ });
+ }
+
+ function handlePinch(event: PinchGestureHandlerGestureEvent) {
+ const start = gestureStartRef.current;
+ updateTransform({
+ ...transformRef.current,
+ scale: clamp(start.scale * event.nativeEvent.scale, minScale, maxScale),
+ });
+ }
+
+ function handleRotation(event: RotationGestureHandlerGestureEvent) {
+ const start = gestureStartRef.current;
+ updateTransform({
+ ...transformRef.current,
+ rotation: start.rotation + event.nativeEvent.rotation,
+ });
+ }
+
+ function handleTransformStateChange(
+ event:
+ | PanGestureHandlerStateChangeEvent
+ | PinchGestureHandlerStateChangeEvent
+ | RotationGestureHandlerStateChangeEvent,
+ ) {
+ if (event.nativeEvent.state === State.BEGAN) {
+ beginTransformGesture();
+ }
+ }
+
+ function handleFling(event: FlingGestureHandlerStateChangeEvent) {
+ if (event.nativeEvent.state === State.ACTIVE) {
+ setCounts((current) => ({ ...current, fling: current.fling + 1 }));
+ }
+ }
+
+ const rotationDegrees = Math.round((transform.rotation * 180) / Math.PI);
+ const statusLabel = `x ${Math.round(transform.offsetX)}, y ${Math.round(
+ transform.offsetY,
+ )}, scale ${transform.scale.toFixed(2)}, rotate ${rotationDegrees}`;
+ const panChanged = Math.abs(transform.offsetX) > 0 || Math.abs(transform.offsetY) > 0;
+ const pinchChanged = Math.abs(transform.scale - 1) > 0.01;
+ const rotateChanged = rotationDegrees !== 0;
+
+ return (
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ {statusLabel}
+
+
+ fling {counts.fling}
+
+
+ pan changed {panChanged ? 'yes' : 'no'}, pinch changed{' '}
+ {pinchChanged ? 'yes' : 'no'}, rotate changed {rotateChanged ? 'yes' : 'no'}
+
+
+
+ );
+}
+
+function clamp(value: number, min: number, max: number): number {
+ return Math.min(Math.max(value, min), max);
+}
+
+function createStyles(colors: AppColors) {
+ return StyleSheet.create({
+ target: {
+ alignItems: 'center',
+ backgroundColor: colors.cardStrong,
+ borderColor: colors.line,
+ borderRadius: 4,
+ borderWidth: StyleSheet.hairlineWidth,
+ height: 220,
+ justifyContent: 'center',
+ overflow: 'hidden',
+ },
+ image: {
+ borderRadius: 4,
+ height: 160,
+ width: 240,
+ },
+ metrics: {
+ gap: 6,
+ },
+ metric: {
+ color: colors.textSoft,
+ fontSize: 13,
+ fontWeight: '600',
+ lineHeight: 18,
+ },
+ });
+}
diff --git a/examples/test-app/src/screens/HomeScreen.tsx b/examples/test-app/src/screens/HomeScreen.tsx
index 416250953..e89831b2a 100644
--- a/examples/test-app/src/screens/HomeScreen.tsx
+++ b/examples/test-app/src/screens/HomeScreen.tsx
@@ -2,6 +2,7 @@ import { Alert, ActivityIndicator, ScrollView, StyleSheet, Text, View } from 're
import { ActionButton, InlineBadge, ScreenTitle, SectionCard, ToggleRow } from '../components';
import { useAppColors, type AppColors } from '../theme';
+import { GestureLab } from './GestureLab';
export interface HomeScreenProps {
cartCount: number;
@@ -49,6 +50,8 @@ export function HomeScreen(props: HomeScreenProps) {
testID="home-title"
/>
+
+
{props.noticeVisible ? (
= 0 ? 1.0 : -1.0)
+ guard velocity.isFinite && velocity != 0 else {
+ return Response(ok: false, error: ErrorPayload(message: "rotateGesture velocity must be non-zero"))
+ }
+ var outcome = RunnerInteractionOutcome.performed
+ let timing = measureGesture {
+ outcome = rotateGesture(
+ app: activeApp,
+ degrees: degrees,
+ x: command.x,
+ y: command.y,
+ velocity: velocity
+ )
+ }
+ if let response = unsupportedResponse(for: outcome) {
+ return response
+ }
+ return Response(
+ ok: true,
+ data: DataPayload(
+ message: "rotatedGesture",
+ gestureStartUptimeMs: timing.gestureStartUptimeMs,
+ gestureEndUptimeMs: timing.gestureEndUptimeMs
+ )
+ )
+ case .transformGesture:
+ guard
+ let x = command.x,
+ let y = command.y,
+ let dx = command.dx,
+ let dy = command.dy,
+ x.isFinite,
+ y.isFinite,
+ dx.isFinite,
+ dy.isFinite
+ else {
+ return Response(ok: false, error: ErrorPayload(message: "transformGesture requires finite x y dx dy"))
+ }
+ guard let scale = command.scale, scale.isFinite, scale > 0 else {
+ return Response(ok: false, error: ErrorPayload(message: "transformGesture requires scale > 0"))
+ }
+ guard let degrees = command.degrees, degrees.isFinite else {
+ return Response(ok: false, error: ErrorPayload(message: "transformGesture requires finite degrees"))
+ }
+ let durationMs = command.durationMs ?? 300
+ guard durationMs.isFinite && durationMs >= 16 else {
+ return Response(ok: false, error: ErrorPayload(message: "transformGesture durationMs must be >= 16"))
+ }
+ var outcome = RunnerInteractionOutcome.performed
+ let timing = measureGesture {
+ outcome = transformGesture(
+ app: activeApp,
+ x: x,
+ y: y,
+ dx: dx,
+ dy: dy,
+ scale: scale,
+ degrees: degrees,
+ durationMs: durationMs
+ )
+ }
+ if let response = unsupportedResponse(for: outcome) {
+ return response
+ }
+ return Response(
+ ok: true,
+ data: DataPayload(
+ message: "transformedGesture",
+ gestureStartUptimeMs: timing.gestureStartUptimeMs,
+ gestureEndUptimeMs: timing.gestureEndUptimeMs
+ )
+ )
}
}
diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift
index 188897624..778b81a13 100644
--- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift
+++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift
@@ -1266,6 +1266,50 @@ extension RunnerTests {
return performCoordinatePinch(app: app, scale: scale, x: x, y: y)
}
+ func rotateGesture(app: XCUIApplication, degrees: Double, x: Double?, y: Double?, velocity: Double) -> RunnerInteractionOutcome {
+ return performCoordinateRotateGesture(app: app, degrees: degrees, x: x, y: y, velocity: velocity)
+ }
+
+ func transformGesture(
+ app: XCUIApplication,
+ x: Double,
+ y: Double,
+ dx: Double,
+ dy: Double,
+ scale: Double,
+ degrees: Double,
+ durationMs: Double
+ ) -> RunnerInteractionOutcome {
+#if os(iOS)
+ let holdDuration = max(0.02, min(durationMs / 1000.0, 10.0) / 3.0)
+ let panOutcome = performCoordinateDrag(
+ app: app,
+ x: x,
+ y: y,
+ x2: x + dx,
+ y2: y + dy,
+ holdDuration: holdDuration
+ )
+ guard case .performed = panOutcome else {
+ return panOutcome
+ }
+
+ let target = gestureElement(app: app, x: x, y: y)
+ target.pinch(withScale: CGFloat(scale), velocity: CGFloat(scale >= 1.0 ? 1.0 : -1.0))
+ return performCoordinateRotateGesture(
+ app: app,
+ degrees: degrees,
+ x: x,
+ y: y,
+ velocity: degrees >= 0 ? 1.0 : -1.0
+ )
+#elseif os(tvOS)
+ return .unsupported("transformGesture is not supported on tvOS")
+#else
+ return .unsupported("transformGesture is not supported on macOS")
+#endif
+ }
+
private func performCoordinatePinch(app: XCUIApplication, scale: Double, x: Double?, y: Double?) -> RunnerInteractionOutcome {
#if os(tvOS)
return .unsupported("pinch is not supported on tvOS")
@@ -1304,6 +1348,34 @@ extension RunnerTests {
#endif
}
+ private func performCoordinateRotateGesture(app: XCUIApplication, degrees: Double, x: Double?, y: Double?, velocity: Double) -> RunnerInteractionOutcome {
+#if os(iOS)
+ let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
+ let radians = CGFloat(degrees * .pi / 180.0)
+ target.rotate(radians, withVelocity: CGFloat(velocity))
+ return .performed
+#elseif os(tvOS)
+ return .unsupported("rotate-gesture is not supported on tvOS")
+#else
+ return .unsupported("rotate-gesture is not supported on macOS")
+#endif
+ }
+
+#if os(iOS)
+ private func gestureElement(app: XCUIApplication, x: Double, y: Double) -> XCUIElement {
+ let point = CGPoint(x: x, y: y)
+ let matches = app.descendants(matching: .any).allElementsBoundByIndex.filter { element in
+ element.exists && element.frame.contains(point) && !element.frame.isEmpty
+ }
+ if let smallest = matches.min(by: { left, right in
+ (left.frame.width * left.frame.height) < (right.frame.width * right.frame.height)
+ }) {
+ return smallest
+ }
+ return interactionRoot(app: app)
+ }
+#endif
+
private func interactionRoot(app: XCUIApplication) -> XCUIElement {
let windows = app.windows.allElementsBoundByIndex
if let window = windows.first(where: { $0.exists && !$0.frame.isEmpty }) {
diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift
index 87a5036c0..d10dfbd22 100644
--- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift
+++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift
@@ -248,7 +248,9 @@ extension RunnerTests {
.rotate,
.appSwitcher,
.keyboardDismiss,
- .pinch:
+ .pinch,
+ .rotateGesture,
+ .transformGesture:
return true
default:
return false
diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift
index 59ccb88a6..de5fa632f 100644
--- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift
+++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift
@@ -25,6 +25,8 @@ enum CommandType: String, Codable {
case keyboardDismiss
case alert
case pinch
+ case rotateGesture
+ case transformGesture
case recordStart
case recordStop
case uptime
@@ -52,10 +54,14 @@ struct Command: Codable {
let pattern: String?
let x2: Double?
let y2: Double?
+ let dx: Double?
+ let dy: Double?
let durationMs: Double?
let direction: String?
let orientation: String?
let scale: Double?
+ let degrees: Double?
+ let velocity: Double?
let outPath: String?
let fps: Int?
let quality: Int?
diff --git a/package.json b/package.json
index a3379e89f..a1ee3c3c5 100644
--- a/package.json
+++ b/package.json
@@ -90,6 +90,9 @@
"build:android-snapshot-helper": "sh ./scripts/build-android-snapshot-helper.sh $(node -p \"require('./package.json').version\") .tmp/android-snapshot-helper",
"package:android-snapshot-helper": "sh ./scripts/package-android-snapshot-helper.sh $(node -p \"require('./package.json').version\") v$(node -p \"require('./package.json').version\") .tmp/android-snapshot-helper",
"package:android-snapshot-helper:npm": "rm -rf android-snapshot-helper/dist && sh ./scripts/package-android-snapshot-helper.sh $(node -p \"require('./package.json').version\") v$(node -p \"require('./package.json').version\") android-snapshot-helper/dist",
+ "build:android-multitouch-helper": "sh ./scripts/build-android-multitouch-helper.sh $(node -p \"require('./package.json').version\") .tmp/android-multitouch-helper",
+ "package:android-multitouch-helper": "sh ./scripts/package-android-multitouch-helper.sh $(node -p \"require('./package.json').version\") .tmp/android-multitouch-helper",
+ "package:android-multitouch-helper:npm": "rm -rf android-multitouch-helper/dist && sh ./scripts/package-android-multitouch-helper.sh $(node -p \"require('./package.json').version\") android-multitouch-helper/dist",
"build:macos-helper": "swift build -c release --package-path macos-helper",
"build:all": "pnpm build:node && pnpm build:xcuitest",
"ad": "node bin/agent-device.mjs",
@@ -104,7 +107,7 @@
"check:tooling": "pnpm lint && pnpm typecheck && pnpm check:mcp-metadata && pnpm build",
"check:unit": "pnpm test:unit && pnpm test:smoke",
"check": "pnpm check:tooling && pnpm check:fallow && pnpm check:unit",
- "prepack": "pnpm check:mcp-metadata && pnpm build:all && pnpm package:android-snapshot-helper:npm",
+ "prepack": "pnpm check:mcp-metadata && pnpm build:all && pnpm package:android-snapshot-helper:npm && pnpm package:android-multitouch-helper:npm",
"typecheck": "tsc -p tsconfig.json",
"test-app:install": "pnpm install --dir examples/test-app --ignore-workspace",
"test-app:start": "pnpm --dir examples/test-app start",
@@ -145,6 +148,8 @@
"!macos-helper/**/.build",
"android-snapshot-helper/dist",
"!android-snapshot-helper/dist/*.idsig",
+ "android-multitouch-helper/dist",
+ "!android-multitouch-helper/dist/*.idsig",
"src/platforms/linux/atspi-dump.py",
"skills",
"server.json",
diff --git a/scripts/build-android-multitouch-helper.sh b/scripts/build-android-multitouch-helper.sh
new file mode 100644
index 000000000..eafb24ae8
--- /dev/null
+++ b/scripts/build-android-multitouch-helper.sh
@@ -0,0 +1,100 @@
+#!/bin/sh
+set -eu
+
+if [ "$#" -ne 2 ]; then
+ echo "Usage: $0 " >&2
+ exit 1
+fi
+
+VERSION="$1"
+OUTPUT_DIR="$2"
+PROJECT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
+HELPER_DIR="$PROJECT_DIR/android-multitouch-helper"
+PACKAGE_NAME="com.callstack.agentdevice.multitouchhelper"
+MIN_SDK=23
+TARGET_SDK=36
+APK_BASENAME="agent-device-android-multitouch-helper-$VERSION.apk"
+
+SDK_ROOT="${ANDROID_HOME:-${ANDROID_SDK_ROOT:-}}"
+if [ -z "$SDK_ROOT" ] || [ ! -d "$SDK_ROOT" ]; then
+ echo "ANDROID_HOME or ANDROID_SDK_ROOT must point to an Android SDK" >&2
+ exit 1
+fi
+
+ANDROID_JAR="$SDK_ROOT/platforms/android-$TARGET_SDK/android.jar"
+if [ ! -f "$ANDROID_JAR" ]; then
+ echo "Missing Android platform jar: $ANDROID_JAR" >&2
+ exit 1
+fi
+
+BUILD_TOOLS_DIR="$(
+ find "$SDK_ROOT/build-tools" -maxdepth 1 -mindepth 1 -type d 2>/dev/null | sort -V | tail -n 1
+)"
+if [ -z "$BUILD_TOOLS_DIR" ] || [ ! -x "$BUILD_TOOLS_DIR/aapt2" ]; then
+ echo "Missing Android build tools under $SDK_ROOT/build-tools" >&2
+ exit 1
+fi
+
+VERSION_CODE="$(
+ printf '%s\n' "$VERSION" | awk -F. '
+ /^[0-9]+[.][0-9]+[.][0-9]+$/ {
+ print ($1 * 1000000) + ($2 * 1000) + $3
+ next
+ }
+ { print 1 }
+ '
+)"
+
+BUILD_DIR="$HELPER_DIR/build"
+CLASSES_DIR="$BUILD_DIR/classes"
+DEX_DIR="$BUILD_DIR/dex"
+KEYSTORE="$PROJECT_DIR/android-snapshot-helper/debug.keystore"
+UNSIGNED_APK="$BUILD_DIR/helper-unsigned.apk"
+ALIGNED_APK="$BUILD_DIR/helper-aligned.apk"
+APK_PATH="$OUTPUT_DIR/$APK_BASENAME"
+
+rm -rf "$BUILD_DIR"
+mkdir -p "$CLASSES_DIR" "$DEX_DIR" "$OUTPUT_DIR"
+
+javac \
+ --release 11 \
+ -classpath "$ANDROID_JAR" \
+ -d "$CLASSES_DIR" \
+ $(find "$HELPER_DIR/src/main/java" -name '*.java' | sort)
+
+"$BUILD_TOOLS_DIR/d8" \
+ --min-api "$MIN_SDK" \
+ --classpath "$ANDROID_JAR" \
+ --output "$DEX_DIR" \
+ $(find "$CLASSES_DIR" -name '*.class' | sort)
+
+"$BUILD_TOOLS_DIR/aapt2" link \
+ --manifest "$HELPER_DIR/AndroidManifest.xml" \
+ -I "$ANDROID_JAR" \
+ --min-sdk-version "$MIN_SDK" \
+ --target-sdk-version "$TARGET_SDK" \
+ --version-code "$VERSION_CODE" \
+ --version-name "$VERSION" \
+ -o "$UNSIGNED_APK"
+
+zip -q -j "$UNSIGNED_APK" "$DEX_DIR/classes.dex"
+
+"$BUILD_TOOLS_DIR/zipalign" -f 4 "$UNSIGNED_APK" "$ALIGNED_APK"
+
+if [ ! -f "$KEYSTORE" ]; then
+ echo "Missing Android helper signing keystore: $KEYSTORE" >&2
+ exit 1
+fi
+
+"$BUILD_TOOLS_DIR/apksigner" sign \
+ --ks "$KEYSTORE" \
+ --ks-pass pass:android \
+ --key-pass pass:android \
+ --out "$APK_PATH" \
+ "$ALIGNED_APK"
+
+"$BUILD_TOOLS_DIR/apksigner" verify --min-sdk-version "$MIN_SDK" "$APK_PATH"
+
+printf 'apk=%s\n' "$APK_PATH"
+printf 'package=%s\n' "$PACKAGE_NAME"
+printf 'version_code=%s\n' "$VERSION_CODE"
diff --git a/scripts/integration-progress.mjs b/scripts/integration-progress.mjs
index 52e2ac35b..6930416e4 100644
--- a/scripts/integration-progress.mjs
+++ b/scripts/integration-progress.mjs
@@ -489,7 +489,7 @@ function summarizeCommandFamilyOwnership(files) {
commands: ['snapshot', 'diff', 'screenshot'],
},
{
- name: 'press/click/fill/type/scroll/swipe/pinch/rotate/app-switcher',
+ name: 'press/click/fill/type/scroll/swipe/gesture/rotate/app-switcher',
commands: [
'press',
'click',
@@ -497,9 +497,9 @@ function summarizeCommandFamilyOwnership(files) {
'longpress',
'swipe',
'scroll',
+ 'gesture',
'type',
'fill',
- 'pinch',
'rotate',
'app-switcher',
'back',
@@ -610,7 +610,10 @@ function extractProviderScenarioCommandReferences(text) {
['interactions.get', 'get'],
['interactions.is', 'is'],
['interactions.longPress', 'longpress'],
- ['interactions.pinch', 'pinch'],
+ ['interactions.pan', 'gesture'],
+ ['interactions.fling', 'gesture'],
+ ['interactions.pinch', 'gesture'],
+ ['interactions.rotateGesture', 'gesture'],
['interactions.press', 'press'],
['interactions.scroll', 'scroll'],
['interactions.swipe', 'swipe'],
diff --git a/scripts/package-android-multitouch-helper.sh b/scripts/package-android-multitouch-helper.sh
new file mode 100644
index 000000000..f10adbebc
--- /dev/null
+++ b/scripts/package-android-multitouch-helper.sh
@@ -0,0 +1,63 @@
+#!/bin/sh
+set -eu
+
+if [ "$#" -ne 2 ]; then
+ echo "Usage: $0 " >&2
+ exit 1
+fi
+
+VERSION="$1"
+OUTPUT_DIR="$2"
+PROJECT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
+PACKAGE_NAME="com.callstack.agentdevice.multitouchhelper"
+INSTRUMENTATION_RUNNER="$PACKAGE_NAME/.MultiTouchInstrumentation"
+APK_BASENAME="agent-device-android-multitouch-helper-$VERSION.apk"
+CHECKSUM_BASENAME="$APK_BASENAME.sha256"
+MANIFEST_BASENAME="agent-device-android-multitouch-helper-$VERSION.manifest.json"
+
+write_github_output() {
+ if [ -n "${GITHUB_OUTPUT:-}" ]; then
+ printf '%s\n' "$1" >> "$GITHUB_OUTPUT"
+ fi
+}
+
+mkdir -p "$OUTPUT_DIR"
+
+BUILD_OUTPUT="$(sh "$PROJECT_DIR/scripts/build-android-multitouch-helper.sh" "$VERSION" "$OUTPUT_DIR")"
+APK_PATH="$(printf '%s\n' "$BUILD_OUTPUT" | awk -F= '$1 == "apk" { print $2 }')"
+VERSION_CODE="$(printf '%s\n' "$BUILD_OUTPUT" | awk -F= '$1 == "version_code" { print $2 }')"
+CHECKSUM_PATH="$OUTPUT_DIR/$CHECKSUM_BASENAME"
+MANIFEST_PATH="$OUTPUT_DIR/$MANIFEST_BASENAME"
+
+if [ ! -f "$APK_PATH" ]; then
+ echo "Helper APK was not created at $APK_PATH" >&2
+ exit 1
+fi
+
+SHA256="$(shasum -a 256 "$APK_PATH" | awk '{print $1}')"
+printf '%s %s\n' "$SHA256" "$APK_BASENAME" > "$CHECKSUM_PATH"
+
+{
+ printf '{\n'
+ printf ' "name": "android-multitouch-helper",\n'
+ printf ' "version": "%s",\n' "$VERSION"
+ printf ' "assetName": "%s",\n' "$APK_BASENAME"
+ printf ' "sha256": "%s",\n' "$SHA256"
+ printf ' "packageName": "%s",\n' "$PACKAGE_NAME"
+ printf ' "versionCode": %s,\n' "$VERSION_CODE"
+ printf ' "instrumentationRunner": "%s",\n' "$INSTRUMENTATION_RUNNER"
+ printf ' "statusProtocol": "android-multitouch-helper-v1"\n'
+ printf '}\n'
+} > "$MANIFEST_PATH"
+
+write_github_output "apk_path=$APK_PATH"
+write_github_output "checksum_path=$CHECKSUM_PATH"
+write_github_output "manifest_path=$MANIFEST_PATH"
+write_github_output "apk_name=$APK_BASENAME"
+write_github_output "sha256=$SHA256"
+write_github_output "package_name=$PACKAGE_NAME"
+write_github_output "version_code=$VERSION_CODE"
+
+printf 'apk=%s\n' "$APK_PATH"
+printf 'checksum=%s\n' "$CHECKSUM_PATH"
+printf 'manifest=%s\n' "$MANIFEST_PATH"
diff --git a/src/__tests__/client.test.ts b/src/__tests__/client.test.ts
index b70cd9c94..1ed0a2174 100644
--- a/src/__tests__/client.test.ts
+++ b/src/__tests__/client.test.ts
@@ -322,6 +322,31 @@ test('apps.installFromSource forwards GitHub Actions artifact sources unchanged'
});
});
+test('interactions.rotateGesture serializes a complete center without undefined literals', async () => {
+ const setup = createTransport(async () => ({ ok: true, data: {} }));
+ const client = createAgentDeviceClient(setup.config, { transport: setup.transport });
+
+ await client.interactions.rotateGesture({ degrees: 35, x: 200, y: 420 });
+
+ assert.deepEqual(setup.calls[0]?.positionals, ['rotate', '35', '200', '420']);
+});
+
+test('interactions.rotateGesture rejects partial centers on the client side', async () => {
+ const setup = createTransport(async () => {
+ throw new Error('transport should not run for invalid input');
+ });
+ const client = createAgentDeviceClient(setup.config, { transport: setup.transport });
+
+ await assert.rejects(
+ () => client.interactions.rotateGesture({ degrees: 35, x: 200 }),
+ (error: unknown) =>
+ error instanceof AppError &&
+ error.code === 'INVALID_ARGS' &&
+ error.message === 'gesture rotate center requires both x and y',
+ );
+ assert.equal(setup.calls.length, 0);
+});
+
test('apps.list forwards filters and returns daemon app names', async () => {
const setup = createTransport(async () => ({
ok: true,
diff --git a/src/cli/commands/generic.ts b/src/cli/commands/generic.ts
index fb44325a4..b75902085 100644
--- a/src/cli/commands/generic.ts
+++ b/src/cli/commands/generic.ts
@@ -16,7 +16,7 @@ import {
import { selectorSnapshotOptionsFromFlags } from '../../command-codecs/flags.ts';
import { buildSelectionOptions } from './shared.ts';
import { writeCommandCliOutput } from './output.ts';
-import type { PublicCommandName } from '../../command-catalog.ts';
+import { GESTURE_SUBCOMMAND_ERROR, type PublicCommandName } from '../../command-catalog.ts';
import type { ClientCommandHandler } from './router-types.ts';
type GenericClientCommandRunner = (params: {
@@ -111,6 +111,12 @@ const genericClientCommandRunners = {
pauseMs: flags.pauseMs,
pattern: flags.pattern,
}),
+ gesture: ({ client, positionals, flags }) =>
+ runGestureCommand({
+ client,
+ positionals,
+ flags,
+ }),
focus: ({ client, positionals, flags }) =>
client.interactions.focus({
...buildSelectionOptions(flags),
@@ -135,13 +141,6 @@ const genericClientCommandRunners = {
amount: optionalNumber(positionals[1]),
pixels: flags.pixels,
}),
- pinch: ({ client, positionals, flags }) =>
- client.interactions.pinch({
- ...buildSelectionOptions(flags),
- scale: Number(positionals[0]),
- x: optionalNumber(positionals[1]),
- y: optionalNumber(positionals[2]),
- }),
'trigger-app-event': ({ client, positionals, flags }) =>
client.apps.triggerEvent({
...buildSelectionOptions(flags),
@@ -192,6 +191,64 @@ const genericClientCommandRunners = {
client.settings.update(settingsCommandCodec.decode(positionals, flags)),
} satisfies Partial>;
+function runGestureCommand(params: {
+ client: AgentDeviceClient;
+ positionals: string[];
+ flags: CliFlags;
+}): Promise {
+ const { client, positionals, flags } = params;
+ const subcommand = required(positionals[0], 'gesture requires subcommand');
+ const args = positionals.slice(1);
+ switch (subcommand) {
+ case 'pan':
+ return client.interactions.pan({
+ ...buildSelectionOptions(flags),
+ x: Number(args[0]),
+ y: Number(args[1]),
+ dx: Number(args[2]),
+ dy: Number(args[3]),
+ durationMs: optionalNumber(args[4]),
+ });
+ case 'fling':
+ return client.interactions.fling({
+ ...buildSelectionOptions(flags),
+ direction: readGestureDirection(args[0], 'gesture fling'),
+ x: Number(args[1]),
+ y: Number(args[2]),
+ distance: optionalNumber(args[3]),
+ durationMs: optionalNumber(args[4]),
+ });
+ case 'pinch':
+ return client.interactions.pinch({
+ ...buildSelectionOptions(flags),
+ scale: Number(args[0]),
+ x: optionalNumber(args[1]),
+ y: optionalNumber(args[2]),
+ });
+ case 'rotate':
+ return client.interactions.rotateGesture({
+ ...buildSelectionOptions(flags),
+ degrees: Number(args[0]),
+ x: optionalNumber(args[1]),
+ y: optionalNumber(args[2]),
+ velocity: optionalNumber(args[3]),
+ });
+ case 'transform':
+ return client.interactions.transformGesture({
+ ...buildSelectionOptions(flags),
+ x: Number(args[0]),
+ y: Number(args[1]),
+ dx: Number(args[2]),
+ dy: Number(args[3]),
+ scale: Number(args[4]),
+ degrees: Number(args[5]),
+ durationMs: optionalNumber(args[6]),
+ });
+ default:
+ throw new AppError('INVALID_ARGS', GESTURE_SUBCOMMAND_ERROR);
+ }
+}
+
export const genericClientCommandHandlers = Object.fromEntries(
Object.entries(genericClientCommandRunners).map(([command, run]) => [
command,
@@ -237,6 +294,14 @@ function readScrollDirection(
throw new AppError('INVALID_ARGS', `Unknown direction: ${String(value)}`);
}
+function readGestureDirection(
+ value: string | undefined,
+ command: string,
+): 'up' | 'down' | 'left' | 'right' {
+ if (value === 'up' || value === 'down' || value === 'left' || value === 'right') return value;
+ throw new AppError('INVALID_ARGS', `${command} direction must be up, down, left, or right`);
+}
+
function readStartStop(value: string | undefined, command: string): 'start' | 'stop' {
if (value === 'start' || value === 'stop') return value;
throw new AppError('INVALID_ARGS', `${command} requires start|stop`);
diff --git a/src/client-types.ts b/src/client-types.ts
index 20e227cfd..2a44be877 100644
--- a/src/client-types.ts
+++ b/src/client-types.ts
@@ -577,6 +577,22 @@ export type SwipeOptions = ClientCommandBaseOptions & {
pattern?: 'one-way' | 'ping-pong';
};
+export type PanOptions = ClientCommandBaseOptions & {
+ x: number;
+ y: number;
+ dx: number;
+ dy: number;
+ durationMs?: number;
+};
+
+export type FlingOptions = ClientCommandBaseOptions & {
+ direction: 'up' | 'down' | 'left' | 'right';
+ x: number;
+ y: number;
+ distance?: number;
+ durationMs?: number;
+};
+
export type FocusOptions = ClientCommandBaseOptions & {
x: number;
y: number;
@@ -606,6 +622,23 @@ export type PinchOptions = ClientCommandBaseOptions & {
y?: number;
};
+export type RotateGestureOptions = ClientCommandBaseOptions & {
+ degrees: number;
+ x?: number;
+ y?: number;
+ velocity?: number;
+};
+
+export type TransformGestureOptions = ClientCommandBaseOptions & {
+ x: number;
+ y: number;
+ dx: number;
+ dy: number;
+ scale: number;
+ degrees: number;
+ durationMs?: number;
+};
+
export type GetOptions = ClientCommandBaseOptions &
SelectorSnapshotCommandOptions &
ElementTarget & {
@@ -876,11 +909,15 @@ export type AgentDeviceClient = {
press: (options: PressOptions) => Promise;
longPress: (options: LongPressOptions) => Promise;
swipe: (options: SwipeOptions) => Promise;
+ pan: (options: PanOptions) => Promise;
+ fling: (options: FlingOptions) => Promise;
focus: (options: FocusOptions) => Promise;
type: (options: TypeTextOptions) => Promise;
fill: (options: FillOptions) => Promise;
scroll: (options: ScrollOptions) => Promise;
pinch: (options: PinchOptions) => Promise;
+ rotateGesture: (options: RotateGestureOptions) => Promise;
+ transformGesture: (options: TransformGestureOptions) => Promise;
get: (options: GetOptions) => Promise;
is: (options: IsOptions) => Promise;
find: (options: FindOptions) => Promise;
diff --git a/src/client.ts b/src/client.ts
index 16b0681f6..3d2f5e1ad 100644
--- a/src/client.ts
+++ b/src/client.ts
@@ -52,6 +52,7 @@ import type {
MetroPrepareOptions,
NetworkOptions,
} from './client-types.ts';
+import { AppError } from './utils/errors.ts';
export function createAgentDeviceClient(
config: AgentDeviceClientConfig = {},
@@ -331,6 +332,35 @@ export function createAgentDeviceClient(
],
options,
),
+ pan: async (options) =>
+ await executeCommandRequest(
+ PUBLIC_COMMANDS.gesture,
+ [
+ 'pan',
+ String(options.x),
+ String(options.y),
+ String(options.dx),
+ String(options.dy),
+ ...optionalNumber(options.durationMs),
+ ],
+ options,
+ ),
+ fling: async (options) => {
+ const distance =
+ options.durationMs !== undefined ? (options.distance ?? 180) : options.distance;
+ return await executeCommandRequest(
+ PUBLIC_COMMANDS.gesture,
+ [
+ 'fling',
+ options.direction,
+ String(options.x),
+ String(options.y),
+ ...optionalNumber(distance),
+ ...optionalNumber(options.durationMs),
+ ],
+ options,
+ );
+ },
focus: async (options) =>
await executeCommandRequest(
PUBLIC_COMMANDS.focus,
@@ -357,8 +387,45 @@ export function createAgentDeviceClient(
),
pinch: async (options) =>
await executeCommandRequest(
- PUBLIC_COMMANDS.pinch,
- [String(options.scale), ...optionalNumber(options.x), ...optionalNumber(options.y)],
+ PUBLIC_COMMANDS.gesture,
+ [
+ 'pinch',
+ String(options.scale),
+ ...optionalNumber(options.x),
+ ...optionalNumber(options.y),
+ ],
+ options,
+ ),
+ rotateGesture: async (options) => {
+ if (
+ (options.x === undefined && options.y !== undefined) ||
+ (options.x !== undefined && options.y === undefined)
+ ) {
+ throw new AppError('INVALID_ARGS', 'gesture rotate center requires both x and y');
+ }
+ const center =
+ options.x === undefined || options.y === undefined
+ ? []
+ : [String(options.x), String(options.y)];
+ return await executeCommandRequest(
+ PUBLIC_COMMANDS.gesture,
+ ['rotate', String(options.degrees), ...center, ...optionalNumber(options.velocity)],
+ options,
+ );
+ },
+ transformGesture: async (options) =>
+ await executeCommandRequest(
+ PUBLIC_COMMANDS.gesture,
+ [
+ 'transform',
+ String(options.x),
+ String(options.y),
+ String(options.dx),
+ String(options.dy),
+ String(options.scale),
+ String(options.degrees),
+ ...optionalNumber(options.durationMs),
+ ],
options,
),
get: async (options) =>
diff --git a/src/command-catalog.ts b/src/command-catalog.ts
index 86a67f08c..9279926ac 100644
--- a/src/command-catalog.ts
+++ b/src/command-catalog.ts
@@ -14,6 +14,7 @@ export const PUBLIC_COMMANDS = {
fill: 'fill',
find: 'find',
focus: 'focus',
+ gesture: 'gesture',
get: 'get',
home: 'home',
install: 'install',
@@ -25,7 +26,6 @@ export const PUBLIC_COMMANDS = {
network: 'network',
open: 'open',
perf: 'perf',
- pinch: 'pinch',
press: 'press',
push: 'push',
record: 'record',
@@ -54,6 +54,9 @@ export const INTERNAL_COMMANDS = {
sessionList: 'session_list',
} as const;
+const GESTURE_SUBCOMMANDS = ['pan', 'fling', 'pinch', 'rotate', 'transform'] as const;
+export const GESTURE_SUBCOMMAND_ERROR = `gesture requires one of: ${GESTURE_SUBCOMMANDS.join(', ')}`;
+
export type PublicCommandName = (typeof PUBLIC_COMMANDS)[keyof typeof PUBLIC_COMMANDS];
export type CliCommandName =
| PublicCommandName
@@ -88,12 +91,13 @@ export const DAEMON_COMMAND_GROUPS = {
PUBLIC_COMMANDS.diff,
PUBLIC_COMMANDS.fill,
PUBLIC_COMMANDS.find,
+ PUBLIC_COMMANDS.gesture,
PUBLIC_COMMANDS.get,
PUBLIC_COMMANDS.home,
PUBLIC_COMMANDS.is,
PUBLIC_COMMANDS.keyboard,
PUBLIC_COMMANDS.longPress,
- PUBLIC_COMMANDS.pinch,
+ 'pinch',
PUBLIC_COMMANDS.press,
PUBLIC_COMMANDS.record,
PUBLIC_COMMANDS.reactNative,
diff --git a/src/core/__tests__/capabilities.test.ts b/src/core/__tests__/capabilities.test.ts
index bbe6bcffa..a846894b3 100644
--- a/src/core/__tests__/capabilities.test.ts
+++ b/src/core/__tests__/capabilities.test.ts
@@ -81,7 +81,7 @@ test('device capability matrix stays consistent across shared command groups', (
checks: [
{ device: iosSimulator, expected: true, label: 'on iOS sim' },
{ device: iosDevice, expected: false, label: 'on iOS device' },
- { device: androidDevice, expected: false, label: 'on Android' },
+ { device: androidDevice, expected: true, label: 'on Android' },
{ device: macOsDevice, expected: true, label: 'on macOS' },
],
},
@@ -138,6 +138,46 @@ test('device capability matrix stays consistent across shared command groups', (
{ device: macOsDevice, expected: true, label: 'on macOS' },
],
},
+ {
+ commands: ['pan'],
+ checks: [
+ { device: iosSimulator, expected: true, label: 'on iOS sim' },
+ { device: iosDevice, expected: true, label: 'on iOS device' },
+ { device: androidDevice, expected: true, label: 'on Android' },
+ { device: macOsDevice, expected: true, label: 'on macOS' },
+ { device: linuxDevice, expected: true, label: 'on Linux' },
+ ],
+ },
+ {
+ commands: ['fling'],
+ checks: [
+ { device: iosSimulator, expected: true, label: 'on iOS sim' },
+ { device: iosDevice, expected: true, label: 'on iOS device' },
+ { device: androidDevice, expected: true, label: 'on Android' },
+ { device: macOsDevice, expected: true, label: 'on macOS' },
+ { device: linuxDevice, expected: false, label: 'on Linux' },
+ ],
+ },
+ {
+ commands: ['rotate-gesture'],
+ checks: [
+ { device: iosSimulator, expected: true, label: 'on iOS sim' },
+ { device: iosDevice, expected: false, label: 'on iOS device' },
+ { device: androidDevice, expected: true, label: 'on Android' },
+ { device: macOsDevice, expected: false, label: 'on macOS' },
+ { device: tvOsSimulator, expected: false, label: 'on tvOS simulator' },
+ ],
+ },
+ {
+ commands: ['transform-gesture'],
+ checks: [
+ { device: iosSimulator, expected: true, label: 'on iOS sim' },
+ { device: iosDevice, expected: false, label: 'on iOS device' },
+ { device: androidDevice, expected: true, label: 'on Android' },
+ { device: macOsDevice, expected: false, label: 'on macOS' },
+ { device: tvOsSimulator, expected: false, label: 'on tvOS simulator' },
+ ],
+ },
];
for (const scenario of scenarios) {
diff --git a/src/core/__tests__/dispatch-interactions.test.ts b/src/core/__tests__/dispatch-interactions.test.ts
index c6fdcc5ec..08c35190b 100644
--- a/src/core/__tests__/dispatch-interactions.test.ts
+++ b/src/core/__tests__/dispatch-interactions.test.ts
@@ -1,8 +1,19 @@
import { test, vi } from 'vitest';
import assert from 'node:assert/strict';
-import { handlePressCommand } from '../dispatch-interactions.ts';
+import {
+ handleFlingCommand,
+ handlePanCommand,
+ handlePinchCommand,
+ handlePressCommand,
+ handleRotateGestureCommand,
+ handleTransformGestureCommand,
+} from '../dispatch-interactions.ts';
import type { Interactor } from '../interactor-types.ts';
-import { MACOS_DEVICE } from '../../__tests__/test-utils/device-fixtures.ts';
+import {
+ ANDROID_EMULATOR,
+ IOS_SIMULATOR,
+ MACOS_DEVICE,
+} from '../../__tests__/test-utils/device-fixtures.ts';
vi.mock('../../platforms/ios/macos-helper.ts', async (importOriginal) => {
const actual = await importOriginal();
@@ -25,16 +36,21 @@ function makeUnusedInteractor(): Interactor {
tap: fail,
doubleTap: fail,
swipe: fail,
+ pan: fail,
+ fling: fail,
longPress: fail,
focus: fail,
type: fail,
fill: fail,
scroll: fail,
+ pinch: fail,
screenshot: fail,
snapshot: fail,
back: fail,
home: fail,
rotate: fail,
+ rotateGesture: fail,
+ transformGesture: fail,
appSwitcher: fail,
readClipboard: fail,
writeClipboard: fail,
@@ -63,3 +79,215 @@ test('handlePressCommand routes macOS menubar press through the helper', async (
{ bundleId: 'com.example.menubarapp', surface: 'menubar' },
]);
});
+
+test('handlePanCommand preserves the requested drag duration and moves by delta', async () => {
+ const calls: unknown[][] = [];
+ const interactor = {
+ ...makeUnusedInteractor(),
+ pan: async (...args: unknown[]) => {
+ calls.push(args);
+ },
+ };
+
+ const result = await handlePanCommand(interactor, ['200', '420', '0', '-80', '500']);
+
+ assert.deepEqual(calls, [[200, 420, 200, 340, 500]]);
+ assert.deepEqual(result, {
+ x: 200,
+ y: 420,
+ dx: 0,
+ dy: -80,
+ x2: 200,
+ y2: 340,
+ durationMs: 500,
+ message: 'Panned (200, 420) by (0, -80)',
+ });
+});
+
+test('handleFlingCommand converts direction and distance into a short drag', async () => {
+ const calls: unknown[][] = [];
+ const interactor = {
+ ...makeUnusedInteractor(),
+ fling: async (...args: unknown[]) => {
+ calls.push(args);
+ },
+ };
+
+ const result = await handleFlingCommand(interactor, ['right', '200', '420', '180']);
+
+ assert.deepEqual(calls, [[200, 420, 380, 420, 50]]);
+ assert.deepEqual(result, {
+ direction: 'right',
+ x: 200,
+ y: 420,
+ x2: 380,
+ y2: 420,
+ distance: 180,
+ durationMs: 50,
+ message: 'Flung right',
+ });
+});
+
+test('handlePinchCommand routes Android through the interactor', async () => {
+ const calls: unknown[][] = [];
+ const interactor = {
+ ...makeUnusedInteractor(),
+ pinch: async (...args: unknown[]) => {
+ calls.push(args);
+ return { backend: 'android-multitouch-helper' };
+ },
+ };
+
+ const result = await handlePinchCommand(
+ ANDROID_EMULATOR,
+ interactor,
+ ['2', '200', '420'],
+ undefined,
+ );
+
+ assert.deepEqual(calls, [[2, 200, 420]]);
+ assert.deepEqual(result, {
+ scale: 2,
+ x: 200,
+ y: 420,
+ backend: 'android-multitouch-helper',
+ message: 'Pinched to scale 2',
+ });
+});
+
+test('handleRotateGestureCommand defaults velocity sign to match degrees', async () => {
+ const calls: unknown[][] = [];
+ const interactor = {
+ ...makeUnusedInteractor(),
+ rotateGesture: async (...args: unknown[]) => {
+ calls.push(args);
+ },
+ };
+
+ const result = await handleRotateGestureCommand(IOS_SIMULATOR, interactor, [
+ '-215',
+ '200',
+ '420',
+ ]);
+
+ assert.deepEqual(calls, [[-215, 200, 420, -1]]);
+ assert.deepEqual(result, {
+ degrees: -215,
+ x: 200,
+ y: 420,
+ velocity: -1,
+ message: 'Rotated gesture -215 degrees',
+ });
+});
+
+test('handleRotateGestureCommand keeps direction owned by degrees when velocity sign conflicts', async () => {
+ const calls: unknown[][] = [];
+ const interactor = {
+ ...makeUnusedInteractor(),
+ rotateGesture: async (...args: unknown[]) => {
+ calls.push(args);
+ },
+ };
+
+ const result = await handleRotateGestureCommand(IOS_SIMULATOR, interactor, [
+ '145',
+ '200',
+ '420',
+ '-2',
+ ]);
+
+ assert.deepEqual(calls, [[145, 200, 420, 2]]);
+ assert.equal(result.velocity, 2);
+});
+
+test('handleRotateGestureCommand routes Android through the interactor', async () => {
+ const calls: unknown[][] = [];
+ const interactor = {
+ ...makeUnusedInteractor(),
+ rotateGesture: async (...args: unknown[]) => {
+ calls.push(args);
+ return { backend: 'android-multitouch-helper' };
+ },
+ };
+
+ const result = await handleRotateGestureCommand(ANDROID_EMULATOR, interactor, ['145']);
+
+ assert.deepEqual(calls, [[145, undefined, undefined, 1]]);
+ assert.deepEqual(result, {
+ degrees: 145,
+ velocity: 1,
+ backend: 'android-multitouch-helper',
+ message: 'Rotated gesture 145 degrees',
+ });
+});
+
+test('handleTransformGestureCommand routes Android through the interactor', async () => {
+ const calls: unknown[][] = [];
+ const interactor = {
+ ...makeUnusedInteractor(),
+ transformGesture: async (...args: unknown[]) => {
+ calls.push(args);
+ return { backend: 'android-multitouch-helper' };
+ },
+ };
+
+ const result = await handleTransformGestureCommand(ANDROID_EMULATOR, interactor, [
+ '200',
+ '420',
+ '80',
+ '-40',
+ '2',
+ '35',
+ '700',
+ ]);
+
+ assert.deepEqual(calls, [
+ [{ x: 200, y: 420, dx: 80, dy: -40, scale: 2, degrees: 35, durationMs: 700 }],
+ ]);
+ assert.deepEqual(result, {
+ x: 200,
+ y: 420,
+ dx: 80,
+ dy: -40,
+ scale: 2,
+ degrees: 35,
+ durationMs: 700,
+ backend: 'android-multitouch-helper',
+ message: 'Requested transform gesture by (80, -40), scale 2, rotate 35 degrees',
+ });
+});
+
+test('handleTransformGestureCommand routes iOS simulator through the interactor', async () => {
+ const calls: unknown[][] = [];
+ const interactor = {
+ ...makeUnusedInteractor(),
+ transformGesture: async (...args: unknown[]) => {
+ calls.push(args);
+ return { backend: 'xctest' };
+ },
+ };
+
+ const result = await handleTransformGestureCommand(IOS_SIMULATOR, interactor, [
+ '200',
+ '420',
+ '80',
+ '-40',
+ '2',
+ '35',
+ ]);
+
+ assert.deepEqual(calls, [
+ [{ x: 200, y: 420, dx: 80, dy: -40, scale: 2, degrees: 35, durationMs: undefined }],
+ ]);
+ assert.deepEqual(result, {
+ x: 200,
+ y: 420,
+ dx: 80,
+ dy: -40,
+ scale: 2,
+ degrees: 35,
+ durationMs: undefined,
+ backend: 'xctest',
+ message: 'Requested transform gesture by (80, -40), scale 2, rotate 35 degrees',
+ });
+});
diff --git a/src/core/capabilities.ts b/src/core/capabilities.ts
index c0cfd5e88..b96573010 100644
--- a/src/core/capabilities.ts
+++ b/src/core/capabilities.ts
@@ -24,6 +24,8 @@ const isMacOsOrAppleSimulator = (device: DeviceInfo): boolean =>
device.platform === 'macos' || device.kind === 'simulator';
const isMacOsOrMobileAppleSimulator = (device: DeviceInfo): boolean =>
device.platform === 'macos' || (device.kind === 'simulator' && device.target !== 'tv');
+const isIosMobileSimulator = (device: DeviceInfo): boolean =>
+ device.platform === 'ios' && device.kind === 'simulator' && device.target !== 'tv';
// Linux desktop supports these commands via xdotool/ydotool + AT-SPI2.
// Linux device kind is always 'device' (local desktop).
@@ -44,9 +46,21 @@ const COMMAND_CAPABILITY_MATRIX: Record = {
// macOS desktop targets report kind=device, so this stays enabled here and the
// supports() guard excludes iOS physical devices.
apple: { simulator: true, device: true },
- android: {},
+ android: { emulator: true, device: true, unknown: true },
+ linux: LINUX_NONE,
+ supports: (device) => device.platform === 'android' || isMacOsOrMobileAppleSimulator(device),
+ },
+ 'rotate-gesture': {
+ apple: { simulator: true, device: true },
+ android: { emulator: true, device: true, unknown: true },
+ linux: LINUX_NONE,
+ supports: (device) => device.platform === 'android' || isIosMobileSimulator(device),
+ },
+ 'transform-gesture': {
+ apple: { simulator: true, device: true },
+ android: { emulator: true, device: true, unknown: true },
linux: LINUX_NONE,
- supports: isMacOsOrMobileAppleSimulator,
+ supports: (device) => device.platform === 'android' || isIosMobileSimulator(device),
},
'app-switcher': {
apple: { simulator: true, device: true },
@@ -94,6 +108,11 @@ const COMMAND_CAPABILITY_MATRIX: Record = {
android: { emulator: true, device: true, unknown: true },
linux: LINUX_DEVICE,
},
+ fling: {
+ apple: { simulator: true, device: true },
+ android: { emulator: true, device: true, unknown: true },
+ linux: LINUX_NONE,
+ },
...CAPTURE_COMMAND_CAPABILITIES,
...SELECTOR_COMMAND_CAPABILITIES,
focus: {
@@ -127,6 +146,11 @@ const COMMAND_CAPABILITY_MATRIX: Record = {
android: { emulator: true, device: true, unknown: true },
linux: LINUX_NONE,
},
+ pan: {
+ apple: { simulator: true, device: true },
+ android: { emulator: true, device: true, unknown: true },
+ linux: LINUX_DEVICE,
+ },
press: {
apple: { simulator: true, device: true },
android: { emulator: true, device: true, unknown: true },
diff --git a/src/core/dispatch-interactions.ts b/src/core/dispatch-interactions.ts
index 1b0ae9b5b..530862e7f 100644
--- a/src/core/dispatch-interactions.ts
+++ b/src/core/dispatch-interactions.ts
@@ -447,6 +447,65 @@ export async function handleSwipeCommand(
);
}
+export async function handlePanCommand(
+ interactor: Interactor,
+ positionals: string[],
+): Promise> {
+ const x = Number(positionals[0]);
+ const y = Number(positionals[1]);
+ const dx = Number(positionals[2]);
+ const dy = Number(positionals[3]);
+ if ([x, y, dx, dy].some((value) => !Number.isFinite(value))) {
+ throw new AppError('INVALID_ARGS', 'gesture pan requires x y dx dy [durationMs]');
+ }
+ const requestedDurationMs = positionals[4] ? Number(positionals[4]) : 500;
+ const durationMs = requireIntInRange(requestedDurationMs, 'durationMs', 16, 10_000);
+ const x2 = x + dx;
+ const y2 = y + dy;
+ await interactor.pan(x, y, x2, y2, durationMs);
+ return {
+ x,
+ y,
+ dx,
+ dy,
+ x2,
+ y2,
+ durationMs,
+ ...successText(`Panned (${x}, ${y}) by (${dx}, ${dy})`),
+ };
+}
+
+export async function handleFlingCommand(
+ interactor: Interactor,
+ positionals: string[],
+): Promise> {
+ const direction = parseGestureDirection(positionals[0], 'fling direction');
+ const x = Number(positionals[1]);
+ const y = Number(positionals[2]);
+ if (![x, y].every(Number.isFinite)) {
+ throw new AppError(
+ 'INVALID_ARGS',
+ 'gesture fling requires direction x y [distance] [durationMs]',
+ );
+ }
+ const distanceInput = positionals[3] ? Number(positionals[3]) : 180;
+ const distance = requireFinitePositiveNumber(distanceInput, 'distance');
+ const requestedDurationMs = positionals[4] ? Number(positionals[4]) : 50;
+ const durationMs = requireIntInRange(requestedDurationMs, 'durationMs', 16, 1_000);
+ const { x2, y2 } = pointOffsetByDirection(x, y, direction, distance);
+ await interactor.fling(x, y, x2, y2, durationMs);
+ return {
+ direction,
+ x,
+ y,
+ x2,
+ y2,
+ distance,
+ durationMs,
+ ...successText(`Flung ${direction}`),
+ };
+}
+
export async function handleScrollCommand(
interactor: Interactor,
positionals: string[],
@@ -539,41 +598,187 @@ function parseScrollTarget(input: string): {
export async function handlePinchCommand(
device: DeviceInfo,
+ interactor: Interactor,
positionals: string[],
context: DispatchContext | undefined,
): Promise> {
- if (device.platform === 'android') {
- throw new AppError(
- 'UNSUPPORTED_OPERATION',
- 'Android pinch is not supported in current adb backend; requires instrumentation-based backend.',
- );
- }
if (device.target === 'tv') {
- throw new AppError('UNSUPPORTED_OPERATION', 'pinch is not supported on tvOS');
+ throw new AppError('UNSUPPORTED_OPERATION', 'gesture pinch is not supported on tvOS');
}
if (device.platform === 'macos' && context?.surface && context.surface !== 'app') {
throw new AppError(
'UNSUPPORTED_OPERATION',
- 'pinch is only supported in macOS app sessions. Re-open the target app without --surface desktop|menubar|frontmost-app first.',
+ 'gesture pinch is only supported in macOS app sessions. Re-open the target app without --surface desktop|menubar|frontmost-app first.',
);
}
const scale = Number(positionals[0]);
const x = positionals[1] ? Number(positionals[1]) : undefined;
const y = positionals[2] ? Number(positionals[2]) : undefined;
if (Number.isNaN(scale) || scale <= 0) {
- throw new AppError('INVALID_ARGS', 'pinch requires scale > 0');
+ throw new AppError('INVALID_ARGS', 'gesture pinch requires scale > 0');
+ }
+ const interactionResult = await interactor.pinch(scale, x, y);
+ return { scale, x, y, ...interactionResult, ...successText(`Pinched to scale ${scale}`) };
+}
+
+export async function handleRotateGestureCommand(
+ device: DeviceInfo,
+ interactor: Interactor,
+ positionals: string[],
+): Promise> {
+ if (device.target === 'tv') {
+ throw new AppError('UNSUPPORTED_OPERATION', 'gesture rotate is not supported on tvOS');
+ }
+ if (device.platform === 'macos') {
+ throw new AppError(
+ 'UNSUPPORTED_OPERATION',
+ 'gesture rotate is not supported on macOS; XCTest rotation gestures are available only for iOS app sessions.',
+ );
+ }
+
+ const { degrees, x, y, velocity } = parseRotateGestureParams(positionals);
+
+ const interactionResult = await interactor.rotateGesture(degrees, x, y, velocity);
+ return {
+ degrees,
+ ...(x !== undefined && y !== undefined ? { x, y } : {}),
+ velocity,
+ ...interactionResult,
+ ...successText(`Rotated gesture ${degrees} degrees`),
+ };
+}
+
+export async function handleTransformGestureCommand(
+ device: DeviceInfo,
+ interactor: Interactor,
+ positionals: string[],
+): Promise> {
+ if (device.target === 'tv') {
+ throw new AppError('UNSUPPORTED_OPERATION', 'gesture transform is not supported on tvOS');
+ }
+ const supportedIosSimulator = device.platform === 'ios' && device.kind === 'simulator';
+ if (device.platform !== 'android' && !supportedIosSimulator) {
+ throw new AppError(
+ 'UNSUPPORTED_OPERATION',
+ 'gesture transform is currently supported on Android and iOS simulators',
+ );
+ }
+
+ const params = parseTransformGestureParams(positionals);
+ const interactionResult = await interactor.transformGesture(params);
+ return {
+ ...params,
+ ...interactionResult,
+ ...successText(
+ `Requested transform gesture by (${params.dx}, ${params.dy}), scale ${params.scale}, rotate ${params.degrees} degrees`,
+ ),
+ };
+}
+
+type GestureDirection = 'up' | 'down' | 'left' | 'right';
+
+type RotateGestureParams = {
+ degrees: number;
+ x?: number;
+ y?: number;
+ velocity: number;
+};
+
+type TransformGestureParams = {
+ x: number;
+ y: number;
+ dx: number;
+ dy: number;
+ scale: number;
+ degrees: number;
+ durationMs?: number;
+};
+
+function parseRotateGestureParams(positionals: string[]): RotateGestureParams {
+ const degrees = Number(positionals[0]);
+ if (!Number.isFinite(degrees)) {
+ throw new AppError('INVALID_ARGS', 'gesture rotate requires degrees [x] [y] [velocity]');
+ }
+
+ const center = parseOptionalGestureCenter(positionals[1], positionals[2]);
+ const velocity = Number(positionals[3] ?? (degrees >= 0 ? 1 : -1));
+ if (!Number.isFinite(velocity) || velocity === 0) {
+ throw new AppError('INVALID_ARGS', 'gesture rotate velocity must be a non-zero number');
+ }
+
+ return { degrees, ...center, velocity: Math.abs(velocity) * (degrees >= 0 ? 1 : -1) };
+}
+
+function parseTransformGestureParams(positionals: string[]): TransformGestureParams {
+ const x = Number(positionals[0]);
+ const y = Number(positionals[1]);
+ const dx = Number(positionals[2]);
+ const dy = Number(positionals[3]);
+ const scale = Number(positionals[4]);
+ const degrees = Number(positionals[5]);
+ if (![x, y, dx, dy, scale, degrees].every(Number.isFinite)) {
+ throw new AppError(
+ 'INVALID_ARGS',
+ 'gesture transform requires x y dx dy scale degrees [durationMs]',
+ );
+ }
+ if (scale <= 0) {
+ throw new AppError('INVALID_ARGS', 'gesture transform scale must be > 0');
+ }
+ const durationMs =
+ positionals[6] === undefined
+ ? undefined
+ : requireIntInRange(Number(positionals[6]), 'durationMs', 16, 10_000);
+ return { x, y, dx, dy, scale, degrees, durationMs };
+}
+
+function parseOptionalGestureCenter(
+ xInput: string | undefined,
+ yInput: string | undefined,
+): Pick {
+ if (xInput === undefined && yInput === undefined) return {};
+ if (xInput === undefined || yInput === undefined) {
+ throw new AppError('INVALID_ARGS', 'gesture rotate center requires both x and y');
+ }
+
+ const x = Number(xInput);
+ const y = Number(yInput);
+ if (!Number.isFinite(x) || !Number.isFinite(y)) {
+ throw new AppError('INVALID_ARGS', 'gesture rotate center requires finite x and y');
+ }
+ return { x, y };
+}
+
+function parseGestureDirection(input: string | undefined, field: string): GestureDirection {
+ if (input === 'up' || input === 'down' || input === 'left' || input === 'right') {
+ return input;
+ }
+ throw new AppError('INVALID_ARGS', `${field} must be up, down, left, or right`);
+}
+
+function requireFinitePositiveNumber(value: number, field: string): number {
+ if (!Number.isFinite(value) || value <= 0) {
+ throw new AppError('INVALID_ARGS', `${field} must be a positive number`);
+ }
+ return value;
+}
+
+function pointOffsetByDirection(
+ x: number,
+ y: number,
+ direction: GestureDirection,
+ distance: number,
+): { x2: number; y2: number } {
+ switch (direction) {
+ case 'up':
+ return { x2: x, y2: y - distance };
+ case 'down':
+ return { x2: x, y2: y + distance };
+ case 'left':
+ return { x2: x - distance, y2: y };
+ case 'right':
+ return { x2: x + distance, y2: y };
}
- await runIosRunnerCommand(
- device,
- { command: 'pinch', scale, x, y, appBundleId: context?.appBundleId },
- {
- verbose: context?.verbose,
- logPath: context?.logPath,
- traceLogPath: context?.traceLogPath,
- requestId: context?.requestId,
- },
- );
- return { scale, x, y, ...successText(`Pinched to scale ${scale}`) };
}
export async function handleReadCommand(
diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts
index 543ce191f..f28a3c6e8 100644
--- a/src/core/dispatch.ts
+++ b/src/core/dispatch.ts
@@ -24,13 +24,17 @@ import { screenshotOptionsFromFlags } from '../commands/capture-screenshot-optio
import type { DispatchContext } from './dispatch-context.ts';
import {
handleFillCommand,
+ handleFlingCommand,
handleFocusCommand,
handleLongPressCommand,
+ handlePanCommand,
handlePinchCommand,
handlePressCommand,
handleReadCommand,
+ handleRotateGestureCommand,
handleScrollCommand,
handleSwipeCommand,
+ handleTransformGestureCommand,
handleTypeCommand,
} from './dispatch-interactions.ts';
import { readNotificationPayload } from './dispatch-payload.ts';
@@ -82,6 +86,10 @@ export async function dispatchCommand(
return handlePressCommand(device, interactor, positionals, context);
case 'swipe':
return handleSwipeCommand(device, interactor, positionals, context);
+ case 'pan':
+ return handlePanCommand(interactor, positionals);
+ case 'fling':
+ return handleFlingCommand(interactor, positionals);
case 'longpress':
return handleLongPressCommand(interactor, positionals);
case 'focus':
@@ -93,7 +101,11 @@ export async function dispatchCommand(
case 'scroll':
return handleScrollCommand(interactor, positionals, context);
case 'pinch':
- return handlePinchCommand(device, positionals, context);
+ return handlePinchCommand(device, interactor, positionals, context);
+ case 'rotate-gesture':
+ return handleRotateGestureCommand(device, interactor, positionals);
+ case 'transform-gesture':
+ return handleTransformGestureCommand(device, interactor, positionals);
case 'trigger-app-event': {
const { eventName, payload } = parseTriggerAppEventArgs(positionals);
const eventUrl = resolveAppEventUrl(device.platform, eventName, payload);
diff --git a/src/core/interactor-types.ts b/src/core/interactor-types.ts
index 5da2f3a17..01653a5e9 100644
--- a/src/core/interactor-types.ts
+++ b/src/core/interactor-types.ts
@@ -58,6 +58,20 @@ export type Interactor = {
y2: number,
durationMs?: number,
): Promise | void>;
+ pan(
+ x1: number,
+ y1: number,
+ x2: number,
+ y2: number,
+ durationMs?: number,
+ ): Promise | void>;
+ fling(
+ x1: number,
+ y1: number,
+ x2: number,
+ y2: number,
+ durationMs?: number,
+ ): Promise | void>;
longPress(x: number, y: number, durationMs?: number): Promise | void>;
focus(x: number, y: number): Promise | void>;
type(text: string, delayMs?: number): Promise;
@@ -71,11 +85,27 @@ export type Interactor = {
direction: ScrollDirection,
options?: { amount?: number; pixels?: number },
): Promise | void>;
+ pinch(scale: number, x?: number, y?: number): Promise | void>;
screenshot(outPath: string, options?: ScreenshotOptions): Promise;
snapshot(options?: SnapshotOptions): Promise;
back(mode?: BackMode): Promise;
home(): Promise;
rotate(orientation: DeviceRotation): Promise;
+ rotateGesture(
+ degrees: number,
+ x?: number,
+ y?: number,
+ velocity?: number,
+ ): Promise | void>;
+ transformGesture(options: {
+ x: number;
+ y: number;
+ dx: number;
+ dy: number;
+ scale: number;
+ degrees: number;
+ durationMs?: number;
+ }): Promise | void>;
appSwitcher(): Promise;
readClipboard(): Promise;
writeClipboard(text: string): Promise;
diff --git a/src/core/interactors/android.ts b/src/core/interactors/android.ts
index d7ad27f8f..3dc7f2993 100644
--- a/src/core/interactors/android.ts
+++ b/src/core/interactors/android.ts
@@ -16,6 +16,11 @@ import {
swipeAndroid,
typeAndroid,
} from '../../platforms/android/input-actions.ts';
+import {
+ pinchAndroid,
+ rotateGestureAndroid,
+ transformGestureAndroid,
+} from '../../platforms/android/multitouch-helper.ts';
import {
readAndroidClipboardText,
writeAndroidClipboardText,
@@ -38,11 +43,14 @@ export function createAndroidInteractor(device: DeviceInfo): Interactor {
await pressAndroid(device, x, y);
},
swipe: (x1, y1, x2, y2, durationMs) => swipeAndroid(device, x1, y1, x2, y2, durationMs),
+ pan: (x1, y1, x2, y2, durationMs) => swipeAndroid(device, x1, y1, x2, y2, durationMs),
+ fling: (x1, y1, x2, y2, durationMs) => swipeAndroid(device, x1, y1, x2, y2, durationMs),
longPress: (x, y, durationMs) => longPressAndroid(device, x, y, durationMs),
focus: (x, y) => focusAndroid(device, x, y),
type: (text, delayMs) => typeAndroid(device, text, delayMs),
fill: (x, y, text, delayMs) => fillAndroid(device, x, y, text, delayMs),
scroll: (direction, options) => scrollAndroid(device, direction, options),
+ pinch: (scale, x, y) => pinchAndroid(device, { scale, x, y }),
screenshot: (outPath, options) => screenshotAndroid(device, outPath, options),
snapshot: async (options) => {
const result = await withDiagnosticTimer(
@@ -68,6 +76,9 @@ export function createAndroidInteractor(device: DeviceInfo): Interactor {
back: (_mode) => backAndroid(device),
home: () => homeAndroid(device),
rotate: (orientation) => rotateAndroid(device, orientation),
+ rotateGesture: (degrees, x, y, velocity) =>
+ rotateGestureAndroid(device, { degrees, x, y, velocity }),
+ transformGesture: (options) => transformGestureAndroid(device, options),
appSwitcher: () => appSwitcherAndroid(device),
readClipboard: () => readAndroidClipboardText(device),
writeClipboard: (text) => writeAndroidClipboardText(device, text),
diff --git a/src/core/interactors/linux.ts b/src/core/interactors/linux.ts
index 377be810b..66a4667bf 100644
--- a/src/core/interactors/linux.ts
+++ b/src/core/interactors/linux.ts
@@ -29,11 +29,18 @@ export function createLinuxInteractor(): Interactor {
tap: (x, y) => pressLinux(x, y),
doubleTap: (x, y) => doubleClickLinux(x, y),
swipe: (x1, y1, x2, y2, durationMs) => swipeLinux(x1, y1, x2, y2, durationMs),
+ pan: (x1, y1, x2, y2, durationMs) => swipeLinux(x1, y1, x2, y2, durationMs),
+ fling: () => {
+ throw new AppError('UNSUPPORTED_OPERATION', 'gesture fling not supported on Linux');
+ },
longPress: (x, y, durationMs) => longPressLinux(x, y, durationMs),
focus: (x, y) => focusLinux(x, y),
type: (text, delayMs) => typeLinux(text, delayMs),
fill: (x, y, text, delayMs) => fillLinux(x, y, text, delayMs),
scroll: (direction, options) => scrollLinux(direction, options),
+ pinch: () => {
+ throw new AppError('UNSUPPORTED_OPERATION', 'gesture pinch not supported on Linux');
+ },
screenshot: (outPath, options) => screenshotLinux(outPath, options),
snapshot: async (options) => {
const result = await withDiagnosticTimer(
@@ -52,6 +59,12 @@ export function createLinuxInteractor(): Interactor {
rotate: () => {
throw new AppError('UNSUPPORTED_OPERATION', 'rotate not supported on Linux');
},
+ rotateGesture: () => {
+ throw new AppError('UNSUPPORTED_OPERATION', 'gesture rotate not supported on Linux');
+ },
+ transformGesture: () => {
+ throw new AppError('UNSUPPORTED_OPERATION', 'gesture transform not supported on Linux');
+ },
appSwitcher: () => {
throw new AppError('UNSUPPORTED_OPERATION', 'appSwitcher not yet supported on Linux');
},
diff --git a/src/daemon/recording-gestures.ts b/src/daemon/recording-gestures.ts
index 44f83cc9f..e3aef8489 100644
--- a/src/daemon/recording-gestures.ts
+++ b/src/daemon/recording-gestures.ts
@@ -165,30 +165,39 @@ function buildGestureEvents(
gestureDurationMs: number,
referenceFrame?: ReferenceFrame,
): RecordingGestureEvent[] {
- switch (command) {
- case 'click':
- case 'press':
- return buildPressEvents(positionals, result, tMs, referenceFrame);
- case 'react-native':
- return positionals[0] === 'dismiss-overlay'
- ? buildPressEvents(positionals, result, tMs, referenceFrame)
- : [];
- case 'fill':
- case 'focus':
- return buildFocusEvents(positionals, result, tMs, referenceFrame);
- case 'longpress':
- return buildLongPressEvents(positionals, result, tMs, gestureDurationMs, referenceFrame);
- case 'scroll':
- return buildScrollEvents(positionals, result, tMs, gestureDurationMs, referenceFrame);
- case 'swipe':
- return buildSwipeEvents(positionals, result, tMs, gestureDurationMs, referenceFrame);
- case 'pinch':
- return buildPinchEvents(positionals, result, tMs, gestureDurationMs, referenceFrame);
- default:
- return [];
- }
+ const builder = gestureEventBuilders[command];
+ return builder?.(positionals, result, tMs, gestureDurationMs, referenceFrame) ?? [];
}
+type GestureEventBuilder = (
+ positionals: string[],
+ result: Record,
+ tMs: number,
+ gestureDurationMs: number,
+ referenceFrame?: ReferenceFrame,
+) => RecordingGestureEvent[];
+
+const gestureEventBuilders: Record = {
+ click: (positionals, result, tMs, _durationMs, referenceFrame) =>
+ buildPressEvents(positionals, result, tMs, referenceFrame),
+ press: (positionals, result, tMs, _durationMs, referenceFrame) =>
+ buildPressEvents(positionals, result, tMs, referenceFrame),
+ 'react-native': (positionals, result, tMs, _durationMs, referenceFrame) =>
+ positionals[0] === 'dismiss-overlay'
+ ? buildPressEvents(positionals, result, tMs, referenceFrame)
+ : [],
+ fill: (positionals, result, tMs, _durationMs, referenceFrame) =>
+ buildFocusEvents(positionals, result, tMs, referenceFrame),
+ focus: (positionals, result, tMs, _durationMs, referenceFrame) =>
+ buildFocusEvents(positionals, result, tMs, referenceFrame),
+ longpress: buildLongPressEvents,
+ scroll: buildScrollEvents,
+ pan: buildSwipeEvents,
+ fling: buildSwipeEvents,
+ swipe: buildSwipeEvents,
+ pinch: buildPinchEvents,
+};
+
function shouldAnchorTapVisualizationNearCompletion(
command: string,
result: Record,
@@ -294,43 +303,51 @@ function buildSwipeEvents(
const count = clampInt(readNumber(result.count), 1) ?? 1;
const pauseMs = clampInt(readNumber(result.pauseMs), 0) ?? 0;
const pattern = result.pattern === 'ping-pong' ? 'ping-pong' : 'one-way';
- const events: RecordingGestureEvent[] = [];
-
- for (let index = 0; index < count; index += 1) {
- const reverse = pattern === 'ping-pong' && index % 2 === 1;
- const startX = reverse ? x2 : x1;
- const startY = reverse ? y2 : y1;
- const endX = reverse ? x1 : x2;
- const endY = reverse ? y1 : y2;
+ return Array.from({ length: count }, (_, index) => {
+ const { startX, startY, endX, endY } = resolveSwipePathForIndex(index, pattern, x1, y1, x2, y2);
const startTime = tMs + index * (durationMs + pauseMs);
- const kind = classifySwipeKind(startX, startY, endX, endY, referenceFrame);
- if (kind === 'back-swipe') {
- events.push({
- kind: 'back-swipe',
- tMs: startTime,
- x: startX,
- y: startY,
- x2: endX,
- y2: endY,
- ...referenceFrame,
- durationMs,
- edge: resolveBackSwipeEdge(startX, endX, referenceFrame),
- });
- continue;
- }
- events.push({
- kind: 'swipe',
- tMs: startTime,
- x: startX,
- y: startY,
- x2: endX,
- y2: endY,
+ return buildSwipeTravelEvent(startTime, startX, startY, endX, endY, durationMs, referenceFrame);
+ });
+}
+
+function resolveSwipePathForIndex(
+ index: number,
+ pattern: 'one-way' | 'ping-pong',
+ x1: number,
+ y1: number,
+ x2: number,
+ y2: number,
+): { startX: number; startY: number; endX: number; endY: number } {
+ const reverse = pattern === 'ping-pong' && index % 2 === 1;
+ return reverse
+ ? { startX: x2, startY: y2, endX: x1, endY: y1 }
+ : { startX: x1, startY: y1, endX: x2, endY: y2 };
+}
+
+function buildSwipeTravelEvent(
+ tMs: number,
+ x: number,
+ y: number,
+ x2: number,
+ y2: number,
+ durationMs: number,
+ referenceFrame?: ReferenceFrame,
+): RecordingGestureEvent {
+ const kind = classifySwipeKind(x, y, x2, y2, referenceFrame);
+ if (kind === 'back-swipe') {
+ return {
+ kind,
+ tMs,
+ x,
+ y,
+ x2,
+ y2,
...referenceFrame,
durationMs,
- });
+ edge: resolveBackSwipeEdge(x, x2, referenceFrame),
+ };
}
-
- return events;
+ return { kind, tMs, x, y, x2, y2, ...referenceFrame, durationMs };
}
function buildScrollEvents(
@@ -504,16 +521,20 @@ function readTravelCoordinates(
result: Record,
positionals: string[],
): { x1: number; y1: number; x2: number; y2: number } | undefined {
- const x1 = readNumber(result.x1) ?? readNumber(positionals[0]);
- const y1 = readNumber(result.y1) ?? readNumber(positionals[1]);
- const x2 = readNumber(result.x2) ?? readNumber(positionals[2]);
- const y2 = readNumber(result.y2) ?? readNumber(positionals[3]);
+ const x1 = readFirstNumber(result.x1, result.x, positionals[0]);
+ const y1 = readFirstNumber(result.y1, result.y, positionals[1]);
+ const x2 = readFirstNumber(result.x2, positionals[2]);
+ const y2 = readFirstNumber(result.y2, positionals[3]);
if (x1 === undefined || y1 === undefined || x2 === undefined || y2 === undefined) {
return undefined;
}
return { x1, y1, x2, y2 };
}
+function readFirstNumber(...values: unknown[]): number | undefined {
+ return values.map(readNumber).find((value) => value !== undefined);
+}
+
function resolveDurationMs(
gestureDurationMs: number,
candidates: Array,
diff --git a/src/daemon/request-generic-dispatch.ts b/src/daemon/request-generic-dispatch.ts
index 23da7e835..db4a10c19 100644
--- a/src/daemon/request-generic-dispatch.ts
+++ b/src/daemon/request-generic-dispatch.ts
@@ -1,4 +1,5 @@
import { dispatchCommand, type CommandFlags } from '../core/dispatch.ts';
+import { GESTURE_SUBCOMMAND_ERROR } from '../command-catalog.ts';
import { isCommandSupportedOnDevice } from '../core/capabilities.ts';
import { SessionStore } from './session-store.ts';
import type { DaemonCommandContext } from './context.ts';
@@ -21,6 +22,14 @@ import {
} from './recording-gestures.ts';
import { markPostGestureStabilization } from './post-gesture-stabilization.ts';
+const GESTURE_PLATFORM_COMMANDS: Readonly> = {
+ pan: 'pan',
+ fling: 'fling',
+ pinch: 'pinch',
+ rotate: 'rotate-gesture',
+ transform: 'transform-gesture',
+};
+
export async function dispatchGenericCommand(params: {
req: DaemonRequest;
session: SessionState;
@@ -34,78 +43,168 @@ export async function dispatchGenericCommand(params: {
) => DaemonCommandContext;
}): Promise {
const { req, session, logPath, sessionStore, contextFromFlags } = params;
- const command = req.command;
-
- if (!isCommandSupportedOnDevice(command, session.device)) {
+ const commandResolution = resolveDispatchCommand(req);
+ if (!commandResolution.ok) {
return {
ok: false,
error: {
- code: 'UNSUPPORTED_OPERATION',
- message: `${command} is not supported on this device`,
+ code: 'INVALID_ARGS',
+ message: commandResolution.message,
},
};
}
+ const { platformCommand, dispatchRequest, recordedCommand } = commandResolution;
- if (session.device.platform === 'android' && session.recording && command !== 'record') {
- const androidRecoveryResult = await recoverAndroidBlockingSystemDialog({ session });
- if (androidRecoveryResult === 'failed') {
- return {
- ok: false,
- error: {
- code: 'COMMAND_FAILED',
- message: 'Android system dialog blocked the recording session',
- },
- };
- }
- }
+ const readinessResponse = await ensureGenericCommandReady(session, platformCommand);
+ if (readinessResponse) return readinessResponse;
const { resolvedPositionals, resolvedOut, recordedPositionals, recordedFlags } =
- resolveCommandPositionals(req);
+ resolveCommandPositionals(dispatchRequest);
const actionStartedAt = Date.now();
const dispatchContext = {
...contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath),
surface: session.surface,
};
- const data =
- command === 'screenshot'
- ? await dispatchScreenshotViaRuntime({
- session,
- sessionName: params.sessionName,
- outPath: resolvedPositionals[0] ?? resolvedOut,
- outputPlacement: resolveScreenshotOutputPlacement(req),
- dispatchContext,
- })
- : await dispatchCommand(session.device, command, resolvedPositionals, resolvedOut, {
- ...dispatchContext,
- });
-
- if (command === 'screenshot' && req.flags?.overlayRefs && typeof data?.path === 'string') {
- await applyScreenshotOverlay(session, data, logPath);
- }
+ const data = await executeGenericPlatformCommand({
+ session,
+ sessionName: params.sessionName,
+ logPath,
+ command: platformCommand,
+ request: dispatchRequest,
+ positionals: resolvedPositionals,
+ out: resolvedOut,
+ dispatchContext,
+ });
const actionFinishedAt = Date.now();
+ const actionRecordedPositionals =
+ recordedCommand === platformCommand ? recordedPositionals : (req.positionals ?? []);
+ const actionRecordedFlags =
+ recordedCommand === platformCommand ? recordedFlags : (req.flags ?? {});
recordVisualizationAndAction({
session,
sessionStore,
- command,
+ command: platformCommand,
+ recordedCommand,
resolvedPositionals,
- recordedPositionals,
- recordedFlags,
+ recordedPositionals: actionRecordedPositionals,
+ recordedFlags: actionRecordedFlags,
data,
actionStartedAt,
actionFinishedAt,
flags: req.flags ?? {},
});
- if (isNavigationSensitiveAction(command)) {
- markAndroidSnapshotFreshness(session, command);
+ if (isNavigationSensitiveAction(platformCommand)) {
+ markAndroidSnapshotFreshness(session, platformCommand);
}
- markPostGestureStabilization(session, command);
+ markPostGestureStabilization(session, platformCommand);
return { ok: true, data: data ?? {} };
}
+async function ensureGenericCommandReady(
+ session: SessionState,
+ platformCommand: string,
+): Promise {
+ if (!isCommandSupportedOnDevice(platformCommand, session.device)) {
+ return {
+ ok: false,
+ error: {
+ code: 'UNSUPPORTED_OPERATION',
+ message: `${platformCommand} is not supported on this device`,
+ },
+ };
+ }
+ if (
+ session.device.platform !== 'android' ||
+ !session.recording ||
+ platformCommand === 'record' ||
+ (await recoverAndroidBlockingSystemDialog({ session })) !== 'failed'
+ ) {
+ return null;
+ }
+ return {
+ ok: false,
+ error: {
+ code: 'COMMAND_FAILED',
+ message: 'Android system dialog blocked the recording session',
+ },
+ };
+}
+
+async function executeGenericPlatformCommand(params: {
+ session: SessionState;
+ sessionName: string;
+ logPath: string;
+ command: string;
+ request: DaemonRequest;
+ positionals: string[];
+ out: string | undefined;
+ dispatchContext: DaemonCommandContext;
+}): Promise | void> {
+ const { session, command, request, positionals, out, dispatchContext } = params;
+ if (command !== 'screenshot') {
+ return await dispatchCommand(session.device, command, positionals, out, {
+ ...dispatchContext,
+ });
+ }
+ const data = await dispatchScreenshotViaRuntime({
+ session,
+ sessionName: params.sessionName,
+ outPath: positionals[0] ?? out,
+ outputPlacement: resolveScreenshotOutputPlacement(request),
+ dispatchContext,
+ });
+ if (request.flags?.overlayRefs && typeof data?.path === 'string') {
+ await applyScreenshotOverlay(session, data, params.logPath);
+ }
+ return data;
+}
+
+type DispatchCommandResolution =
+ | {
+ ok: true;
+ platformCommand: string;
+ dispatchRequest: DaemonRequest;
+ recordedCommand: string;
+ }
+ | { ok: false; message: string };
+
+function resolveDispatchCommand(req: DaemonRequest): DispatchCommandResolution {
+ if (
+ req.command === 'pan' ||
+ req.command === 'fling' ||
+ req.command === 'rotate-gesture' ||
+ req.command === 'transform-gesture'
+ ) {
+ return {
+ ok: false,
+ message: 'Use gesture pan, gesture fling, gesture rotate, or gesture transform.',
+ };
+ }
+ if (req.command !== 'gesture') {
+ return {
+ ok: true,
+ platformCommand: req.command,
+ dispatchRequest: req,
+ recordedCommand: req.command,
+ };
+ }
+ const [subcommand, ...positionals] = req.positionals ?? [];
+ const platformCommand = subcommand ? GESTURE_PLATFORM_COMMANDS[subcommand] : undefined;
+ if (!platformCommand) {
+ return { ok: false, message: GESTURE_SUBCOMMAND_ERROR };
+ }
+ return {
+ ok: true,
+ platformCommand,
+ dispatchRequest: { ...req, command: platformCommand, positionals },
+ recordedCommand: req.command,
+ };
+}
+
function resolveScreenshotOutputPlacement(req: DaemonRequest): ScreenshotOutputPlacement {
if (req.command !== 'screenshot') return 'default';
if ((req.positionals ?? [])[0]) return 'positional';
@@ -197,6 +296,7 @@ function recordVisualizationAndAction(params: {
session: SessionState;
sessionStore: SessionStore;
command: string;
+ recordedCommand: string;
resolvedPositionals: string[];
recordedPositionals: string[];
recordedFlags: Record;
@@ -209,6 +309,7 @@ function recordVisualizationAndAction(params: {
session,
sessionStore,
command,
+ recordedCommand,
resolvedPositionals,
recordedPositionals,
recordedFlags,
@@ -233,7 +334,7 @@ function recordVisualizationAndAction(params: {
actionFinishedAt,
);
sessionStore.recordAction(session, {
- command,
+ command: recordedCommand,
positionals: recordedPositionals,
flags: recordedFlags,
result: data ?? {},
diff --git a/src/platforms/android/__tests__/multitouch-helper.test.ts b/src/platforms/android/__tests__/multitouch-helper.test.ts
new file mode 100644
index 000000000..619f700f1
--- /dev/null
+++ b/src/platforms/android/__tests__/multitouch-helper.test.ts
@@ -0,0 +1,220 @@
+import assert from 'node:assert/strict';
+import crypto from 'node:crypto';
+import { promises as fs } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { beforeEach, test } from 'vitest';
+import { ANDROID_EMULATOR } from '../../../__tests__/test-utils/index.ts';
+import {
+ ensureAndroidMultiTouchHelper,
+ parseAndroidMultiTouchHelperOutput,
+ pinchAndroid,
+ resetAndroidMultiTouchHelperInstallCache,
+ rotateGestureAndroid,
+ runAndroidMultiTouchHelperGesture,
+ transformGestureAndroid,
+} from '../multitouch-helper.ts';
+import {
+ withAndroidAdbProvider,
+ type AndroidAdbExecutor,
+ type AndroidAdbProvider,
+} from '../adb-executor.ts';
+
+const manifest = {
+ name: 'android-multitouch-helper' as const,
+ version: '0.15.0',
+ assetName: 'helper.apk',
+ sha256: 'a'.repeat(64),
+ packageName: 'com.callstack.agentdevice.multitouchhelper',
+ versionCode: 15000,
+ instrumentationRunner: 'com.callstack.agentdevice.multitouchhelper/.MultiTouchInstrumentation',
+ statusProtocol: 'android-multitouch-helper-v1' as const,
+};
+
+beforeEach(() => {
+ resetAndroidMultiTouchHelperInstallCache();
+});
+
+test('parseAndroidMultiTouchHelperOutput returns final instrumentation gesture metadata', () => {
+ const parsed = parseAndroidMultiTouchHelperOutput(
+ [
+ resultRecord({
+ ok: 'true',
+ kind: 'pinch',
+ helperApiVersion: '1',
+ injectedEvents: '24',
+ elapsedMs: '315',
+ }),
+ 'INSTRUMENTATION_CODE: 0',
+ ].join('\n'),
+ );
+
+ assert.deepEqual(parsed, {
+ kind: 'pinch',
+ helperApiVersion: '1',
+ injectedEvents: 24,
+ elapsedMs: 315,
+ });
+});
+
+test('runAndroidMultiTouchHelperGesture encodes protocol payload for instrumentation', async () => {
+ let capturedArgs: string[] | undefined;
+ const result = await runAndroidMultiTouchHelperGesture({
+ adb: async (args) => {
+ capturedArgs = args;
+ return {
+ exitCode: 0,
+ stdout: [resultRecord({ ok: 'true', kind: 'rotate' }), 'INSTRUMENTATION_CODE: 0'].join(
+ '\n',
+ ),
+ stderr: '',
+ };
+ },
+ request: { kind: 'rotate', x: 100, y: 200, degrees: 145, radius: 120, durationMs: 250 },
+ packageName: manifest.packageName,
+ instrumentationRunner: manifest.instrumentationRunner,
+ });
+
+ assert.equal(result.kind, 'rotate');
+ assert.ok(capturedArgs);
+ assert.deepEqual(capturedArgs.slice(0, 7), [
+ 'shell',
+ 'am',
+ 'instrument',
+ '-w',
+ '-e',
+ 'payloadBase64',
+ capturedArgs[6],
+ ]);
+ assert.deepEqual(JSON.parse(Buffer.from(capturedArgs[6]!, 'base64').toString('utf8')), {
+ protocol: 'android-multitouch-helper-v1',
+ kind: 'rotate',
+ x: 100,
+ y: 200,
+ degrees: 145,
+ radius: 120,
+ durationMs: 250,
+ });
+ assert.equal(capturedArgs.at(-1), manifest.instrumentationRunner);
+});
+
+test('pinchAndroid, rotateGestureAndroid, and transformGestureAndroid prefer provider-native touch injection', async () => {
+ const calls: unknown[] = [];
+ await withAndroidAdbProvider(
+ {
+ exec: async () => {
+ throw new Error('adb should not run when native touch is available');
+ },
+ touch: async (request) => {
+ calls.push(request);
+ return { backendDetail: 'native' };
+ },
+ },
+ { serial: ANDROID_EMULATOR.id },
+ async () => {
+ const pinch = await pinchAndroid(ANDROID_EMULATOR, { scale: 2, x: 100, y: 200 });
+ const rotate = await rotateGestureAndroid(ANDROID_EMULATOR, {
+ degrees: -215,
+ x: 100,
+ y: 200,
+ });
+ const transform = await transformGestureAndroid(ANDROID_EMULATOR, {
+ x: 100,
+ y: 200,
+ dx: 30,
+ dy: -20,
+ scale: 1.5,
+ degrees: 35,
+ });
+
+ assert.equal(pinch.backend, 'provider-native-touch');
+ assert.equal(rotate.backend, 'provider-native-touch');
+ assert.equal(transform.backend, 'provider-native-touch');
+ },
+ );
+
+ assert.deepEqual(calls, [
+ { kind: 'pinch', x: 100, y: 200, scale: 2, durationMs: undefined },
+ { kind: 'rotate', x: 100, y: 200, degrees: -215, durationMs: undefined },
+ {
+ kind: 'transform',
+ x: 100,
+ y: 200,
+ dx: 30,
+ dy: -20,
+ scale: 1.5,
+ degrees: 35,
+ durationMs: undefined,
+ },
+ ]);
+});
+
+test('rotateGestureAndroid rejects zero velocity before provider dispatch', async () => {
+ await withAndroidAdbProvider(
+ {
+ exec: async () => {
+ throw new Error('adb should not run for invalid input');
+ },
+ touch: async () => {
+ throw new Error('native touch should not run for invalid input');
+ },
+ },
+ { serial: ANDROID_EMULATOR.id },
+ async () => {
+ await assert.rejects(
+ () => rotateGestureAndroid(ANDROID_EMULATOR, { degrees: 90, velocity: 0 }),
+ { code: 'INVALID_ARGS' },
+ );
+ },
+ );
+});
+
+test('ensureAndroidMultiTouchHelper installs with semantic provider install options', async () => {
+ const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'multitouch-helper-install-'));
+ const apkPath = path.join(tmpDir, 'helper.apk');
+ await fs.writeFile(apkPath, 'helper-apk');
+ const installCalls: Array<{
+ apkPath: string;
+ replace?: boolean;
+ allowTestPackages?: boolean;
+ }> = [];
+ const adb: AndroidAdbExecutor = async (args) => {
+ if (args.includes('--show-versioncode')) {
+ return { exitCode: 1, stdout: '', stderr: 'not found' };
+ }
+ throw new Error(`unexpected adb call: ${args.join(' ')}`);
+ };
+ const adbProvider: AndroidAdbProvider = {
+ exec: adb,
+ install: async (path, options) => {
+ installCalls.push({
+ apkPath: path,
+ replace: options?.replace,
+ allowTestPackages: options?.allowTestPackages,
+ });
+ return { exitCode: 0, stdout: '', stderr: '' };
+ },
+ };
+
+ const result = await ensureAndroidMultiTouchHelper({
+ adb,
+ adbProvider,
+ artifact: { apkPath, manifest: { ...manifest, sha256: sha256Text('helper-apk') } },
+ deviceKey: 'android:emulator-5554',
+ });
+
+ assert.equal(result.installed, true);
+ assert.equal(result.reason, 'missing');
+ assert.deepEqual(installCalls, [{ apkPath, replace: true, allowTestPackages: true }]);
+});
+
+function resultRecord(values: Record): string {
+ return [
+ 'INSTRUMENTATION_RESULT: agentDeviceProtocol=android-multitouch-helper-v1',
+ ...Object.entries(values).map(([key, value]) => `INSTRUMENTATION_RESULT: ${key}=${value}`),
+ ].join('\n');
+}
+
+function sha256Text(text: string): string {
+ return crypto.createHash('sha256').update(text).digest('hex');
+}
diff --git a/src/platforms/android/adb-executor.ts b/src/platforms/android/adb-executor.ts
index 95ef9fd76..cef62f1b7 100644
--- a/src/platforms/android/adb-executor.ts
+++ b/src/platforms/android/adb-executor.ts
@@ -119,6 +119,36 @@ export type AndroidTextInjectionRequest = {
export type AndroidTextInjector = (request: AndroidTextInjectionRequest) => Promise;
+export type AndroidTouchGestureRequest =
+ | {
+ kind: 'pinch';
+ x: number;
+ y: number;
+ scale: number;
+ durationMs?: number;
+ }
+ | {
+ kind: 'rotate';
+ x: number;
+ y: number;
+ degrees: number;
+ durationMs?: number;
+ }
+ | {
+ kind: 'transform';
+ x: number;
+ y: number;
+ dx: number;
+ dy: number;
+ scale: number;
+ degrees: number;
+ durationMs?: number;
+ };
+
+export type AndroidTouchInjector = (
+ request: AndroidTouchGestureRequest,
+) => Promise | void>;
+
export type AndroidAdbProvider = {
/**
* Fallback executor for device-scoped adb arguments. Providers may omit explicit
@@ -131,6 +161,7 @@ export type AndroidAdbProvider = {
install?: AndroidAdbInstaller;
installBundle?: AndroidBundleInstaller;
text?: AndroidTextInjector;
+ touch?: AndroidTouchInjector;
};
export type AndroidAdbProviderScopeOptions = {
@@ -210,6 +241,11 @@ export function resolveAndroidTextInjector(device: DeviceInfo): AndroidTextInjec
return scoped?.serial === device.id ? scoped.provider.text : undefined;
}
+export function resolveAndroidTouchInjector(device: DeviceInfo): AndroidTouchInjector | undefined {
+ const scoped = androidAdbProviderScope.getStore();
+ return scoped?.serial === device.id ? scoped.provider.touch : undefined;
+}
+
export function createAndroidPortReverseManager(
provider: AndroidAdbProvider | AndroidAdbExecutor,
): AndroidPortReverseProvider {
diff --git a/src/platforms/android/multitouch-helper.ts b/src/platforms/android/multitouch-helper.ts
new file mode 100644
index 000000000..c939fd6de
--- /dev/null
+++ b/src/platforms/android/multitouch-helper.ts
@@ -0,0 +1,586 @@
+import crypto from 'node:crypto';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { AppError, normalizeError } from '../../utils/errors.ts';
+import { emitDiagnostic, withDiagnosticTimer } from '../../utils/diagnostics.ts';
+import { findProjectRoot, readVersion } from '../../utils/version.ts';
+import type { DeviceInfo } from '../../utils/device.ts';
+import {
+ installAndroidAdbPackage,
+ resolveAndroidAdbExecutor,
+ resolveAndroidAdbProvider,
+ resolveAndroidTouchInjector,
+ type AndroidAdbExecutor,
+ type AndroidAdbProvider,
+ type AndroidTouchGestureRequest,
+} from './adb-executor.ts';
+import { getAndroidScreenSize } from './input-actions.ts';
+
+const ANDROID_MULTITOUCH_HELPER_NAME = 'android-multitouch-helper';
+const ANDROID_MULTITOUCH_HELPER_PACKAGE = 'com.callstack.agentdevice.multitouchhelper';
+const ANDROID_MULTITOUCH_HELPER_RUNNER =
+ 'com.callstack.agentdevice.multitouchhelper/.MultiTouchInstrumentation';
+const ANDROID_MULTITOUCH_HELPER_PROTOCOL = 'android-multitouch-helper-v1';
+const ANDROID_MULTITOUCH_HELPER_INSTALL_TIMEOUT_MS = 30_000;
+const ANDROID_MULTITOUCH_HELPER_GESTURE_TIMEOUT_MS = 15_000;
+const ANDROID_MULTITOUCH_HELPER_DEFAULT_DURATION_MS = 300;
+const ANDROID_MULTITOUCH_HELPER_DEFAULT_RADIUS = 160;
+const ANDROID_MULTITOUCH_HELPER_ROTATE_MAX_DEGREES_PER_FRAME = 3;
+const ANDROID_MULTITOUCH_HELPER_ROTATE_FRAME_INTERVAL_MS = 16;
+const ANDROID_MULTITOUCH_HELPER_ROTATE_MAX_DURATION_MS = 2_400;
+
+type AndroidMultiTouchHelperManifest = {
+ name: 'android-multitouch-helper';
+ version: string;
+ assetName: string;
+ sha256: string;
+ packageName: string;
+ versionCode: number;
+ instrumentationRunner: string;
+ statusProtocol: 'android-multitouch-helper-v1';
+};
+
+type AndroidMultiTouchHelperArtifact = {
+ apkPath: string;
+ manifest: AndroidMultiTouchHelperManifest;
+};
+
+type AndroidMultiTouchHelperGestureRequest =
+ | {
+ kind: 'pinch';
+ x: number;
+ y: number;
+ scale: number;
+ radius: number;
+ durationMs: number;
+ }
+ | {
+ kind: 'rotate';
+ x: number;
+ y: number;
+ degrees: number;
+ radius: number;
+ durationMs: number;
+ }
+ | {
+ kind: 'transform';
+ x: number;
+ y: number;
+ dx: number;
+ dy: number;
+ scale: number;
+ degrees: number;
+ durationMs: number;
+ };
+
+export type AndroidPinchGestureOptions = {
+ scale: number;
+ x?: number;
+ y?: number;
+ durationMs?: number;
+};
+
+export type AndroidRotateGestureOptions = {
+ degrees: number;
+ x?: number;
+ y?: number;
+ velocity?: number;
+ durationMs?: number;
+};
+
+export type AndroidTransformGestureOptions = {
+ x: number;
+ y: number;
+ dx: number;
+ dy: number;
+ scale: number;
+ degrees: number;
+ durationMs?: number;
+};
+
+export async function pinchAndroid(
+ device: DeviceInfo,
+ options: AndroidPinchGestureOptions,
+): Promise> {
+ if (!Number.isFinite(options.scale) || options.scale <= 0) {
+ throw new AppError('INVALID_ARGS', 'gesture pinch requires scale > 0');
+ }
+ const center = await resolveGestureCenter(device, options.x, options.y);
+ return await runAndroidMultiTouchGesture(device, {
+ kind: 'pinch',
+ x: center.x,
+ y: center.y,
+ scale: options.scale,
+ durationMs: options.durationMs,
+ });
+}
+
+export async function rotateGestureAndroid(
+ device: DeviceInfo,
+ options: AndroidRotateGestureOptions,
+): Promise> {
+ if (!Number.isFinite(options.degrees)) {
+ throw new AppError('INVALID_ARGS', 'gesture rotate requires finite degrees');
+ }
+ if (
+ options.velocity !== undefined &&
+ (!Number.isFinite(options.velocity) || options.velocity === 0)
+ ) {
+ throw new AppError('INVALID_ARGS', 'gesture rotate velocity must be a non-zero number');
+ }
+ const center = await resolveGestureCenter(device, options.x, options.y);
+ const degrees = options.degrees;
+ return await runAndroidMultiTouchGesture(device, {
+ kind: 'rotate',
+ x: center.x,
+ y: center.y,
+ degrees,
+ durationMs: options.durationMs,
+ });
+}
+
+export async function transformGestureAndroid(
+ device: DeviceInfo,
+ options: AndroidTransformGestureOptions,
+): Promise> {
+ if (!Number.isFinite(options.scale) || options.scale <= 0) {
+ throw new AppError('INVALID_ARGS', 'gesture transform requires scale > 0');
+ }
+ if (!Number.isFinite(options.degrees)) {
+ throw new AppError('INVALID_ARGS', 'gesture transform requires finite degrees');
+ }
+ if (![options.x, options.y, options.dx, options.dy].every(Number.isFinite)) {
+ throw new AppError('INVALID_ARGS', 'gesture transform requires finite x y dx dy');
+ }
+ return await runAndroidMultiTouchGesture(device, {
+ kind: 'transform',
+ x: options.x,
+ y: options.y,
+ dx: options.dx,
+ dy: options.dy,
+ scale: options.scale,
+ degrees: options.degrees,
+ durationMs: options.durationMs,
+ });
+}
+
+async function resolveGestureCenter(
+ device: DeviceInfo,
+ x: number | undefined,
+ y: number | undefined,
+): Promise<{ x: number; y: number }> {
+ if (x !== undefined && y !== undefined) return { x, y };
+ const size = await getAndroidScreenSize(device);
+ return { x: Math.round(size.width / 2), y: Math.round(size.height / 2) };
+}
+
+async function runAndroidMultiTouchGesture(
+ device: DeviceInfo,
+ request: AndroidTouchGestureRequest,
+): Promise> {
+ const providerTouch = resolveAndroidTouchInjector(device);
+ if (providerTouch) {
+ const result = (await providerTouch(request)) ?? {};
+ return { backend: 'provider-native-touch', ...result };
+ }
+
+ const adb = resolveAndroidAdbExecutor(device);
+ const artifact = await resolveAndroidMultiTouchHelperArtifact();
+ const adbProvider = resolveAndroidAdbProvider(device);
+ const install = await withDiagnosticTimer(
+ 'android_multitouch_helper_install',
+ async () =>
+ await ensureAndroidMultiTouchHelper({
+ adb,
+ adbProvider,
+ artifact,
+ deviceKey: getAndroidMultiTouchHelperDeviceKey(device),
+ }),
+ {
+ packageName: artifact.manifest.packageName,
+ versionCode: artifact.manifest.versionCode,
+ },
+ );
+ emitDiagnostic({
+ phase: 'android_multitouch_helper_install_decision',
+ data: install,
+ });
+ const output = await withDiagnosticTimer(
+ 'android_multitouch_helper_gesture',
+ async () =>
+ await runAndroidMultiTouchHelperGesture({
+ adb,
+ request: normalizeHelperGestureRequest(request),
+ packageName: artifact.manifest.packageName,
+ instrumentationRunner: artifact.manifest.instrumentationRunner,
+ }),
+ {
+ packageName: artifact.manifest.packageName,
+ version: artifact.manifest.version,
+ },
+ );
+ return {
+ backend: 'android-multitouch-helper',
+ helperVersion: artifact.manifest.version,
+ installReason: install.reason,
+ ...output,
+ };
+}
+
+function normalizeHelperGestureRequest(
+ request: AndroidTouchGestureRequest,
+): AndroidMultiTouchHelperGestureRequest {
+ const durationMs = Math.round(resolveHelperGestureDurationMs(request));
+ switch (request.kind) {
+ case 'pinch':
+ return {
+ kind: 'pinch',
+ x: Math.round(request.x),
+ y: Math.round(request.y),
+ scale: request.scale,
+ radius: ANDROID_MULTITOUCH_HELPER_DEFAULT_RADIUS,
+ durationMs,
+ };
+ case 'rotate':
+ return {
+ kind: 'rotate',
+ x: Math.round(request.x),
+ y: Math.round(request.y),
+ degrees: request.degrees,
+ radius: ANDROID_MULTITOUCH_HELPER_DEFAULT_RADIUS,
+ durationMs,
+ };
+ case 'transform':
+ return {
+ kind: 'transform',
+ x: Math.round(request.x),
+ y: Math.round(request.y),
+ dx: Math.round(request.dx),
+ dy: Math.round(request.dy),
+ scale: request.scale,
+ degrees: request.degrees,
+ durationMs,
+ };
+ }
+}
+
+function resolveHelperGestureDurationMs(request: AndroidTouchGestureRequest): number {
+ if (request.durationMs !== undefined) {
+ return request.durationMs;
+ }
+ if (request.kind === 'pinch') {
+ return ANDROID_MULTITOUCH_HELPER_DEFAULT_DURATION_MS;
+ }
+ const angleBasedDuration =
+ Math.ceil(Math.abs(request.degrees) / ANDROID_MULTITOUCH_HELPER_ROTATE_MAX_DEGREES_PER_FRAME) *
+ ANDROID_MULTITOUCH_HELPER_ROTATE_FRAME_INTERVAL_MS;
+ return Math.min(
+ Math.max(ANDROID_MULTITOUCH_HELPER_DEFAULT_DURATION_MS, angleBasedDuration),
+ ANDROID_MULTITOUCH_HELPER_ROTATE_MAX_DURATION_MS,
+ );
+}
+
+export async function runAndroidMultiTouchHelperGesture(options: {
+ adb: AndroidAdbExecutor;
+ request: AndroidMultiTouchHelperGestureRequest;
+ packageName: string;
+ instrumentationRunner: string;
+}): Promise> {
+ const payloadBase64 = Buffer.from(
+ JSON.stringify({
+ protocol: ANDROID_MULTITOUCH_HELPER_PROTOCOL,
+ ...options.request,
+ }),
+ ).toString('base64');
+ const result = await options.adb(
+ [
+ 'shell',
+ 'am',
+ 'instrument',
+ '-w',
+ '-e',
+ 'payloadBase64',
+ payloadBase64,
+ options.instrumentationRunner,
+ ],
+ { allowFailure: true, timeoutMs: ANDROID_MULTITOUCH_HELPER_GESTURE_TIMEOUT_MS },
+ );
+ let output: Record;
+ try {
+ output = parseAndroidMultiTouchHelperOutput(`${result.stdout}\n${result.stderr}`);
+ } catch (error) {
+ throw new AppError(
+ 'COMMAND_FAILED',
+ result.exitCode === 0
+ ? 'Android multi-touch helper output could not be parsed'
+ : 'Android multi-touch helper failed before returning parseable output',
+ {
+ stdout: result.stdout,
+ stderr: result.stderr,
+ exitCode: result.exitCode,
+ },
+ error,
+ );
+ }
+ if (result.exitCode !== 0) {
+ throw new AppError('COMMAND_FAILED', 'Android multi-touch helper failed', {
+ stdout: result.stdout,
+ stderr: result.stderr,
+ exitCode: result.exitCode,
+ helper: output,
+ });
+ }
+ return output;
+}
+
+export function parseAndroidMultiTouchHelperOutput(output: string): Record {
+ const finalResult = parseInstrumentationResults(output).find(
+ (record) => record.agentDeviceProtocol === ANDROID_MULTITOUCH_HELPER_PROTOCOL,
+ );
+ if (!finalResult) {
+ throw new AppError(
+ 'COMMAND_FAILED',
+ 'Android multi-touch helper did not return a final result',
+ );
+ }
+ if (finalResult.ok !== 'true') {
+ throw new AppError('COMMAND_FAILED', readHelperErrorMessage(finalResult), {
+ errorType: finalResult.errorType,
+ helper: finalResult,
+ });
+ }
+ return {
+ kind: finalResult.kind,
+ helperApiVersion: finalResult.helperApiVersion,
+ injectedEvents: readOptionalNumber(finalResult.injectedEvents),
+ elapsedMs: readOptionalNumber(finalResult.elapsedMs),
+ };
+}
+
+function readHelperErrorMessage(finalResult: Record): string {
+ return finalResult.message && finalResult.message !== 'null'
+ ? finalResult.message
+ : finalResult.errorType || 'Android multi-touch helper returned an error';
+}
+
+function parseInstrumentationResults(output: string): Array> {
+ const results: Array> = [];
+ let current: Record | null = null;
+ for (const line of output.split(/\r?\n/)) {
+ if (line.startsWith('INSTRUMENTATION_RESULT: ')) {
+ current ??= {};
+ readKeyValue(line.slice('INSTRUMENTATION_RESULT: '.length), current);
+ } else if (line.startsWith('INSTRUMENTATION_CODE: ') && current) {
+ results.push(current);
+ current = null;
+ }
+ }
+ if (current) results.push(current);
+ return results;
+}
+
+function readKeyValue(line: string, target: Record): void {
+ const separator = line.indexOf('=');
+ if (separator >= 0) target[line.slice(0, separator)] = line.slice(separator + 1);
+}
+
+function readOptionalNumber(value: string | undefined): number | undefined {
+ if (value === undefined) return undefined;
+ const parsed = Number(value);
+ return Number.isFinite(parsed) ? parsed : undefined;
+}
+
+async function resolveAndroidMultiTouchHelperArtifact(): Promise {
+ const version = readVersion();
+ const helperDir = path.join(findProjectRoot(), 'android-multitouch-helper', 'dist');
+ const manifestPath = path.join(
+ helperDir,
+ `agent-device-android-multitouch-helper-${version}.manifest.json`,
+ );
+ try {
+ const manifest = parseAndroidMultiTouchHelperManifest(
+ JSON.parse(await fs.readFile(manifestPath, 'utf8')),
+ );
+ const apkPath = path.join(helperDir, manifest.assetName);
+ await fs.access(apkPath);
+ return { apkPath, manifest };
+ } catch (error) {
+ throw new AppError(
+ 'UNSUPPORTED_OPERATION',
+ 'gesture pinch/rotate/transform on Android requires the bundled Android multi-touch helper artifact, but it was not found or could not be read',
+ { manifestPath, error: normalizeError(error).message },
+ error,
+ );
+ }
+}
+
+function parseAndroidMultiTouchHelperManifest(value: unknown): AndroidMultiTouchHelperManifest {
+ if (!value || typeof value !== 'object' || Array.isArray(value)) {
+ throw new AppError('INVALID_ARGS', 'Android multi-touch helper manifest must be an object.');
+ }
+ const record = value as Record;
+ return {
+ name: readLiteral(record.name, 'name', ANDROID_MULTITOUCH_HELPER_NAME),
+ version: readString(record.version, 'version'),
+ assetName: readString(record.assetName, 'assetName'),
+ sha256: readSha256(record.sha256),
+ packageName: readLiteral(record.packageName, 'packageName', ANDROID_MULTITOUCH_HELPER_PACKAGE),
+ versionCode: readNumber(record.versionCode, 'versionCode'),
+ instrumentationRunner: readLiteral(
+ record.instrumentationRunner,
+ 'instrumentationRunner',
+ ANDROID_MULTITOUCH_HELPER_RUNNER,
+ ),
+ statusProtocol: readLiteral(
+ record.statusProtocol,
+ 'statusProtocol',
+ ANDROID_MULTITOUCH_HELPER_PROTOCOL,
+ ),
+ };
+}
+
+export async function ensureAndroidMultiTouchHelper(options: {
+ adb: AndroidAdbExecutor;
+ adbProvider: AndroidAdbProvider;
+ artifact: AndroidMultiTouchHelperArtifact;
+ deviceKey: string;
+}): Promise<{
+ packageName: string;
+ versionCode: number;
+ installedVersionCode?: number;
+ installed: boolean;
+ reason: 'missing' | 'outdated' | 'current';
+}> {
+ const { adb, artifact } = options;
+ const packageName = artifact.manifest.packageName;
+ const versionCode = artifact.manifest.versionCode;
+ const cacheKey = `${options.deviceKey}\0${packageName}\0${versionCode}`;
+ if (installedMultiTouchHelpers.has(cacheKey)) {
+ return { packageName, versionCode, installed: false, reason: 'current' };
+ }
+ const installedVersionCode = await readInstalledVersionCode(adb, packageName);
+ if (installedVersionCode !== undefined && installedVersionCode >= versionCode) {
+ installedMultiTouchHelpers.add(cacheKey);
+ return {
+ packageName,
+ versionCode,
+ installedVersionCode,
+ installed: false,
+ reason: 'current',
+ };
+ }
+ await verifyAndroidMultiTouchHelperArtifact(artifact);
+ const result = await installAndroidAdbPackage(artifact.apkPath, {
+ provider: options.adbProvider,
+ replace: true,
+ allowTestPackages: true,
+ allowFailure: true,
+ timeoutMs: ANDROID_MULTITOUCH_HELPER_INSTALL_TIMEOUT_MS,
+ });
+ if (result.exitCode !== 0) {
+ throw new AppError('COMMAND_FAILED', 'Failed to install Android multi-touch helper', {
+ packageName,
+ versionCode,
+ stdout: result.stdout,
+ stderr: result.stderr,
+ exitCode: result.exitCode,
+ });
+ }
+ installedMultiTouchHelpers.add(cacheKey);
+ return {
+ packageName,
+ versionCode,
+ installedVersionCode,
+ installed: true,
+ reason: installedVersionCode === undefined ? 'missing' : 'outdated',
+ };
+}
+
+const installedMultiTouchHelpers = new Set();
+
+export function resetAndroidMultiTouchHelperInstallCache(): void {
+ installedMultiTouchHelpers.clear();
+}
+
+async function readInstalledVersionCode(
+ adb: AndroidAdbExecutor,
+ packageName: string,
+): Promise {
+ const result = await adb(
+ ['shell', 'cmd', 'package', 'list', 'packages', '--show-versioncode', packageName],
+ {
+ allowFailure: true,
+ timeoutMs: 5_000,
+ },
+ );
+ if (result.exitCode !== 0) return undefined;
+ const match = new RegExp(
+ `package:${escapeRegExp(packageName)}(?:\\s|$).*versionCode:(\\d+)`,
+ ).exec(`${result.stdout}\n${result.stderr}`);
+ return match ? Number(match[1]) : undefined;
+}
+
+async function verifyAndroidMultiTouchHelperArtifact(
+ artifact: AndroidMultiTouchHelperArtifact,
+): Promise {
+ const actual = await sha256File(artifact.apkPath);
+ if (actual !== artifact.manifest.sha256) {
+ throw new AppError('COMMAND_FAILED', 'Android multi-touch helper APK checksum mismatch', {
+ apkPath: artifact.apkPath,
+ expectedSha256: artifact.manifest.sha256,
+ actualSha256: actual,
+ });
+ }
+}
+
+async function sha256File(filePath: string): Promise {
+ const hash = crypto.createHash('sha256');
+ hash.update(await fs.readFile(filePath));
+ return hash.digest('hex');
+}
+
+function getAndroidMultiTouchHelperDeviceKey(device: DeviceInfo): string {
+ return `${device.platform}:${device.id}`;
+}
+
+function readString(value: unknown, field: string): string {
+ if (typeof value !== 'string' || value.length === 0) {
+ throw new AppError('INVALID_ARGS', `Android multi-touch helper manifest ${field} is required.`);
+ }
+ return value;
+}
+
+function readNumber(value: unknown, field: string): number {
+ if (!Number.isInteger(value)) {
+ throw new AppError(
+ 'INVALID_ARGS',
+ `Android multi-touch helper manifest ${field} must be an integer.`,
+ );
+ }
+ return value as number;
+}
+
+function readLiteral(value: unknown, field: string, expected: T): T {
+ if (value !== expected) {
+ throw new AppError(
+ 'INVALID_ARGS',
+ `Android multi-touch helper manifest ${field} must be "${expected}".`,
+ );
+ }
+ return expected;
+}
+
+function readSha256(value: unknown): string {
+ const sha256 = readString(value, 'sha256').trim().toLowerCase();
+ if (sha256.length !== 64 || !/^[0-9a-f]+$/.test(sha256)) {
+ throw new AppError(
+ 'INVALID_ARGS',
+ 'Android multi-touch helper manifest sha256 must be a 64-character hex string.',
+ );
+ }
+ return sha256;
+}
+
+function escapeRegExp(value: string): string {
+ return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
diff --git a/src/platforms/ios/__tests__/index.test.ts b/src/platforms/ios/__tests__/index.test.ts
index 7ac43ad51..fa72cd050 100644
--- a/src/platforms/ios/__tests__/index.test.ts
+++ b/src/platforms/ios/__tests__/index.test.ts
@@ -69,6 +69,7 @@ import {
import { ensureBootedSimulator, openIosSimulatorApp } from '../simulator.ts';
import { prepareSimulatorStatusBarForScreenshot as prepareStatusBarForScreenshot } from '../screenshot-status-bar.ts';
import { runIosRunnerCommand } from '../runner-client.ts';
+import { iosRunnerOverrides } from '../interactions.ts';
import type { DeviceInfo } from '../../../utils/device.ts';
import { withDiagnosticsScope } from '../../../utils/diagnostics.ts';
import { AppError } from '../../../utils/errors.ts';
@@ -137,6 +138,46 @@ test('resolveMacOsHelperPackageRootFrom finds helper package from source and dis
}
});
+test('iosRunnerOverrides maps pan duration to the XCUITest drag hold', async () => {
+ mockRunIosRunnerCommand.mockResolvedValue({});
+
+ const { overrides } = iosRunnerOverrides(IOS_TEST_SIMULATOR, {
+ appBundleId: 'com.example.App',
+ });
+
+ await overrides.pan(100, 200, 180, 200, 500);
+
+ assert.deepEqual(mockRunIosRunnerCommand.mock.calls[0]?.[1], {
+ command: 'drag',
+ x: 100,
+ y: 200,
+ x2: 180,
+ y2: 200,
+ durationMs: 500,
+ appBundleId: 'com.example.App',
+ });
+});
+
+test('iosRunnerOverrides gives fling a short default XCUITest drag hold', async () => {
+ mockRunIosRunnerCommand.mockResolvedValue({});
+
+ const { overrides } = iosRunnerOverrides(IOS_TEST_SIMULATOR, {
+ appBundleId: 'com.example.App',
+ });
+
+ await overrides.fling(100, 200, 180, 200, undefined);
+
+ assert.deepEqual(mockRunIosRunnerCommand.mock.calls[0]?.[1], {
+ command: 'drag',
+ x: 100,
+ y: 200,
+ x2: 180,
+ y2: 200,
+ durationMs: 16,
+ appBundleId: 'com.example.App',
+ });
+});
+
test('AGENT_DEVICE_MACOS_HELPER_BIN rejects relative override paths', async () => {
const previousHelperPath = process.env.AGENT_DEVICE_MACOS_HELPER_BIN;
process.env.AGENT_DEVICE_MACOS_HELPER_BIN = './agent-device-macos-helper';
diff --git a/src/platforms/ios/__tests__/recording-scripts.test.ts b/src/platforms/ios/__tests__/recording-scripts.test.ts
index e9bee36e4..59ef3c296 100644
--- a/src/platforms/ios/__tests__/recording-scripts.test.ts
+++ b/src/platforms/ios/__tests__/recording-scripts.test.ts
@@ -11,7 +11,7 @@ const recordingScriptsDir = path.resolve(
'../../../../ios-runner/AgentDeviceRunner/RecordingScripts',
);
const recordingTestSupportDir = path.resolve(__dirname, '../../../../test/integration/support');
-const SWIFT_TYPECHECK_TIMEOUT_MS = 30_000;
+const SWIFT_TYPECHECK_TIMEOUT_MS = 60_000;
async function assertSwiftScriptTypechecks(scriptPath: string): Promise {
const result = await runCmd('xcrun', ['swiftc', '-typecheck', scriptPath], {
diff --git a/src/platforms/ios/__tests__/runner-client.test.ts b/src/platforms/ios/__tests__/runner-client.test.ts
index 199a6cf1d..cdcfeb02b 100644
--- a/src/platforms/ios/__tests__/runner-client.test.ts
+++ b/src/platforms/ios/__tests__/runner-client.test.ts
@@ -141,6 +141,17 @@ const runnerProtocolCommandFixtures: Record;
export function resolveAppleBackRunnerCommand(mode?: BackMode): AppleBackRunnerCommand {
@@ -103,6 +108,36 @@ export function iosRunnerOverrides(
runnerOpts,
);
},
+ pan: async (x1, y1, x2, y2, durationMs) => {
+ return await runIosRunnerCommand(
+ device,
+ {
+ command: 'drag',
+ x: x1,
+ y: y1,
+ x2,
+ y2,
+ durationMs: durationMs ?? 500,
+ appBundleId: ctx.appBundleId,
+ },
+ runnerOpts,
+ );
+ },
+ fling: async (x1, y1, x2, y2, durationMs) => {
+ return await runIosRunnerCommand(
+ device,
+ {
+ command: 'drag',
+ x: x1,
+ y: y1,
+ x2,
+ y2,
+ durationMs: durationMs ?? 16,
+ appBundleId: ctx.appBundleId,
+ },
+ runnerOpts,
+ );
+ },
longPress: async (x, y, durationMs) => {
return await runIosRunnerCommand(
device,
@@ -155,6 +190,50 @@ export function iosRunnerOverrides(
options,
);
},
+ pinch: async (scale, x, y) => {
+ await runIosRunnerCommand(
+ device,
+ {
+ command: 'pinch',
+ scale,
+ x,
+ y,
+ appBundleId: ctx.appBundleId,
+ },
+ runnerOpts,
+ );
+ },
+ rotateGesture: async (degrees, x, y, velocity) => {
+ await runIosRunnerCommand(
+ device,
+ {
+ command: 'rotateGesture',
+ degrees,
+ x,
+ y,
+ velocity,
+ appBundleId: ctx.appBundleId,
+ },
+ runnerOpts,
+ );
+ },
+ transformGesture: async (options) => {
+ return await runIosRunnerCommand(
+ device,
+ {
+ command: 'transformGesture',
+ x: options.x,
+ y: options.y,
+ dx: options.dx,
+ dy: options.dy,
+ scale: options.scale,
+ degrees: options.degrees,
+ durationMs: options.durationMs,
+ appBundleId: ctx.appBundleId,
+ },
+ runnerOpts,
+ );
+ },
},
};
}
diff --git a/src/platforms/ios/runner-contract.ts b/src/platforms/ios/runner-contract.ts
index 17d12f140..598ed94f2 100644
--- a/src/platforms/ios/runner-contract.ts
+++ b/src/platforms/ios/runner-contract.ts
@@ -30,6 +30,8 @@ export type RunnerCommand = {
| 'backSystem'
| 'home'
| 'rotate'
+ | 'rotateGesture'
+ | 'transformGesture'
| 'appSwitcher'
| 'keyboardDismiss'
| 'alert'
@@ -56,10 +58,14 @@ export type RunnerCommand = {
pattern?: 'one-way' | 'ping-pong';
x2?: number;
y2?: number;
+ dx?: number;
+ dy?: number;
durationMs?: number;
direction?: 'up' | 'down' | 'left' | 'right';
orientation?: DeviceRotation;
scale?: number;
+ degrees?: number;
+ velocity?: number;
outPath?: string;
fps?: number;
quality?: number;
diff --git a/src/replay/__tests__/script.test.ts b/src/replay/__tests__/script.test.ts
index 9c79d84f4..9a24995c1 100644
--- a/src/replay/__tests__/script.test.ts
+++ b/src/replay/__tests__/script.test.ts
@@ -107,6 +107,28 @@ test('snapshot replay script parses full refresh flags', () => {
assert.equal(parsed[0]?.flags.snapshotScope, '@e1');
});
+test('gesture replay script parses pan, fling, pinch, and rotate gesture commands', () => {
+ const parsed = parseReplayScript(
+ [
+ 'gesture pan 195 443 80 0',
+ 'wait "pan changed yes" 5000',
+ 'gesture fling right 195 443 180',
+ 'gesture pinch 1.25 195 443',
+ 'gesture rotate 35 195 443',
+ '',
+ ].join('\n'),
+ );
+
+ assert.deepEqual(
+ parsed.map((action) => action.command),
+ ['gesture', 'wait', 'gesture', 'gesture', 'gesture'],
+ );
+ assert.deepEqual(parsed[0]?.positionals, ['pan', '195', '443', '80', '0']);
+ assert.deepEqual(parsed[2]?.positionals, ['fling', 'right', '195', '443', '180']);
+ assert.deepEqual(parsed[3]?.positionals, ['pinch', '1.25', '195', '443']);
+ assert.deepEqual(parsed[4]?.positionals, ['rotate', '35', '195', '443']);
+});
+
test('type and fill replay scripts round-trip typing delay flags', () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-script-typing-'));
const replayPath = path.join(root, 'flow.ad');
diff --git a/src/utils/__tests__/args.test.ts b/src/utils/__tests__/args.test.ts
index fa365cedf..742e5f8dd 100644
--- a/src/utils/__tests__/args.test.ts
+++ b/src/utils/__tests__/args.test.ts
@@ -611,6 +611,32 @@ test('parseArgs recognizes swipe positional + pattern flags', () => {
assert.equal(parsed.flags.pattern, 'ping-pong');
});
+test('parseArgs recognizes gesture subcommand positionals', () => {
+ const pan = parseArgs(['gesture', 'pan', '200', '420', '0', '-80', '500'], {
+ strictFlags: true,
+ });
+ assert.equal(pan.command, 'gesture');
+ assert.deepEqual(pan.positionals, ['pan', '200', '420', '0', '-80', '500']);
+
+ const fling = parseArgs(['gesture', 'fling', 'right', '200', '420', '180'], {
+ strictFlags: true,
+ });
+ assert.equal(fling.command, 'gesture');
+ assert.deepEqual(fling.positionals, ['fling', 'right', '200', '420', '180']);
+
+ const rotate = parseArgs(['gesture', 'rotate', '35', '200', '420'], {
+ strictFlags: true,
+ });
+ assert.equal(rotate.command, 'gesture');
+ assert.deepEqual(rotate.positionals, ['rotate', '35', '200', '420']);
+
+ const transform = parseArgs(['gesture', 'transform', '200', '420', '80', '-40', '2', '35'], {
+ strictFlags: true,
+ });
+ assert.equal(transform.command, 'gesture');
+ assert.deepEqual(transform.positionals, ['transform', '200', '420', '80', '-40', '2', '35']);
+});
+
test('parseArgs recognizes type and fill delay flags', () => {
const typeParsed = parseArgs(['type', 'hello', '--delay-ms', '75'], {
strictFlags: true,
@@ -787,7 +813,11 @@ test('usage includes concise top-level commands', () => {
assert.match(usageText, /clipboard read \| clipboard write /);
assert.match(usageText, /keyboard \[action\]/);
assert.match(usageText, /trigger-app-event \[payloadJson\]/);
- assert.match(usageText, /pinch \[x\] \[y\]/);
+ assert.match(usageText, /gesture \.\.\./);
+ assert.doesNotMatch(usageText, /^ pan \[durationMs\]/m);
+ assert.doesNotMatch(usageText, /^ fling /m);
+ assert.doesNotMatch(usageText, /^ pinch \[x\] \[y\]/m);
+ assert.doesNotMatch(usageText, /^ rotate-gesture /m);
assert.match(usageText, /rotate /);
assert.match(usageText, /record start \[path\] \| record stop/);
assert.match(usageText, /trace start \| trace stop /);
@@ -916,6 +946,8 @@ test('usageForCommand resolves workflow help topic', () => {
assert.match(help, /report that gap instead of typing\/searching\/navigating/);
assert.match(help, /App-owned action sheets, menus, and camera\/scan screens are normal UI/);
assert.match(help, /wait for a concrete result before returning to chat\/form state/);
+ assert.match(help, /choose a point near the center of the intended app-owned target/);
+ assert.match(help, /Avoid screen edges, tab bars, navigation bars, and home indicators/);
assert.match(help, /longpress accepts coordinates, @refs, or selectors/);
assert.match(help, /use help react-native for Metro\/Fast Refresh/);
assert.match(help, /iOS Allow Paste prompt cannot be exercised under XCUITest/);
@@ -929,6 +961,7 @@ test('usageForCommand resolves workflow help topic', () => {
/Do not run open\/press\/fill\/type\/scroll\/back\/alert\/replay\/batch\/close commands in parallel/,
);
assert.match(help, /agent-device clipboard write "some text"/);
+ assert.match(help, /For gesture-heavy iOS simulator proof videos, prefer --hide-touches/);
assert.match(help, /Android Gboard handwriting\/stylus UI can capture text/);
assert.match(help, /targetInput\/actualInput details/);
assert.match(help, /Do not keep retrying fill\/type against the same field/);
@@ -1085,15 +1118,29 @@ test('apps defaults to user-installed filter and allows overrides', () => {
);
});
-test('every capability command has a parser schema entry', () => {
+const INTERNAL_GESTURE_CAPABILITY_COMMANDS = new Set([
+ 'pan',
+ 'fling',
+ 'pinch',
+ 'rotate-gesture',
+ 'transform-gesture',
+]);
+
+test('every public capability command has a parser schema entry', () => {
const schemaCommands = new Set(getCliCommandNames());
for (const command of listCapabilityCommands()) {
+ if (INTERNAL_GESTURE_CAPABILITY_COMMANDS.has(command)) continue;
assert.equal(schemaCommands.has(command), true, `Missing schema for command: ${command}`);
}
});
test('schema capability mappings match capability source-of-truth', () => {
- assert.deepEqual(getSchemaCapabilityKeys(), listCapabilityCommands());
+ assert.deepEqual(
+ getSchemaCapabilityKeys(),
+ listCapabilityCommands().filter(
+ (command) => !INTERNAL_GESTURE_CAPABILITY_COMMANDS.has(command),
+ ),
+ );
});
test('compat mode warns and strips unsupported command flags', () => {
diff --git a/src/utils/__tests__/video.test.ts b/src/utils/__tests__/video.test.ts
index be1b4de7c..4e56c6b9e 100644
--- a/src/utils/__tests__/video.test.ts
+++ b/src/utils/__tests__/video.test.ts
@@ -18,12 +18,15 @@ test('isPlayableVideo falls back to MP4 container validation when swift is unava
await fs.writeFile(videoPath, Buffer.concat([makeAtom('ftyp'), makeAtom('moov')]));
const previousPath = process.env.PATH;
+ const previousSwiftCacheDir = process.env.AGENT_DEVICE_SWIFT_CACHE_DIR;
process.env.PATH = '';
+ process.env.AGENT_DEVICE_SWIFT_CACHE_DIR = path.join(tmpDir, 'swift-cache');
try {
assert.equal(await isPlayableVideo(videoPath), true);
} finally {
process.env.PATH = previousPath;
+ restoreEnv('AGENT_DEVICE_SWIFT_CACHE_DIR', previousSwiftCacheDir);
await fs.rm(tmpDir, { recursive: true, force: true });
}
});
@@ -34,12 +37,23 @@ test('isPlayableVideo fallback rejects files without playable MP4 atoms', async
await fs.writeFile(videoPath, Buffer.concat([makeAtom('ftyp'), makeAtom('mdat')]));
const previousPath = process.env.PATH;
+ const previousSwiftCacheDir = process.env.AGENT_DEVICE_SWIFT_CACHE_DIR;
process.env.PATH = '';
+ process.env.AGENT_DEVICE_SWIFT_CACHE_DIR = path.join(tmpDir, 'swift-cache');
try {
assert.equal(await isPlayableVideo(videoPath), false);
} finally {
process.env.PATH = previousPath;
+ restoreEnv('AGENT_DEVICE_SWIFT_CACHE_DIR', previousSwiftCacheDir);
await fs.rm(tmpDir, { recursive: true, force: true });
}
});
+
+function restoreEnv(name: string, value: string | undefined): void {
+ if (value === undefined) {
+ delete process.env[name];
+ return;
+ }
+ process.env[name] = value;
+}
diff --git a/src/utils/command-schema.ts b/src/utils/command-schema.ts
index 0eebb42e8..28710d73c 100644
--- a/src/utils/command-schema.ts
+++ b/src/utils/command-schema.ts
@@ -236,7 +236,7 @@ Command shape:
Snapshot refs look like @e12. After snapshot -i, use the exact @eN ref from that output.
If the exact ref is not known yet, first output snapshot -i, then use a concrete example shape like press @e12 in the next command; do not write @[, @ref, @Label_Name, or @eN placeholders.
Close means agent-device close. App-owned back means back; system back means back --system.
- Taps are press or click. Gestures are direct commands: swipe, longpress, pinch.
+ Taps are press or click. Gestures use swipe, longpress, or gesture . Android pinch, rotate, and transform use provider-native touch injection when available, then the bundled multi-touch helper.
Bootstrap:
agent-device devices --platform ios
@@ -309,15 +309,21 @@ Read-only and waits:
Ambiguous find: add --first or --last. If info is not visible/exposed, report that gap instead of typing/searching/navigating to reveal it.
Navigation and gestures:
- Use scroll for lists; swipe for coordinate gestures/carousels.
+ Use scroll for lists; swipe for coordinate gestures/carousels; gesture pan for deliberate drags; gesture fling for fast directional throws.
+ For raw coordinate gestures, run snapshot -i first and choose a point near the center of the intended app-owned target. Avoid screen edges, tab bars, navigation bars, and home indicators because those areas can trigger system or app navigation instead of the gesture under test.
If app-owned back is ambiguous or has just misrouted, prefer a visible nav/back button ref, tab-bar ref, or deep link over repeated back/system back.
App-owned action sheets, menus, and camera/scan screens are normal UI. After opening one, run snapshot -i or wait for the option, press by label/ref, handle visible permission sheets through UI or platform-supported native alerts, then wait for a concrete result before returning to chat/form state.
Keep count/pause/pattern on one swipe; flags are --count, --pause-ms, --pattern ping-pong.
- longpress accepts coordinates, @refs, or selectors. Prefer @ref/selector from snapshot -i; use coordinates only as a fallback when accessibility refs miss the exact target. Duration and pinch scale/center are positional:
+ longpress accepts coordinates, @refs, or selectors. Prefer @ref/selector from snapshot -i; use coordinates only as a fallback when accessibility refs miss the exact target. Duration and gesture scale/center are positional:
agent-device longpress 300 500 800
agent-device longpress @e12 800
agent-device swipe 320 500 40 500 --count 8 --pause-ms 30 --pattern ping-pong
- agent-device pinch 0.5 200 400
+ agent-device gesture pan 200 420 0 -80 500
+ agent-device gesture fling right 200 420 180
+ agent-device gesture pinch 0.5 200 400
+ agent-device gesture rotate 35 200 420
+ agent-device gesture transform 200 420 80 -40 2 35 700
+ iOS simulator transform uses XCTest gesture primitives; verify app metrics instead of assuming requested degrees map exactly to recognizer output.
Validation and evidence:
Nearby mutation diff: agent-device diff snapshot -i.
@@ -326,7 +332,7 @@ Validation and evidence:
If task says snapshot, use snapshot. If it asks visual evidence, use screenshot.
Icon/tappable visual proof: screenshot --overlay-refs. Flag is --overlay-refs.
Startup/frame health/CPU/memory: perf --json or metrics. Replay maintenance: replay -u ./flow.ad.
- Recording: record start/stop. By default, stop burns touch overlays into the video; use record start --hide-touches for the fastest raw recording. Tracing: trace start ./trace.log, trace stop ./trace.log. Paths are positional.
+ Recording: record start/stop. By default, stop burns touch overlays into the video; use record start --hide-touches for the fastest raw recording. For gesture-heavy iOS simulator proof videos, prefer --hide-touches because overlay timing depends on a stable runner session while gestures are executing. Tracing: trace start ./trace.log, trace stop ./trace.log. Paths are positional.
Stable known flow: batch ./steps.json, not workflow batch.
Inline batch JSON example:
agent-device batch --steps '[{"command":"open","positionals":["settings"],"flags":{}},{"command":"wait","positionals":["100"],"flags":{}}]'
@@ -1728,6 +1734,17 @@ const COMMAND_SCHEMAS: Record = {
positionalArgs: ['x1', 'y1', 'x2', 'y2', 'durationMs?'],
allowedFlags: ['count', 'pauseMs', 'pattern'],
},
+ gesture: {
+ usageOverride: 'gesture ...',
+ listUsageOverride: 'gesture ...',
+ helpDescription:
+ 'Run touch gestures: pan [durationMs], fling [distance] [durationMs], pinch [x] [y], rotate [x] [y] [velocity], or transform [durationMs]',
+ summary: 'Run pan, fling, pinch, rotate, or transform gestures',
+ positionalArgs: ['pan|fling|pinch|rotate|transform', 'args?'],
+ allowsExtraPositionals: true,
+ allowedFlags: [],
+ skipCapabilityCheck: true,
+ },
focus: {
helpDescription: 'Focus input at coordinates',
positionalArgs: ['x', 'y'],
@@ -1748,11 +1765,6 @@ const COMMAND_SCHEMAS: Record = {
positionalArgs: ['directionOrEdge', 'amount?'],
allowedFlags: ['pixels'],
},
- pinch: {
- helpDescription: 'Pinch/zoom gesture (Apple simulator or macOS app session)',
- positionalArgs: ['scale', 'x?', 'y?'],
- allowedFlags: [],
- },
'trigger-app-event': {
usageOverride: 'trigger-app-event [payloadJson]',
helpDescription: 'Trigger app-defined event hook via deep link template',
diff --git a/test/integration/provider-scenarios/android-lifecycle.test.ts b/test/integration/provider-scenarios/android-lifecycle.test.ts
index 7062d8eee..634d202cb 100644
--- a/test/integration/provider-scenarios/android-lifecycle.test.ts
+++ b/test/integration/provider-scenarios/android-lifecycle.test.ts
@@ -72,6 +72,70 @@ test('Provider-backed integration Android text provider handles Unicode without
);
});
+test('Provider-backed integration Android touch provider handles multi-touch gestures', async () => {
+ await withProviderScenarioResource(
+ async () => await createAndroidSettingsWorld({ nativeTouchInjection: true }),
+ async (world) => {
+ const client = world.daemon.client();
+ await client.apps.open({ app: 'settings', ...world.selection });
+
+ const pinch = await client.interactions.pinch({
+ scale: 2,
+ x: 195,
+ y: 320,
+ ...world.selection,
+ });
+ assert.equal(pinch.scale, 2);
+ assert.equal(pinch.backend, 'provider-native-touch');
+
+ const rotate = await client.interactions.rotateGesture({
+ degrees: 145,
+ x: 195,
+ y: 320,
+ ...world.selection,
+ });
+ assert.equal(rotate.degrees, 145);
+ assert.equal(rotate.backend, 'provider-native-touch');
+
+ const transform = await client.interactions.transformGesture({
+ x: 195,
+ y: 320,
+ dx: 40,
+ dy: -20,
+ scale: 1.5,
+ degrees: 35,
+ durationMs: 700,
+ ...world.selection,
+ });
+ assert.equal(transform.scale, 1.5);
+ assert.equal(transform.degrees, 35);
+ assert.equal(transform.backend, 'provider-native-touch');
+
+ assert.deepEqual(world.touchInjectionCalls, [
+ { kind: 'pinch', x: 195, y: 320, scale: 2, durationMs: undefined },
+ { kind: 'rotate', x: 195, y: 320, degrees: 145, durationMs: undefined },
+ {
+ kind: 'transform',
+ x: 195,
+ y: 320,
+ dx: 40,
+ dy: -20,
+ scale: 1.5,
+ degrees: 35,
+ durationMs: 700,
+ },
+ ]);
+ assert.equal(
+ world.adbCalls.some(
+ (call) => call[0] === 'shell' && call[1] === 'am' && call[2] === 'instrument',
+ ),
+ false,
+ JSON.stringify(world.adbCalls),
+ );
+ },
+ );
+});
+
test('Provider-backed integration Android alert handles runtime permission dialog', async () => {
await withProviderScenarioResource(
async () => await createAndroidSettingsWorld({ snapshotXml: androidRuntimePermissionXml }),
@@ -834,6 +898,34 @@ async function runAndroidCaptureInteractionAndReplayWorkflow(
assert.equal(swipe.pauseMs, 1);
assert.equal(swipe.pattern, 'ping-pong');
+ const pan = await client.interactions.pan({
+ x: 100,
+ y: 200,
+ dx: 50,
+ dy: -20,
+ durationMs: 400,
+ ...selection,
+ });
+ assert.equal(pan.x, 100);
+ assert.equal(pan.y, 200);
+ assert.equal(pan.x2, 150);
+ assert.equal(pan.y2, 180);
+ assert.equal(pan.durationMs, 400);
+
+ const fling = await client.interactions.fling({
+ direction: 'right',
+ x: 100,
+ y: 200,
+ distance: 180,
+ ...selection,
+ });
+ assert.equal(fling.direction, 'right');
+ assert.equal(fling.x, 100);
+ assert.equal(fling.y, 200);
+ assert.equal(fling.x2, 280);
+ assert.equal(fling.y2, 200);
+ assert.equal(fling.distance, 180);
+
const batch = await client.batch.run({
steps: [
{
@@ -1139,6 +1231,8 @@ function assertAndroidInteractionContract(world: AndroidSettingsWorld): void {
assertCommandCall(adbCalls, ['shell', 'input', 'swipe', '31', '40', '31', '40', '5']);
assertCommandCall(adbCalls, ['shell', 'input', 'swipe', '20', '200', '20', '100', '250']);
assertCommandCall(adbCalls, ['shell', 'input', 'swipe', '20', '100', '20', '200', '250']);
+ assertCommandCall(adbCalls, ['shell', 'input', 'swipe', '100', '200', '150', '180', '400']);
+ assertCommandCall(adbCalls, ['shell', 'input', 'swipe', '100', '200', '280', '200', '50']);
assert.equal(
adbCalls.filter((call) => arrayEqual(call, ['shell', 'input', 'tap', '88', '151'])).length,
5,
diff --git a/test/integration/provider-scenarios/android-recording.test.ts b/test/integration/provider-scenarios/android-recording.test.ts
index f138f70c0..11ceb3c61 100644
--- a/test/integration/provider-scenarios/android-recording.test.ts
+++ b/test/integration/provider-scenarios/android-recording.test.ts
@@ -42,7 +42,9 @@ test('Provider-backed integration Android recording flow uses scripted ADB provi
});
const previousPath = process.env.PATH;
+ const previousSwiftCacheDir = process.env.AGENT_DEVICE_SWIFT_CACHE_DIR;
process.env.PATH = tmpDir;
+ process.env.AGENT_DEVICE_SWIFT_CACHE_DIR = path.join(tmpDir, 'swift-cache');
try {
const open = await daemon.callCommand('open', ['settings'], {
@@ -82,6 +84,7 @@ test('Provider-backed integration Android recording flow uses scripted ADB provi
} finally {
await daemon.close();
restoreEnv('PATH', previousPath);
+ restoreEnv('AGENT_DEVICE_SWIFT_CACHE_DIR', previousSwiftCacheDir);
}
},
);
diff --git a/test/integration/provider-scenarios/android-world.ts b/test/integration/provider-scenarios/android-world.ts
index 145d11f4e..63fabae08 100644
--- a/test/integration/provider-scenarios/android-world.ts
+++ b/test/integration/provider-scenarios/android-world.ts
@@ -27,6 +27,11 @@ type AndroidSettingsWorld = {
delayMs?: number;
target?: { x: number; y: number };
}>;
+ touchInjectionCalls: NonNullable extends (
+ request: infer TRequest,
+ ) => unknown
+ ? TRequest[]
+ : never;
inventoryRequests: DeviceInventoryRequest[];
apkInstallCalls: Array<{ apkPath: string; replace?: boolean }>;
bundleInstallCalls: Array<{ bundlePath: string; mode: string }>;
@@ -42,11 +47,13 @@ type AndroidSettingsWorld = {
export async function createAndroidSettingsWorld(options?: {
nativeTextInjection?: boolean;
+ nativeTouchInjection?: boolean;
snapshotXml?: () => string;
}): Promise {
const hostAdbGuard = installFakeHostAdbGuard();
const adbCalls: string[][] = [];
const textInjectionCalls: AndroidSettingsWorld['textInjectionCalls'] = [];
+ const touchInjectionCalls: AndroidSettingsWorld['touchInjectionCalls'] = [];
const inventoryRequests: DeviceInventoryRequest[] = [];
const apkInstallCalls: Array<{ apkPath: string; replace?: boolean }> = [];
const bundleInstallCalls: Array<{ bundlePath: string; mode: string }> = [];
@@ -116,6 +123,12 @@ export async function createAndroidSettingsWorld(options?: {
searchText = request.text;
};
}
+ if (options?.nativeTouchInjection) {
+ adbProvider.touch = async (request) => {
+ touchInjectionCalls.push({ ...request });
+ return { backend: 'provider-native-touch' };
+ };
+ }
const daemon = await createProviderScenarioHarness({
androidAdbProvider: () => adbProvider,
deviceInventoryProvider: async (request) => {
@@ -129,6 +142,7 @@ export async function createAndroidSettingsWorld(options?: {
daemon,
adbCalls,
textInjectionCalls,
+ touchInjectionCalls,
inventoryRequests,
apkInstallCalls,
bundleInstallCalls,
diff --git a/test/integration/provider-scenarios/ios-lifecycle.test.ts b/test/integration/provider-scenarios/ios-lifecycle.test.ts
index 9d9ee44a2..7dcdb67a8 100644
--- a/test/integration/provider-scenarios/ios-lifecycle.test.ts
+++ b/test/integration/provider-scenarios/ios-lifecycle.test.ts
@@ -111,10 +111,50 @@ test('Provider-backed integration iOS Settings flow uses scripted simctl and run
},
{
name: 'pinch current app',
- command: 'pinch',
- positionals: ['0.8', '196', '122'],
+ command: 'gesture',
+ positionals: ['pinch', '0.8', '196', '122'],
expectData: { scale: 0.8, x: 196, y: 122 },
},
+ {
+ name: 'pan current app',
+ command: 'gesture',
+ positionals: ['pan', '196', '122', '80', '0', '500'],
+ expectData: { x: 196, y: 122, dx: 80, dy: 0, x2: 276, y2: 122, durationMs: 500 },
+ },
+ {
+ name: 'fling current app',
+ command: 'gesture',
+ positionals: ['fling', 'right', '196', '122', '180'],
+ expectData: {
+ direction: 'right',
+ x: 196,
+ y: 122,
+ x2: 376,
+ y2: 122,
+ distance: 180,
+ durationMs: 50,
+ },
+ },
+ {
+ name: 'rotate current app content',
+ command: 'gesture',
+ positionals: ['rotate', '35', '196', '122'],
+ expectData: { degrees: 35, x: 196, y: 122, velocity: 1 },
+ },
+ {
+ name: 'transform current app content',
+ command: 'gesture',
+ positionals: ['transform', '196', '122', '40', '-20', '1.5', '35', '700'],
+ expectData: {
+ x: 196,
+ y: 122,
+ dx: 40,
+ dy: -20,
+ scale: 1.5,
+ degrees: 35,
+ durationMs: 700,
+ },
+ },
{
name: 'get ref attrs',
command: 'get',
diff --git a/test/integration/provider-scenarios/ios-record-trace.test.ts b/test/integration/provider-scenarios/ios-record-trace.test.ts
index 8526b8ae8..7a45aadf8 100644
--- a/test/integration/provider-scenarios/ios-record-trace.test.ts
+++ b/test/integration/provider-scenarios/ios-record-trace.test.ts
@@ -196,7 +196,9 @@ test('Provider-backed integration iOS simulator recording flow uses semantic rec
deviceInventoryProvider: async () => [PROVIDER_SCENARIO_IOS_SIMULATOR],
});
const previousPath = process.env.PATH;
+ const previousSwiftCacheDir = process.env.AGENT_DEVICE_SWIFT_CACHE_DIR;
process.env.PATH = tmpDir;
+ process.env.AGENT_DEVICE_SWIFT_CACHE_DIR = path.join(tmpDir, 'swift-cache');
try {
const open = await daemon.callCommand('open', ['com.apple.Preferences'], {
@@ -240,6 +242,7 @@ test('Provider-backed integration iOS simulator recording flow uses semantic rec
} finally {
await daemon.close();
restoreEnv('PATH', previousPath);
+ restoreEnv('AGENT_DEVICE_SWIFT_CACHE_DIR', previousSwiftCacheDir);
}
},
);
diff --git a/test/integration/provider-scenarios/ios-world.ts b/test/integration/provider-scenarios/ios-world.ts
index 7c83bc323..4dd2a34aa 100644
--- a/test/integration/provider-scenarios/ios-world.ts
+++ b/test/integration/provider-scenarios/ios-world.ts
@@ -51,6 +51,67 @@ export async function createIosSettingsWorld(): Promise {
},
result: { pinched: true },
},
+ {
+ command: 'ios.runner.drag',
+ deviceId: PROVIDER_SCENARIO_IOS_SIMULATOR.id,
+ platform: 'ios',
+ request: {
+ command: 'drag',
+ x: 196,
+ y: 122,
+ x2: 276,
+ y2: 122,
+ durationMs: 500,
+ appBundleId: 'com.apple.Preferences',
+ },
+ result: { dragged: true },
+ },
+ {
+ command: 'ios.runner.drag',
+ deviceId: PROVIDER_SCENARIO_IOS_SIMULATOR.id,
+ platform: 'ios',
+ request: {
+ command: 'drag',
+ x: 196,
+ y: 122,
+ x2: 376,
+ y2: 122,
+ durationMs: 50,
+ appBundleId: 'com.apple.Preferences',
+ },
+ result: { flung: true },
+ },
+ {
+ command: 'ios.runner.rotateGesture',
+ deviceId: PROVIDER_SCENARIO_IOS_SIMULATOR.id,
+ platform: 'ios',
+ request: {
+ command: 'rotateGesture',
+ degrees: 35,
+ x: 196,
+ y: 122,
+ velocity: 1,
+ appBundleId: 'com.apple.Preferences',
+ },
+ result: { rotated: true },
+ },
+ {
+ command: 'ios.runner.transformGesture',
+ deviceId: PROVIDER_SCENARIO_IOS_SIMULATOR.id,
+ platform: 'ios',
+ request: {
+ command: 'transformGesture',
+ x: 196,
+ y: 122,
+ dx: 40,
+ dy: -20,
+ scale: 1.5,
+ degrees: 35,
+ durationMs: 700,
+ appBundleId: 'com.apple.Preferences',
+ },
+ result: { transformed: true },
+ },
runnerSnapshot(),
{
command: 'ios.runner.querySelector',
diff --git a/test/skillgym/suites/agent-device-smoke-suite.ts b/test/skillgym/suites/agent-device-smoke-suite.ts
index 70ee1b16a..a2ab28153 100644
--- a/test/skillgym/suites/agent-device-smoke-suite.ts
+++ b/test/skillgym/suites/agent-device-smoke-suite.ts
@@ -1480,7 +1480,7 @@ const SKILL_GUIDANCE_CASES: Case[] = [
'Zoom-out scale: 0.5',
],
task: 'Plan the gesture command to pinch zoom out at the specified center.',
- outputs: [plannedCommand('pinch'), /0\.5/i, /200\s+400/i],
+ outputs: [plannedCommand('gesture pinch'), /0\.5/i, /200\s+400/i],
forbiddenOutputs: [
/(?:^|\s)--scale(?!\w)/i,
/(?:^|\s)--x(?!\w)/i,
@@ -1489,6 +1489,57 @@ const SKILL_GUIDANCE_CASES: Case[] = [
plannedCommand('swipe'),
],
}),
+ makeCase({
+ id: 'gesture-pan-fling-rotate',
+ contract: [
+ 'Platform: iOS simulator',
+ 'Current screen: gesture lab',
+ 'Target center is x=200 y=420',
+ 'The target point is app-owned content away from screen edges, tab bars, navigation bars, and home indicators',
+ 'Need to test a slow upward pan, a right fling, and app-content rotation',
+ 'Pan delta is dx=0 dy=-80 over 500ms',
+ 'Fling distance is 180px',
+ 'Rotation is 35 degrees',
+ ],
+ task: 'Plan direct agent-device gesture commands for the pan, fling, and rotate gesture.',
+ outputs: [
+ plannedCommand('gesture pan'),
+ /200\s+420\s+0\s+-80\s+500/i,
+ plannedCommand('gesture fling'),
+ /right\s+200\s+420\s+180/i,
+ plannedCommand('gesture rotate'),
+ /35\s+200\s+420/i,
+ ],
+ forbiddenOutputs: [
+ plannedCommand('swipe'),
+ plannedCommand('pan'),
+ plannedCommand('fling'),
+ plannedCommand('rotate'),
+ plannedCommand('rotate-gesture'),
+ /--duration-ms/i,
+ ],
+ }),
+ makeCase({
+ id: 'android-gesture-transform',
+ contract: [
+ 'Platform: Android',
+ 'Current screen: gesture lab',
+ 'Target center is x=200 y=420',
+ 'Need one continuous two-finger gesture without lifting fingers',
+ 'Pan delta is dx=80 dy=-40',
+ 'Zoom scale is 2',
+ 'Rotation is 35 degrees',
+ 'Duration is 700ms',
+ ],
+ task: 'Plan the direct agent-device command for the combined pan, zoom, and rotate gesture.',
+ outputs: [plannedCommand('gesture transform'), /200\s+420\s+80\s+-40\s+2\s+35\s+700/i],
+ forbiddenOutputs: [
+ plannedCommand('gesture pan'),
+ plannedCommand('gesture pinch'),
+ plannedCommand('gesture rotate'),
+ plannedCommand('compose-gestures'),
+ ],
+ }),
makeCase({
id: 'settings-animation-stabilizer',
contract: [
diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md
index 9e4ba6032..1c5e99393 100644
--- a/website/docs/docs/client-api.md
+++ b/website/docs/docs/client-api.md
@@ -246,7 +246,7 @@ Additional CLI-backed methods are exposed on their domain groups with typed opti
- `client.apps.push()`
- `client.apps.triggerEvent()`
- `client.capture.diff()`
-- `client.interactions.click()`, `press()`, `longPress()`, `swipe()`, `focus()`, `type()`, `fill()`, `scroll()`, `pinch()`, `get()`, `is()`, `find()`
+- `client.interactions.click()`, `press()`, `longPress()`, `swipe()`, `pan()`, `fling()`, `focus()`, `type()`, `fill()`, `scroll()`, `pinch()`, `rotateGesture()`, `transformGesture()`, `get()`, `is()`, `find()`
- `client.replay.run()` and `client.replay.test()`
- `client.batch.run()`
- `client.observability.perf()`, `logs()`, and `network()`
diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md
index 15a3cd226..75ad3bc77 100644
--- a/website/docs/docs/commands.md
+++ b/website/docs/docs/commands.md
@@ -136,7 +136,7 @@ agent-device screenshot apple-tv.png --platform ios --target tv
- TV target selection supports both simulator/emulator and connected physical devices (AppleTV + AndroidTV).
- tvOS supports the same runner-driven interaction/snapshot flow as iOS (`snapshot`, `wait`, `press`, `fill`, `get`, `scroll`, `back`, `home`, `app-switcher`, `record`, and related selector flows).
- On tvOS, runner `back`/`home`/`app-switcher` map to Siri Remote actions (`menu`, `home`, double-home).
-- tvOS follows iOS simulator-only command semantics for helpers like `pinch`, `settings`, and `push`.
+- tvOS follows iOS simulator-only command semantics for helpers like `gesture pinch`, `settings`, and `push`.
## Desktop targets
@@ -158,7 +158,7 @@ agent-device snapshot -i --platform apple --target desktop
- Status-item apps often expose little or no useful UI through the default macOS `app` surface. Prefer `--surface menubar` for discovery when the app lives in the top menu bar.
- Use `frontmost-app`, `desktop`, and `menubar` mainly for `snapshot`, `get`, `is`, and `wait`.
- If you inspect with `desktop` or `menubar` and then need to click or fill inside one app, open that app in a normal `app` session.
-- macOS also supports `clipboard read|write`, `trigger-app-event`, `logs`, `network dump`, `alert`, `pinch` in app sessions, `settings appearance`, and `settings permission `.
+- macOS also supports `clipboard read|write`, `trigger-app-event`, `logs`, `network dump`, `alert`, `gesture pinch` in app sessions, `settings appearance`, and `settings permission `.
- In macOS app sessions, `screenshot` captures the target app window bounds rather than the full desktop.
- Prefer selector or `@ref`-driven interactions on macOS. Window position can shift between runs, so raw x/y point commands are less stable than snapshot-derived targets.
- Use `click --button secondary` for context menus on macOS, then run `snapshot -i` again.
@@ -255,11 +255,15 @@ agent-device press 300 500 --count 12 --interval-ms 45
agent-device press 300 500 --count 6 --hold-ms 120 --interval-ms 30 --jitter-px 2
agent-device swipe 540 1500 540 500 120
agent-device swipe 540 1500 540 500 120 --count 8 --pause-ms 30 --pattern ping-pong
+agent-device gesture pan 200 420 0 -80 500
+agent-device gesture fling right 200 420 180
agent-device longpress 300 500 800
agent-device scroll down 0.5
agent-device scroll down --pixels 320
-agent-device pinch 2.0 # zoom in 2x (Apple simulator or macOS app session)
-agent-device pinch 0.5 200 400 # zoom out at coordinates (Apple simulator or macOS app session)
+agent-device gesture pinch 2.0 # zoom in 2x
+agent-device gesture pinch 0.5 200 400 # zoom out at coordinates
+agent-device gesture rotate 35 200 420 # rotate app content
+agent-device gesture transform 200 420 80 -40 2 35 700 # combined pan, zoom, and rotate
```
`fill` clears then types. `type` does not clear.
@@ -272,6 +276,11 @@ Android text entry is owned by `agent-device`: provider-native injection when av
`click --button middle` is reserved for future runner support and currently returns an explicit unsupported-operation error on macOS.
`swipe` accepts an optional `durationMs` argument (default `250ms`, range `16..10000`).
On iOS, swipe duration is clamped to a safe range (`16..60ms`) to avoid longpress side effects.
+`gesture pan` accepts `x y dx dy [durationMs]` for deliberate drags. Android preserves the requested travel duration; iOS uses XCTest drag primitives where this value is the pre-drag hold duration.
+`gesture fling` accepts `up|down|left|right x y [distance] [durationMs]` for fast directional throws.
+`gesture rotate` accepts `degrees [x] [y] [velocity]`; the degree sign controls direction and velocity controls speed.
+`gesture transform` accepts `x y dx dy scale degrees [durationMs]` for one combined pan/zoom/rotate gesture on Android and iOS simulators.
+On iOS simulators it is implemented with XCTest gesture primitives, so verify app-level metrics instead of assuming the requested degrees map exactly to recognizer output.
`scroll` accepts either a relative amount (`0.5` means roughly half of the viewport on that axis) or `--pixels ` for a fixed-distance gesture. Large distances are clamped to the usable drag band so the gesture stays reliable across Android, iOS, and macOS.
Default snapshot text output is visible-first, so off-screen interactive content is summarized instead of shown as tappable refs.
When a target only appears in an off-screen summary, use `scroll ` and then take a fresh `snapshot -i`. For repeated checks, a small shell loop is enough:
@@ -289,7 +298,9 @@ done
```
`longpress` is supported on iOS and Android.
-`pinch` is supported on Apple simulators and macOS app sessions.
+`gesture pinch` is supported on Android, Apple simulators, and macOS app sessions.
+`gesture rotate` is supported on Android and iOS simulator app sessions. Use `rotate` for device orientation.
+`gesture transform` is supported on Android and iOS simulator app sessions.
## Find (semantic)
diff --git a/website/docs/index.md b/website/docs/index.md
index 244562c7b..6f7185b93 100644
--- a/website/docs/index.md
+++ b/website/docs/index.md
@@ -18,7 +18,7 @@ features:
- title: Accessibility-first snapshots
details: Accessibility trees give agents compact UI context without forcing screenshot-only reasoning.
- title: Agent-native interactions
- details: Tap, swipe, scroll, focus, type, assert, and find visible UI through refs, selectors, and semantic finders.
+ details: Tap, pan, fling, pinch, rotate, scroll, focus, type, assert, and find visible UI through refs, selectors, and semantic finders.
- title: Built-in debugging and profiling
details: Collect session logs, inspect recent HTTP traffic, capture screenshots and recordings, and sample CPU, memory, startup, and frame-health metrics.
- title: Session and replay
]