From 93e61ba53d2c146af4d1d4e306c890a1f42668c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 21 May 2026 21:34:05 +0200 Subject: [PATCH 1/6] feat: add Maestro replay compatibility --- .../RunnerTests+CommandExecution.swift | 18 +- .../RunnerTests+Interaction.swift | 41 ++- .../RunnerTests+Models.swift | 1 + src/compat/__tests__/replay-input.test.ts | 4 +- .../maestro/__tests__/replay-flow.test.ts | 242 +++++++------ src/compat/maestro/command-mapper.ts | 157 ++------- src/compat/maestro/device-actions.ts | 233 ++----------- src/compat/maestro/flow-control.ts | 203 +++++++++++ src/compat/maestro/interactions.ts | 138 ++++---- src/compat/maestro/points.ts | 57 ++++ src/compat/maestro/run-script.ts | 136 ++++++++ src/compat/maestro/runtime-commands.ts | 7 + src/compat/maestro/support.ts | 2 +- src/compat/maestro/types.ts | 2 - src/core/dispatch-context.ts | 6 + src/core/dispatch.ts | 2 + src/core/interactor-types.ts | 9 +- src/core/interactors/apple.ts | 1 + src/daemon/__tests__/context.test.ts | 6 + src/daemon/context.ts | 2 + src/daemon/direct-ios-selector.ts | 1 + .../handlers/__tests__/interaction.test.ts | 37 ++ .../__tests__/session-replay-vars.test.ts | 260 ++++++++++++++ src/daemon/handlers/interaction-touch.ts | 7 +- .../session-replay-maestro-runtime.ts | 318 ++++++++++++++++++ src/daemon/handlers/session-replay-runtime.ts | 37 +- src/daemon/types.ts | 1 + src/platforms/ios/apps.ts | 12 +- src/platforms/ios/interactions.ts | 3 +- src/platforms/ios/runner-contract.ts | 1 + src/utils/__tests__/args.test.ts | 3 +- src/utils/__tests__/video.test.ts | 12 +- src/utils/command-schema.ts | 2 +- website/docs/docs/replay-e2e.md | 4 +- 34 files changed, 1418 insertions(+), 547 deletions(-) create mode 100644 src/compat/maestro/flow-control.ts create mode 100644 src/compat/maestro/points.ts create mode 100644 src/compat/maestro/run-script.ts create mode 100644 src/compat/maestro/runtime-commands.ts create mode 100644 src/daemon/handlers/session-replay-maestro-runtime.ts diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 17c1323da..b71f570cb 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -252,7 +252,12 @@ extension RunnerTests { ) case .tap: if let selectorKey = command.selectorKey, let selectorValue = command.selectorValue { - let match = findElement(app: activeApp, selectorKey: selectorKey, selectorValue: selectorValue) + let match = findElement( + app: activeApp, + selectorKey: selectorKey, + selectorValue: selectorValue, + allowNonHittableFallback: command.allowNonHittableSelectorTap == true + ) if match.isAmbiguous { return Response(ok: false, error: ErrorPayload(code: "AMBIGUOUS_MATCH", message: "selector matched multiple elements")) } @@ -264,7 +269,14 @@ extension RunnerTests { var outcome = RunnerInteractionOutcome.performed let timing = measureGesture { withTemporaryScrollIdleTimeoutIfSupported(activeApp) { - outcome = activateElement(app: activeApp, element: element, action: "tap by selector") + if match.usedNonHittableFallback { + // Maestro compatibility: RN E2E backdoor controls can be 1x1 and + // reported non-hittable by XCTest, while Maestro still taps their + // resolved bounds. Keep this behind the explicit replay-only flag. + outcome = tapAt(app: activeApp, x: frame.midX, y: frame.midY) + } else { + outcome = activateElement(app: activeApp, element: element, action: "tap by selector") + } } } if let response = unsupportedResponse(for: outcome) { @@ -273,7 +285,7 @@ extension RunnerTests { return Response( ok: true, data: DataPayload( - message: "tapped", + message: match.usedNonHittableFallback ? "tapped via non-hittable coordinate fallback" : "tapped", gestureStartUptimeMs: timing.gestureStartUptimeMs, gestureEndUptimeMs: timing.gestureEndUptimeMs, x: touchFrame?.x, diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 778b81a13..138117c8d 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -27,6 +27,7 @@ extension RunnerTests { struct SelectorElementMatch { let element: XCUIElement? let isAmbiguous: Bool + let usedNonHittableFallback: Bool } enum TextTypingRepairMode { @@ -177,10 +178,15 @@ extension RunnerTests { return element.exists ? element : nil } - func findElement(app: XCUIApplication, selectorKey: String, selectorValue: String) -> SelectorElementMatch { + func findElement( + app: XCUIApplication, + selectorKey: String, + selectorValue: String, + allowNonHittableFallback: Bool = false + ) -> SelectorElementMatch { let value = selectorValue.trimmingCharacters(in: .whitespacesAndNewlines) guard !value.isEmpty else { - return SelectorElementMatch(element: nil, isAmbiguous: false) + return SelectorElementMatch(element: nil, isAmbiguous: false, usedNonHittableFallback: false) } let predicate: NSPredicate switch selectorKey { @@ -193,21 +199,44 @@ extension RunnerTests { case "text": predicate = NSPredicate(format: "label ==[c] %@ OR identifier ==[c] %@ OR value ==[c] %@", value, value, value) default: - return SelectorElementMatch(element: nil, isAmbiguous: false) + return SelectorElementMatch(element: nil, isAmbiguous: false, usedNonHittableFallback: false) } var matchedElement: XCUIElement? + var nonHittableElement: XCUIElement? let matches = app.descendants(matching: .any).matching(predicate).allElementsBoundByIndex for element in matches where element.exists { - guard element.isHittable else { + if !element.isHittable { + if allowNonHittableFallback && hasTappableFrame(app: app, element: element) { + guard nonHittableElement == nil else { + return SelectorElementMatch(element: nil, isAmbiguous: true, usedNonHittableFallback: false) + } + nonHittableElement = element + } continue } guard matchedElement == nil else { - return SelectorElementMatch(element: nil, isAmbiguous: true) + return SelectorElementMatch(element: nil, isAmbiguous: true, usedNonHittableFallback: false) } matchedElement = element } - return SelectorElementMatch(element: matchedElement, isAmbiguous: false) + if let matchedElement { + return SelectorElementMatch(element: matchedElement, isAmbiguous: false, usedNonHittableFallback: false) + } + return SelectorElementMatch( + element: nonHittableElement, + isAmbiguous: false, + usedNonHittableFallback: nonHittableElement != nil + ) + } + + private func hasTappableFrame(app: XCUIApplication, element: XCUIElement) -> Bool { + let frame = element.frame + if frame.isEmpty { + return false + } + let appFrame = app.frame + return appFrame.isEmpty || appFrame.intersects(frame) } func queryElement(app: XCUIApplication, selectorKey: String, selectorValue: String) -> Response { diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index de5fa632f..13a295e69 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -39,6 +39,7 @@ struct Command: Codable { let text: String? let selectorKey: String? let selectorValue: String? + let allowNonHittableSelectorTap: Bool? let delayMs: Int? let textEntryMode: String? let clearFirst: Bool? diff --git a/src/compat/__tests__/replay-input.test.ts b/src/compat/__tests__/replay-input.test.ts index 225b5bc56..a863f519c 100644 --- a/src/compat/__tests__/replay-input.test.ts +++ b/src/compat/__tests__/replay-input.test.ts @@ -32,7 +32,7 @@ test('parseReplayInput routes compat replay scripts through the selected parser' parsed.actions.map((action) => [action.command, action.positionals]), [ ['open', ['com.callstack.agentdevicelab']], - ['click', ['id="submit-order"']], + ['__maestroTapOn', ['id="submit-order"']], ], ); }); @@ -60,7 +60,7 @@ env: parsed.actions.map((action) => [action.command, action.positionals]), [ ['open', ['cli-app']], - ['click', ['id="shell-button"']], + ['__maestroTapOn', ['id="shell-button"']], ], ); }); diff --git a/src/compat/maestro/__tests__/replay-flow.test.ts b/src/compat/maestro/__tests__/replay-flow.test.ts index cf17308d5..f50fb6c3f 100644 --- a/src/compat/maestro/__tests__/replay-flow.test.ts +++ b/src/compat/maestro/__tests__/replay-flow.test.ts @@ -14,6 +14,8 @@ env: - launchApp - tapOn: id: home-open-form +- tapOn: + point: 20%,20% - doubleTapOn: id: release-notice delay: 150 @@ -37,6 +39,11 @@ env: start: 50%, 75% end: 50%, 35% duration: 300 +- swipe: + direction: LEFT +- scrollUntilVisible: + element: Discover + direction: UP - takeScreenshot: ./screens/form.png - hideKeyboard - stopApp @@ -47,34 +54,87 @@ env: parsed.actions.map((entry) => [entry.command, entry.positionals]), [ ['open', ['com.callstack.agentdevicelab']], - ['click', ['id="home-open-form"']], + ['__maestroTapOn', ['id="home-open-form"']], + ['__maestroTapPointPercent', ['20', '20']], ['click', ['id="release-notice"']], ['click', ['label="Agent Device Tester"']], ['open', ['exp://localhost:8082']], - ['click', ['label="Full name" || text="Full name" || id="Full name"']], + ['__maestroTapOn', ['label="Full name" || text="Full name" || id="Full name"']], ['type', ['Ada Lovelace']], ['wait', ['label="Checkout form"', '5000']], ['is', ['hidden', 'label="Missing banner"']], ['wait', ['id="submit-order"', '7000']], ['scroll', ['down']], ['scroll', ['down', '0.4']], + ['scroll', ['right']], + [ + '__maestroScrollUntilVisible', + ['label="Discover" || text="Discover" || id="Discover"', '5000', 'down'], + ], ['screenshot', ['./screens/form.png']], ['keyboard', ['dismiss']], ['close', ['com.callstack.agentdevicelab']], ], ); - assert.equal(parsed.actions[2]?.flags.doubleTap, true); - assert.equal(parsed.actions[2]?.flags.intervalMs, 150); - assert.equal(parsed.actions[3]?.flags.holdMs, 3000); + assert.equal(parsed.actions[3]?.flags.doubleTap, true); + assert.equal(parsed.actions[3]?.flags.intervalMs, 150); + assert.equal(parsed.actions[4]?.flags.holdMs, 3000); + assert.equal(parsed.actions[1]?.flags.allowNonHittableSelectorTap, true); + assert.equal(parsed.actions[6]?.flags?.allowNonHittableSelectorTap, undefined); +}); + +test('parseMaestroReplayFlow maps iOS openLink through the app id when available', () => { + const parsed = parseMaestroReplayFlow( + `appId: com.callstack.agentdevicelab +--- +- openLink: exp://localhost:8082 +`, + { platform: 'ios' }, + ); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [['open', ['com.callstack.agentdevicelab', 'exp://localhost:8082']]], + ); +}); + +test('parseMaestroReplayFlow executes runScript and exposes output variables', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-maestro-runscript-')); + const scriptPath = path.join(root, 'setup.js'); + const flowPath = path.join(root, 'flow.yml'); + fs.writeFileSync( + scriptPath, + ` +var res = {body: '{"appviewDid":"did:plc:test"}'} +output.result = SERVER_PATH + ':' + json(res.body).appviewDid +`, + ); + + const parsed = parseMaestroReplayFlow( + `appId: com.callstack.agentdevicelab +--- +- runScript: + file: ./setup.js + env: + SERVER_PATH: local +- inputText: \${output.result} +`, + { sourcePath: flowPath }, + ); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [['type', ['local:did:plc:test']]], + ); }); test('parseMaestroReplayFlow rejects unsupported Maestro commands', () => { assert.throws( - () => parseMaestroReplayFlow('---\n- scrollUntilVisible: Save\n'), + () => parseMaestroReplayFlow('---\n- travelThroughTime: Save\n'), (error) => error instanceof AppError && error.code === 'INVALID_ARGS' && - /scrollUntilVisible/.test(error.message) && + /travelThroughTime/.test(error.message) && /issues\/558/.test(error.message) && /issues\/new/.test(error.message) && /line 2/.test(error.message), @@ -103,52 +163,7 @@ test('parseMaestroReplayFlow preserves selector state and absolute swipe command assert.deepEqual(parsed.actionLines, [3, 6]); }); -test('parseMaestroReplayFlow maps easy Maestro device and utility commands', () => { - const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab -env: - VIDEO_PATH: ./recordings/checkout.mp4 ---- -- setAirplaneMode: true -- setAirplaneMode: false -- setLocation: - latitude: 52.2297 - longitude: 21.0122 -- setOrientation: landscapeLeft -- setPermissions: - camera: allow - microphone: deny - photos: unset - location: always -- killApp -- killApp: com.callstack.other -- pasteText: hello there -- startRecording: - path: \${VIDEO_PATH} -- stopRecording -- assertTrue: true -`); - - assert.deepEqual( - parsed.actions.map((entry) => [entry.command, entry.positionals]), - [ - ['settings', ['airplane', 'on']], - ['settings', ['airplane', 'off']], - ['settings', ['location', 'set', '52.2297', '21.0122']], - ['rotate', ['landscape-left']], - ['settings', ['permission', 'grant', 'camera']], - ['settings', ['permission', 'deny', 'microphone']], - ['settings', ['permission', 'reset', 'photos']], - ['settings', ['permission', 'grant', 'location-always']], - ['close', ['com.callstack.agentdevicelab']], - ['close', ['com.callstack.other']], - ['type', ['hello there']], - ['record', ['start', './recordings/checkout.mp4']], - ['record', ['stop']], - ], - ); -}); - -test('parseMaestroReplayFlow rejects unsupported easy-mapping variants loudly', () => { +test('parseMaestroReplayFlow rejects deferred Maestro utility commands loudly', () => { assert.throws( () => parseMaestroReplayFlow('---\n- assertTrue: "${READY}"\n'), (error) => @@ -160,11 +175,11 @@ test('parseMaestroReplayFlow rejects unsupported easy-mapping variants loudly', ); assert.throws( - () => parseMaestroReplayFlow('---\n- setPermissions:\n camera: always\n'), + () => parseMaestroReplayFlow('---\n- setPermissions:\n camera: allow\n'), (error) => error instanceof AppError && error.code === 'INVALID_ARGS' && - /setPermissions state "always"/.test(error.message) && + /setPermissions/.test(error.message) && /issues\/558/.test(error.message) && /line 2/.test(error.message), ); @@ -196,12 +211,12 @@ test('parseMaestroReplayFlow reports top-level command lines around nested lists - runFlow: commands: - tapOn: Nested -- scrollUntilVisible: Save +- travelThroughTime: Save `), (error) => error instanceof AppError && error.code === 'INVALID_ARGS' && - /scrollUntilVisible/.test(error.message) && + /travelThroughTime/.test(error.message) && /line 6/.test(error.message), ); }); @@ -251,14 +266,14 @@ onFlowComplete: assert.deepEqual( parsed.actions.map((entry) => [entry.command, entry.positionals]), [ - ['click', ['label="Before" || text="Before" || id="Before"']], - ['click', ['label="Nested" || text="Nested" || id="Nested"']], - ['click', ['id="child-repeat"']], - ['click', ['id="child-repeat"']], - ['click', ['label="iOS only" || text="iOS only" || id="iOS only"']], - ['click', ['label="Again" || text="Again" || id="Again"']], - ['click', ['label="Again" || text="Again" || id="Again"']], - ['click', ['label="After" || text="After" || id="After"']], + ['__maestroTapOn', ['label="Before" || text="Before" || id="Before"']], + ['__maestroTapOn', ['label="Nested" || text="Nested" || id="Nested"']], + ['__maestroTapOn', ['id="child-repeat"']], + ['__maestroTapOn', ['id="child-repeat"']], + ['__maestroTapOn', ['label="iOS only" || text="iOS only" || id="iOS only"']], + ['__maestroTapOn', ['label="Again" || text="Again" || id="Again"']], + ['__maestroTapOn', ['label="Again" || text="Again" || id="Again"']], + ['__maestroTapOn', ['label="After" || text="After" || id="After"']], ], ); }); @@ -279,57 +294,66 @@ test('parseMaestroReplayFlow skips platform-gated runFlow commands for other pla assert.deepEqual( parsed.actions.map((entry) => [entry.command, entry.positionals]), - [['click', ['label="Shared" || text="Shared" || id="Shared"']]], + [['__maestroTapOn', ['label="Shared" || text="Shared" || id="Shared"']]], + ); +}); + +test('parseMaestroReplayFlow keeps visible-gated runFlow commands for runtime evaluation', () => { + const parsed = parseMaestroReplayFlow( + `appId: com.callstack.agentdevicelab +--- +- runFlow: + when: + visible: Continue + commands: + - tapOn: Continue +`, + { platform: 'ios' }, ); + + assert.equal(parsed.actions[0]?.command, '__maestroRunFlowWhen'); + assert.deepEqual(parsed.actions[0]?.positionals, [ + 'visible', + 'label="Continue" || text="Continue" || id="Continue"', + ]); + assert.deepEqual(parsed.actions[0]?.flags.batchSteps, [ + { + command: '__maestroTapOn', + positionals: ['label="Continue" || text="Continue" || id="Continue"'], + flags: {}, + }, + ]); }); -test('parseMaestroReplayFlow tolerates false launchApp reset options and rejects reset side effects', () => { +test('parseMaestroReplayFlow accepts launchApp reset options without state-reset side effects', () => { const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab --- - launchApp: - clearState: false - clearKeychain: false + clearState: true + clearKeychain: true + arguments: + "-EXDevMenuIsOnboardingFinished": true + launchArguments: + "-Example": "ignored" stopApp: true `); assert.deepEqual( parsed.actions.map((entry) => [entry.command, entry.positionals, entry.flags]), - [['open', ['com.callstack.agentdevicelab'], { relaunch: true }]], - ); - - assert.throws( - () => - parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab ---- -- launchApp: - clearState: true -`), - (error) => - error instanceof AppError && - error.code === 'INVALID_ARGS' && - /clearState: true/.test(error.message) && - /line 3/.test(error.message), + [ + [ + 'open', + ['com.callstack.agentdevicelab'], + { + relaunch: true, + launchArgs: ['-EXDevMenuIsOnboardingFinished', 'true', '-Example', 'ignored'], + }, + ], + ], ); }); -test('parseMaestroReplayFlow rejects runtime-dependent flow control for now', () => { - assert.throws( - () => - parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab ---- -- runFlow: - when: - visible: Continue - commands: - - tapOn: Continue -`), - (error) => - error instanceof AppError && - error.code === 'INVALID_ARGS' && - /when.visible/.test(error.message) && - /line 3/.test(error.message), - ); - +test('parseMaestroReplayFlow rejects unsupported runtime-dependent flow control', () => { assert.throws( () => parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab @@ -360,21 +384,21 @@ test('parseMaestroReplayFlow parses the test-app Maestro suite fixture', () => { parsed.actions.map((entry) => entry.command), [ 'wait', - 'click', + '__maestroTapOn', 'wait', - 'click', + '__maestroTapOn', 'type', - 'click', + '__maestroTapOn', 'type', - 'click', + '__maestroTapOn', 'wait', 'wait', 'scroll', - 'click', + '__maestroTapOn', 'wait', - 'click', + '__maestroTapOn', 'wait', - 'click', + '__maestroTapOn', 'wait', 'wait', ], diff --git a/src/compat/maestro/command-mapper.ts b/src/compat/maestro/command-mapper.ts index 76a3bc1b7..0d1f12c81 100644 --- a/src/compat/maestro/command-mapper.ts +++ b/src/compat/maestro/command-mapper.ts @@ -1,23 +1,13 @@ import type { SessionAction } from '../../daemon/types.ts'; import { AppError } from '../../utils/errors.ts'; -import { - convertAssertTrue, - convertKillApp, - convertLaunchApp, - convertSetAirplaneMode, - convertSetLocation, - convertSetOrientation, - convertSetPermissions, - convertStartRecording, - convertStopApp, - convertStopRecording, -} from './device-actions.ts'; +import { convertLaunchApp, convertStopApp } from './device-actions.ts'; import { convertDoubleTapOn, convertExtendedWaitUntil, convertLongPressOn, convertPressKey, convertScroll, + convertScrollUntilVisible, convertSwipe, convertTapOn, maestroSelector, @@ -25,17 +15,14 @@ import { } from './interactions.ts'; import { action, - assertOnlyKeys, - isPlainRecord, - normalizeCommandList, - normalizePlatformValue, - readEnvMap, readTimeoutMs, + requireAppId, requireStringValue, resolveMaestroString, unsupportedCommand, - unsupportedMaestroSyntax, } from './support.ts'; +import { convertRepeat, convertRunFlow } from './flow-control.ts'; +import { executeRunScript } from './run-script.ts'; import type { MaestroCommand, MaestroCommandMapperDeps, @@ -43,7 +30,6 @@ import type { MaestroParseContext, } from './types.ts'; -const MAX_REPEAT_EXPANSIONS = 100; type MaestroCommandHandler = (params: { value: unknown; config: MaestroFlowConfig; @@ -63,36 +49,33 @@ const MAP_COMMAND_HANDLERS: Record = { pasteText: ({ value, context, name }) => [ action('type', [resolveMaestroString(requireStringValue(name, value), context)]), ], - openLink: ({ value, context, name }) => [ - action('open', [resolveMaestroString(requireStringValue(name, value), context)]), - ], + openLink: ({ value, config, context, name }) => [convertOpenLink(value, config, context, name)], assertVisible: ({ value, context, name }) => [ action('wait', [maestroSelector(value, name, [], context), '5000']), ], assertNotVisible: ({ value, context, name }) => [ action('is', ['hidden', maestroSelector(value, name, [], context)]), ], - assertTrue: ({ value, context }) => convertAssertTrue(value, context), extendedWaitUntil: ({ value, context }) => convertExtendedWaitUntil(value, context), takeScreenshot: ({ value, context, name }) => [ action('screenshot', [resolveMaestroString(requireStringValue(name, value), context)]), ], scroll: ({ value }) => [convertScroll(value)], + scrollUntilVisible: ({ value, context }) => convertScrollUntilVisible(value, context), swipe: ({ value }) => [convertSwipe(value)], hideKeyboard: () => [action('keyboard', ['dismiss'])], pressKey: ({ value }) => [convertPressKey(value)], back: () => [action('back')], waitForAnimationToEnd: ({ value }) => [action('wait', [String(readTimeoutMs(value, 250))])], stopApp: ({ value, config, context }) => [convertStopApp(value, config, context)], - killApp: ({ value, config, context }) => [convertKillApp(value, config, context)], - setAirplaneMode: ({ value, context }) => [convertSetAirplaneMode(value, context)], - setLocation: ({ value, context }) => [convertSetLocation(value, context)], - setOrientation: ({ value, context }) => [convertSetOrientation(value, context)], - setPermissions: ({ value, context }) => convertSetPermissions(value, context), - startRecording: ({ value, context }) => [convertStartRecording(value, context)], - stopRecording: ({ value }) => [convertStopRecording(value)], - runFlow: ({ value, config, context, deps }) => convertRunFlow(value, config, context, deps), - repeat: ({ value, config, context, deps }) => convertRepeat(value, config, context, deps), + runScript: ({ value, context }) => { + executeRunScript(value, context); + return []; + }, + runFlow: ({ value, config, context, deps }) => + convertRunFlow(value, config, context, deps, convertCommandList), + repeat: ({ value, config, context, deps }) => + convertRepeat(value, config, context, deps, convertCommandList), }; const SCALAR_COMMAND_HANDLERS: Record< @@ -105,9 +88,6 @@ const SCALAR_COMMAND_HANDLERS: Record< back: () => [action('back')], waitForAnimationToEnd: () => [action('wait', ['250'])], stopApp: (config, context) => [convertStopApp(undefined, config, context)], - killApp: (config, context) => [convertKillApp(undefined, config, context)], - startRecording: () => [action('record', ['start'])], - stopRecording: () => [action('record', ['stop'])], }; export function convertMaestroCommandWithLine( @@ -156,63 +136,17 @@ function convertScalarCommand( return handler(config, context); } -function convertRunFlow( +function convertOpenLink( value: unknown, config: MaestroFlowConfig, context: MaestroParseContext, - deps: MaestroCommandMapperDeps, -): SessionAction[] { - if (typeof value === 'string') { - return deps.parseRunFlowFile(resolveMaestroString(value, context), context).actions; - } - if (!isPlainRecord(value)) { - throw new AppError('INVALID_ARGS', 'runFlow expects a file path string or map.'); - } - assertOnlyKeys(value, 'runFlow', ['file', 'commands', 'env', 'when', 'label']); - if (!shouldRunFlow(value.when, context)) return []; - - const runContext = { - ...context, - env: { ...context.env, ...readEnvMap(value.env, 'runFlow.env'), ...context.envOverrides }, - }; - if (typeof value.file === 'string') { - return deps.parseRunFlowFile(resolveMaestroString(value.file, runContext), runContext).actions; - } - if (Array.isArray(value.commands)) { - return convertCommandList(normalizeCommandList(value.commands), config, runContext, deps); - } - throw new AppError('INVALID_ARGS', 'runFlow map requires either file or commands.'); -} - -function convertRepeat( - value: unknown, - config: MaestroFlowConfig, - context: MaestroParseContext, - deps: MaestroCommandMapperDeps, -): SessionAction[] { - if (!isPlainRecord(value)) { - throw new AppError('INVALID_ARGS', 'repeat expects a map.'); - } - assertOnlyKeys(value, 'repeat', ['times', 'commands', 'while']); - if (value.while !== undefined) { - throw unsupportedMaestroSyntax( - 'Maestro repeat.while is not supported yet. Only deterministic repeat.times is supported.', - ); - } - const times = readRepeatTimes(value.times, context); - if (!Array.isArray(value.commands)) { - throw new AppError('INVALID_ARGS', 'repeat requires a commands list.'); - } - if (times > MAX_REPEAT_EXPANSIONS) { - throw new AppError( - 'INVALID_ARGS', - `repeat.times must be <= ${MAX_REPEAT_EXPANSIONS} for deterministic replay expansion.`, - ); + name: string, +): SessionAction { + const url = resolveMaestroString(requireStringValue(name, value), context); + if (context.platform === 'ios' && config.appId) { + return action('open', [resolveMaestroString(requireAppId(config, name), context), url]); } - const commands = normalizeCommandList(value.commands); - return Array.from({ length: times }).flatMap(() => - convertCommandList(commands, config, context, deps), - ); + return action('open', [url]); } function convertCommandList( @@ -225,50 +159,3 @@ function convertCommandList( convertMaestroCommandWithLine(command, config, index + 1, context, deps), ); } - -function shouldRunFlow(value: unknown, context: MaestroParseContext): boolean { - if (value === undefined || value === null) return true; - if (!isPlainRecord(value)) { - throw new AppError('INVALID_ARGS', 'runFlow.when expects a map.'); - } - assertOnlyKeys(value, 'runFlow.when', ['platform', 'visible', 'notVisible', 'true']); - rejectUnsupportedCondition(value, 'visible', 'when.visible'); - rejectUnsupportedCondition(value, 'notVisible', 'when.notVisible'); - rejectUnsupportedCondition(value, 'true', 'when.true'); - if (value.platform === undefined) return true; - const platform = normalizePlatformValue(value.platform, 'runFlow.when.platform'); - if (!context.platform) { - throw new AppError( - 'INVALID_ARGS', - 'Maestro runFlow.when.platform requires replay to be run with --platform ios|android.', - ); - } - return platform === context.platform; -} - -function readRepeatTimes(value: unknown, context: MaestroParseContext): number { - const resolved = typeof value === 'string' ? resolveMaestroString(value, context) : value; - const numeric = - typeof resolved === 'number' - ? resolved - : typeof resolved === 'string' && /^\d+$/.test(resolved) - ? Number(resolved) - : undefined; - if (numeric === undefined || !Number.isInteger(numeric) || numeric < 0) { - throw new AppError( - 'INVALID_ARGS', - 'repeat.times must be a non-negative integer or ${VAR} resolving to one.', - ); - } - return numeric; -} - -function rejectUnsupportedCondition( - value: Record, - key: string, - label: string, -): void { - if (value[key] !== undefined) { - throw unsupportedMaestroSyntax(`Maestro ${label} is not supported yet.`); - } -} diff --git a/src/compat/maestro/device-actions.ts b/src/compat/maestro/device-actions.ts index 95e33db4d..8069b2fab 100644 --- a/src/compat/maestro/device-actions.ts +++ b/src/compat/maestro/device-actions.ts @@ -4,50 +4,11 @@ import { action, assertOnlyKeys, isPlainRecord, - normalizeToken, - readBooleanLiteral, requireAppId, resolveMaestroString, - resolveMaybeMaestroString, unsupportedMaestroSyntax, } from './support.ts'; -import type { MaestroFlowConfig, MaestroParseContext, PermissionCommand } from './types.ts'; - -const SUPPORTED_PERMISSION_TARGETS = new Set([ - 'accessibility', - 'calendar', - 'camera', - 'contacts', - 'contacts-limited', - 'input-monitoring', - 'location', - 'location-always', - 'media-library', - 'microphone', - 'motion', - 'notifications', - 'photos', - 'reminders', - 'screen-recording', - 'siri', -]); - -const BASIC_PERMISSION_STATES: Record = { - allow: 'grant', - grant: 'grant', - granted: 'grant', - deny: 'deny', - denied: 'deny', - reset: 'reset', - unset: 'reset', - revoke: 'reset', - revoked: 'reset', -}; - -const MODE_PERMISSION_STATES: Record = { - limited: { command: 'grant', mode: 'limited' }, - full: { command: 'grant', mode: 'full' }, -}; +import type { MaestroFlowConfig, MaestroParseContext } from './types.ts'; export function convertLaunchApp( value: unknown, @@ -70,16 +31,17 @@ export function convertLaunchApp( 'permissions', 'launchArguments', ]); - rejectTruthyLaunchOption(value, 'clearState'); - rejectTruthyLaunchOption(value, 'clearKeychain'); - rejectUnsupportedLaunchOption(value, 'arguments'); rejectUnsupportedLaunchOption(value, 'permissions'); - rejectUnsupportedLaunchOption(value, 'launchArguments'); const appId = resolveMaestroString( typeof value.appId === 'string' ? value.appId : requireAppId(config, 'launchApp'), context, ); - return action('open', [appId], { relaunch: value.stopApp === true }); + const launchArgs = readLaunchArgs(value, context); + const shouldRelaunch = value.stopApp === true || launchArgs.length > 0; + return action('open', [appId], { + relaunch: shouldRelaunch, + ...(launchArgs.length > 0 ? { launchArgs } : {}), + }); } export function convertStopApp( @@ -94,173 +56,32 @@ export function convertStopApp( throw new AppError('INVALID_ARGS', 'stopApp expects a string appId or no value.'); } -export function convertSetAirplaneMode( - value: unknown, - context: MaestroParseContext, -): SessionAction { - const enabled = readBooleanLiteral(resolveMaybeMaestroString(value, context), 'setAirplaneMode'); - return action('settings', ['airplane', enabled ? 'on' : 'off']); -} - -export function convertSetLocation(value: unknown, context: MaestroParseContext): SessionAction { - if (!isPlainRecord(value)) { - throw new AppError('INVALID_ARGS', 'setLocation expects a map.'); - } - assertOnlyKeys(value, 'setLocation', ['latitude', 'longitude', 'lat', 'lon', 'lng']); - const latitude = readCoordinate(value.latitude ?? value.lat, 'setLocation.latitude', context); - const longitude = readCoordinate( - value.longitude ?? value.lon ?? value.lng, - 'setLocation.longitude', - context, - ); - return action('settings', ['location', 'set', latitude, longitude]); -} - -export function convertSetOrientation(value: unknown, context: MaestroParseContext): SessionAction { - const raw = resolveMaybeMaestroString(value, context); - if (typeof raw !== 'string') { - throw new AppError('INVALID_ARGS', 'setOrientation expects a string value.'); - } - const orientation = normalizeToken(raw); - switch (orientation) { - case 'portrait': - case 'landscape-left': - case 'landscape-right': - return action('rotate', [orientation]); - case 'portrait-upside-down': - case 'upside-down': - return action('rotate', ['portrait-upside-down']); - default: - throw unsupportedMaestroSyntax( - `Maestro setOrientation "${raw}" cannot be mapped to a supported rotate orientation.`, - ); - } +function readLaunchArgs(value: Record, context: MaestroParseContext): string[] { + return [ + ...readLaunchArgValue(value.arguments, 'launchApp.arguments', context), + ...readLaunchArgValue(value.launchArguments, 'launchApp.launchArguments', context), + ]; } -export function convertSetPermissions( - value: unknown, - context: MaestroParseContext, -): SessionAction[] { - if (!isPlainRecord(value)) { - throw new AppError('INVALID_ARGS', 'setPermissions expects a map.'); +function readLaunchArgValue(value: unknown, name: string, context: MaestroParseContext): string[] { + if (value === undefined || value === null) return []; + if (typeof value === 'string') return [resolveMaestroString(value, context)]; + if (Array.isArray(value)) { + return value.map((entry, index) => readLaunchArgScalar(entry, `${name}[${index}]`, context)); } - return Object.entries(value).map(([rawTarget, rawState]) => { - const { target, command, mode } = readPermissionMapping(rawTarget, rawState, context); - return action('settings', ['permission', command, target, ...(mode ? [mode] : [])]); - }); -} - -export function convertKillApp( - value: unknown, - config: MaestroFlowConfig, - context: MaestroParseContext, -): SessionAction { - if (value === null || value === undefined) { - return action('close', [resolveMaestroString(requireAppId(config, 'killApp'), context)]); + if (isPlainRecord(value)) { + return Object.entries(value).flatMap(([key, entry]) => [ + resolveMaestroString(key, context), + readLaunchArgScalar(entry, `${name}.${key}`, context), + ]); } - if (typeof value === 'string') return action('close', [resolveMaestroString(value, context)]); - throw new AppError('INVALID_ARGS', 'killApp expects a string appId or no value.'); + throw new AppError('INVALID_ARGS', `${name} expects a string, list, or map.`); } -export function convertStartRecording(value: unknown, context: MaestroParseContext): SessionAction { - if (value === null || value === undefined) return action('record', ['start']); - if (typeof value === 'string') - return action('record', ['start', resolveMaestroString(value, context)]); - if (!isPlainRecord(value)) { - throw new AppError('INVALID_ARGS', 'startRecording expects a string path, map, or no value.'); - } - assertOnlyKeys(value, 'startRecording', ['path', 'file']); - const rawPath = value.path ?? value.file; - if (rawPath === undefined) return action('record', ['start']); - if (typeof rawPath !== 'string') { - throw new AppError('INVALID_ARGS', 'startRecording path must be a string.'); - } - return action('record', ['start', resolveMaestroString(rawPath, context)]); -} - -export function convertStopRecording(value: unknown): SessionAction { - if (value !== null && value !== undefined) { - throw new AppError('INVALID_ARGS', 'stopRecording expects no value.'); - } - return action('record', ['stop']); -} - -export function convertAssertTrue(value: unknown, context: MaestroParseContext): SessionAction[] { - const resolved = resolveMaybeMaestroString(value, context); - if (resolved === true || (typeof resolved === 'string' && normalizeToken(resolved) === 'true')) { - return []; - } - if ( - resolved === false || - (typeof resolved === 'string' && normalizeToken(resolved) === 'false') - ) { - throw new AppError('INVALID_ARGS', 'Maestro assertTrue literal evaluated to false.'); - } - throw unsupportedMaestroSyntax('Only literal Maestro assertTrue true/false is supported.'); -} - -function readCoordinate(value: unknown, name: string, context: MaestroParseContext): string { - const resolved = resolveMaybeMaestroString(value, context); - const numeric = - typeof resolved === 'number' - ? resolved - : typeof resolved === 'string' && resolved.trim().length > 0 - ? Number(resolved) - : Number.NaN; - if (!Number.isFinite(numeric)) { - throw new AppError('INVALID_ARGS', `${name} must be a finite number.`); - } - return String(numeric); -} - -function readPermissionMapping( - rawTarget: string, - rawState: unknown, - context: MaestroParseContext, -): { target: string; command: PermissionCommand; mode?: string } { - let target = normalizeToken(rawTarget); - const resolvedState = resolveMaybeMaestroString(rawState, context); - if (typeof resolvedState !== 'string') { - throw new AppError('INVALID_ARGS', `setPermissions.${rawTarget} expects a string state.`); - } - const state = normalizeToken(resolvedState); - if (target === 'location' && state === 'always') target = 'location-always'; - - if (!SUPPORTED_PERMISSION_TARGETS.has(target)) { - throw unsupportedMaestroSyntax( - `Maestro setPermissions target "${rawTarget}" cannot be mapped to a supported settings permission target.`, - ); - } - - const basicCommand = BASIC_PERMISSION_STATES[state]; - if (basicCommand) return { target, command: basicCommand }; - - const modeMapping = MODE_PERMISSION_STATES[state]; - if (modeMapping) return { target, ...modeMapping }; - - const locationCommand = readLocationPermissionCommand(target, state); - if (locationCommand) return { target, command: locationCommand }; - - throw unsupportedMaestroSyntax( - `Maestro setPermissions state "${resolvedState}" cannot be mapped to grant, deny, or reset.`, - ); -} - -function readLocationPermissionCommand( - target: string, - state: string, -): PermissionCommand | undefined { - if (target === 'location-always' && state === 'always') return 'grant'; - if (target === 'location' && (state === 'while-in-use' || state === 'when-in-use')) { - return 'grant'; - } - return undefined; -} - -function rejectTruthyLaunchOption(value: Record, key: string): void { - if (value[key] === true) { - throw unsupportedMaestroSyntax(`Maestro launchApp ${key}: true is not supported yet.`); - } +function readLaunchArgScalar(value: unknown, name: string, context: MaestroParseContext): string { + if (typeof value === 'string') return resolveMaestroString(value, context); + if (typeof value === 'number' || typeof value === 'boolean') return String(value); + throw new AppError('INVALID_ARGS', `${name} must be a string, number, or boolean.`); } function rejectUnsupportedLaunchOption(value: Record, key: string): void { diff --git a/src/compat/maestro/flow-control.ts b/src/compat/maestro/flow-control.ts new file mode 100644 index 000000000..d11e8af87 --- /dev/null +++ b/src/compat/maestro/flow-control.ts @@ -0,0 +1,203 @@ +import type { CommandFlags } from '../../core/dispatch.ts'; +import type { SessionAction } from '../../daemon/types.ts'; +import { AppError } from '../../utils/errors.ts'; +import { maestroSelector } from './interactions.ts'; +import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; +import { + action, + assertOnlyKeys, + isPlainRecord, + normalizeCommandList, + normalizePlatformValue, + readEnvMap, + resolveMaestroString, + unsupportedMaestroSyntax, +} from './support.ts'; +import type { + MaestroCommand, + MaestroCommandMapperDeps, + MaestroFlowConfig, + MaestroParseContext, +} from './types.ts'; + +const MAX_REPEAT_EXPANSIONS = 100; + +type ConvertCommandList = ( + commands: MaestroCommand[], + config: MaestroFlowConfig, + context: MaestroParseContext, + deps: MaestroCommandMapperDeps, +) => SessionAction[]; + +export function convertRunFlow( + value: unknown, + config: MaestroFlowConfig, + context: MaestroParseContext, + deps: MaestroCommandMapperDeps, + convertCommandList: ConvertCommandList, +): SessionAction[] { + if (typeof value === 'string') { + return deps.parseRunFlowFile(resolveMaestroString(value, context), context).actions; + } + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'runFlow expects a file path string or map.'); + } + assertOnlyKeys(value, 'runFlow', ['file', 'commands', 'env', 'when', 'label']); + const condition = readRunFlowCondition(value.when, context); + if (!condition.shouldRun) return []; + + const runContext = { + ...context, + env: { ...context.env, ...readEnvMap(value.env, 'runFlow.env'), ...context.envOverrides }, + }; + const actions = readRunFlowActions(value, config, runContext, deps, convertCommandList); + return wrapRunFlowCondition(actions, condition); +} + +export function convertRepeat( + value: unknown, + config: MaestroFlowConfig, + context: MaestroParseContext, + deps: MaestroCommandMapperDeps, + convertCommandList: ConvertCommandList, +): SessionAction[] { + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'repeat expects a map.'); + } + assertOnlyKeys(value, 'repeat', ['times', 'commands', 'while']); + if (value.while !== undefined) { + throw unsupportedMaestroSyntax( + 'Maestro repeat.while is not supported yet. Only deterministic repeat.times is supported.', + ); + } + const times = readRepeatTimes(value.times, context); + if (!Array.isArray(value.commands)) { + throw new AppError('INVALID_ARGS', 'repeat requires a commands list.'); + } + if (times > MAX_REPEAT_EXPANSIONS) { + throw new AppError( + 'INVALID_ARGS', + `repeat.times must be <= ${MAX_REPEAT_EXPANSIONS} for deterministic replay expansion.`, + ); + } + const commands = normalizeCommandList(value.commands); + return Array.from({ length: times }).flatMap(() => + convertCommandList(commands, config, context, deps), + ); +} + +function readRunFlowActions( + value: Record, + config: MaestroFlowConfig, + context: MaestroParseContext, + deps: MaestroCommandMapperDeps, + convertCommandList: ConvertCommandList, +): SessionAction[] { + if (typeof value.file === 'string') { + return deps.parseRunFlowFile(resolveMaestroString(value.file, context), context).actions; + } + if (Array.isArray(value.commands)) { + return convertCommandList(normalizeCommandList(value.commands), config, context, deps); + } + throw new AppError('INVALID_ARGS', 'runFlow map requires either file or commands.'); +} + +type RunFlowCondition = { + shouldRun: boolean; + visibleSelector?: string; + notVisibleSelector?: string; +}; + +function readRunFlowCondition(value: unknown, context: MaestroParseContext): RunFlowCondition { + if (value === undefined || value === null) return { shouldRun: true }; + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'runFlow.when expects a map.'); + } + assertOnlyKeys(value, 'runFlow.when', ['platform', 'visible', 'notVisible', 'true']); + rejectUnsupportedCondition(value, 'true', 'when.true'); + if (value.platform !== undefined) { + const platform = normalizePlatformValue(value.platform, 'runFlow.when.platform'); + if (!context.platform) { + throw new AppError( + 'INVALID_ARGS', + 'Maestro runFlow.when.platform requires replay to be run with --platform ios|android.', + ); + } + if (platform !== context.platform) return { shouldRun: false }; + } + return { + shouldRun: true, + ...(value.visible !== undefined + ? { visibleSelector: maestroSelector(value.visible, 'runFlow.when.visible', [], context) } + : {}), + ...(value.notVisible !== undefined + ? { + notVisibleSelector: maestroSelector( + value.notVisible, + 'runFlow.when.notVisible', + [], + context, + ), + } + : {}), + }; +} + +function wrapRunFlowCondition( + actions: SessionAction[], + condition: RunFlowCondition, +): SessionAction[] { + if (!condition.visibleSelector && !condition.notVisibleSelector) return actions; + if (condition.visibleSelector && condition.notVisibleSelector) { + throw unsupportedMaestroSyntax( + 'Maestro runFlow.when cannot combine visible and notVisible yet.', + ); + } + return [ + action( + MAESTRO_RUNTIME_COMMAND.runFlowWhen, + condition.visibleSelector + ? ['visible', condition.visibleSelector] + : ['notVisible', condition.notVisibleSelector ?? ''], + { batchSteps: actions.map(sessionActionToBatchStep) }, + ), + ]; +} + +function sessionActionToBatchStep( + entry: SessionAction, +): NonNullable[number] { + return { + command: entry.command, + positionals: entry.positionals, + flags: entry.flags, + ...(entry.runtime !== undefined ? { runtime: entry.runtime } : {}), + }; +} + +function readRepeatTimes(value: unknown, context: MaestroParseContext): number { + const resolved = typeof value === 'string' ? resolveMaestroString(value, context) : value; + const numeric = + typeof resolved === 'number' + ? resolved + : typeof resolved === 'string' && /^\d+$/.test(resolved) + ? Number(resolved) + : undefined; + if (numeric === undefined || !Number.isInteger(numeric) || numeric < 0) { + throw new AppError( + 'INVALID_ARGS', + 'repeat.times must be a non-negative integer or ${VAR} resolving to one.', + ); + } + return numeric; +} + +function rejectUnsupportedCondition( + value: Record, + key: string, + label: string, +): void { + if (value[key] !== undefined) { + throw unsupportedMaestroSyntax(`Maestro ${label} is not supported yet.`); + } +} diff --git a/src/compat/maestro/interactions.ts b/src/compat/maestro/interactions.ts index c61271606..8fa5b1b13 100644 --- a/src/compat/maestro/interactions.ts +++ b/src/compat/maestro/interactions.ts @@ -9,12 +9,30 @@ import { resolveMaestroString, unsupportedMaestroSyntax, } from './support.ts'; +import { + parseAbsolutePoint, + parseMaestroPoint, + readScrollPositionalsFromPercentSwipe, +} from './points.ts'; +import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; import type { MaestroParseContext } from './types.ts'; export function convertTapOn(value: unknown, context: MaestroParseContext): SessionAction { + if (typeof value === 'string') { + return action(MAESTRO_RUNTIME_COMMAND.tapOn, [ + visibleTextSelector(resolveMaestroString(value, context)), + ]); + } if (isPlainRecord(value) && typeof value.point === 'string') { assertOnlyKeys(value, 'tapOn', ['point', 'repeat', 'delay']); - const point = parsePoint(value.point); + const point = parseMaestroPoint(value.point); + if (point.kind === 'percent') { + return action( + MAESTRO_RUNTIME_COMMAND.tapPointPercent, + [String(point.x), String(point.y)], + tapFlags(value), + ); + } return action('click', [String(point.x), String(point.y)], tapFlags(value)); } if (isPlainRecord(value)) { @@ -30,16 +48,16 @@ export function convertTapOn(value: unknown, context: MaestroParseContext): Sess ]); } return action( - 'click', + MAESTRO_RUNTIME_COMMAND.tapOn, [maestroSelector(value, 'tapOn', ['repeat', 'delay', 'optional', 'label'], context)], - tapFlags(value), + { ...tapFlags(value), allowNonHittableSelectorTap: true }, ); } export function convertDoubleTapOn(value: unknown, context: MaestroParseContext): SessionAction { if (isPlainRecord(value) && typeof value.point === 'string') { assertOnlyKeys(value, 'doubleTapOn', ['point', 'delay']); - const point = parsePoint(value.point); + const point = parseAbsolutePoint(value.point); return action('click', [String(point.x), String(point.y)], doubleTapFlags(value)); } if (isPlainRecord(value)) { @@ -55,7 +73,7 @@ export function convertDoubleTapOn(value: unknown, context: MaestroParseContext) export function convertLongPressOn(value: unknown, context: MaestroParseContext): SessionAction { if (isPlainRecord(value) && typeof value.point === 'string') { assertOnlyKeys(value, 'longPressOn', ['point']); - const point = parsePoint(value.point); + const point = parseAbsolutePoint(value.point); return action('longpress', [String(point.x), String(point.y), '3000']); } if (isPlainRecord(value)) { @@ -105,16 +123,45 @@ export function convertScroll(value: unknown): SessionAction { return action('scroll', ['down']); } +export function convertScrollUntilVisible( + value: unknown, + context: MaestroParseContext, +): SessionAction[] { + if (typeof value === 'string') { + return [ + action(MAESTRO_RUNTIME_COMMAND.scrollUntilVisible, [ + visibleTextSelector(resolveMaestroString(value, context)), + '5000', + 'down', + ]), + ]; + } + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'scrollUntilVisible expects a string or map.'); + } + assertOnlyKeys(value, 'scrollUntilVisible', ['element', 'direction', 'timeout']); + const selector = maestroSelector(value.element, 'scrollUntilVisible.element', [], context); + const direction = + typeof value.direction === 'string' + ? readScrollPositionalsFromDirectionSwipe(value.direction)[0] + : 'down'; + const timeoutMs = String(readTimeoutMs(value, 5000)); + return [action(MAESTRO_RUNTIME_COMMAND.scrollUntilVisible, [selector, timeoutMs, direction])]; +} + export function convertSwipe(value: unknown): SessionAction { if (!isPlainRecord(value)) { throw new AppError('INVALID_ARGS', 'swipe expects a map.'); } - assertOnlyKeys(value, 'swipe', ['start', 'end', 'duration']); + assertOnlyKeys(value, 'swipe', ['start', 'end', 'direction', 'duration']); + if (typeof value.direction === 'string') { + return action('scroll', readScrollPositionalsFromDirectionSwipe(value.direction)); + } if (typeof value.start !== 'string' || typeof value.end !== 'string') { throw unsupportedMaestroSyntax('Only Maestro swipe start/end coordinates are supported.'); } - const start = parseSwipePoint(value.start); - const end = parseSwipePoint(value.end); + const start = parseMaestroPoint(value.start); + const end = parseMaestroPoint(value.end); const durationMs = typeof value.duration === 'number' && Number.isFinite(value.duration) ? String(Math.max(16, Math.floor(value.duration))) @@ -136,10 +183,25 @@ export function convertSwipe(value: unknown): SessionAction { ); } +function readScrollPositionalsFromDirectionSwipe(direction: string): string[] { + switch (direction.toLowerCase()) { + case 'up': + return ['down']; + case 'down': + return ['up']; + case 'left': + return ['right']; + case 'right': + return ['left']; + default: + throw unsupportedMaestroSyntax('Maestro swipe direction must be UP, DOWN, LEFT, or RIGHT.'); + } +} + export function convertPressKey(value: unknown): SessionAction { const key = requireStringValue('pressKey', value).toLowerCase(); if (key === 'back') return action('back'); - if (key === 'enter' || key === 'return') return action('press', ['return']); + if (key === 'enter' || key === 'return') return action(MAESTRO_RUNTIME_COMMAND.pressEnter); if (key === 'home') return action('home'); throw unsupportedMaestroSyntax(`Maestro pressKey "${key}" is not supported yet.`); } @@ -195,6 +257,9 @@ function tapFlags(value: unknown): SessionAction['flags'] | undefined { if (typeof value.delay === 'number' && Number.isInteger(value.delay) && value.delay >= 0) { flags.intervalMs = value.delay; } + if (value.optional === true) { + flags.maestroOptional = true; + } return Object.keys(flags).length > 0 ? flags : undefined; } @@ -205,58 +270,3 @@ function doubleTapFlags(value: unknown): SessionAction['flags'] { } return flags; } - -function parsePoint(value: string): { x: number; y: number } { - const match = value.match(/^(\d+),(\d+)$/); - if (!match) { - throw unsupportedMaestroSyntax( - 'Only absolute Maestro point selectors like "100,200" are supported.', - ); - } - return { x: Number(match[1]), y: Number(match[2]) }; -} - -type SwipePoint = - | { - kind: 'absolute'; - x: number; - y: number; - } - | { - kind: 'percent'; - x: number; - y: number; - }; - -function parseSwipePoint(value: string): SwipePoint { - const absolute = value.match(/^\s*(\d+)\s*,\s*(\d+)\s*$/); - if (absolute) { - return { kind: 'absolute', x: Number(absolute[1]), y: Number(absolute[2]) }; - } - const percent = value.match(/^\s*(\d+(?:\.\d+)?)%\s*,\s*(\d+(?:\.\d+)?)%\s*$/); - if (percent) { - return { kind: 'percent', x: Number(percent[1]), y: Number(percent[2]) }; - } - throw unsupportedMaestroSyntax( - 'Only Maestro swipe coordinates like "100,200" or "50%,75%" are supported.', - ); -} - -function readScrollPositionalsFromPercentSwipe( - start: Extract, - end: Extract, -): string[] { - const deltaX = end.x - start.x; - const deltaY = end.y - start.y; - if (Math.abs(deltaX) === 0 && Math.abs(deltaY) === 0) { - throw new AppError('INVALID_ARGS', 'swipe start and end cannot be the same point.'); - } - const vertical = Math.abs(deltaY) >= Math.abs(deltaX); - const direction = vertical ? (deltaY < 0 ? 'down' : 'up') : deltaX < 0 ? 'right' : 'left'; - const amount = Math.min(1, Math.max(0.01, Math.abs(vertical ? deltaY : deltaX) / 100)); - return [direction, formatAmount(amount)]; -} - -function formatAmount(value: number): string { - return value.toFixed(2).replace(/0+$/, '').replace(/\.$/, ''); -} diff --git a/src/compat/maestro/points.ts b/src/compat/maestro/points.ts new file mode 100644 index 000000000..c0e25ffb4 --- /dev/null +++ b/src/compat/maestro/points.ts @@ -0,0 +1,57 @@ +import { AppError } from '../../utils/errors.ts'; +import { unsupportedMaestroSyntax } from './support.ts'; + +export type MaestroPoint = + | { + kind: 'absolute'; + x: number; + y: number; + } + | { + kind: 'percent'; + x: number; + y: number; + }; + +export function parseAbsolutePoint(value: string): { x: number; y: number } { + const match = value.match(/^(\d+),(\d+)$/); + if (!match) { + throw unsupportedMaestroSyntax( + 'Only absolute Maestro point selectors like "100,200" are supported.', + ); + } + return { x: Number(match[1]), y: Number(match[2]) }; +} + +export function parseMaestroPoint(value: string): MaestroPoint { + const absolute = value.match(/^\s*(\d+)\s*,\s*(\d+)\s*$/); + if (absolute) { + return { kind: 'absolute', x: Number(absolute[1]), y: Number(absolute[2]) }; + } + const percent = value.match(/^\s*(\d+(?:\.\d+)?)%\s*,\s*(\d+(?:\.\d+)?)%\s*$/); + if (percent) { + return { kind: 'percent', x: Number(percent[1]), y: Number(percent[2]) }; + } + throw unsupportedMaestroSyntax( + 'Only Maestro swipe coordinates like "100,200" or "50%,75%" are supported.', + ); +} + +export function readScrollPositionalsFromPercentSwipe( + start: Extract, + end: Extract, +): string[] { + const deltaX = end.x - start.x; + const deltaY = end.y - start.y; + if (Math.abs(deltaX) === 0 && Math.abs(deltaY) === 0) { + throw new AppError('INVALID_ARGS', 'swipe start and end cannot be the same point.'); + } + const vertical = Math.abs(deltaY) >= Math.abs(deltaX); + const direction = vertical ? (deltaY < 0 ? 'down' : 'up') : deltaX < 0 ? 'right' : 'left'; + const amount = Math.min(1, Math.max(0.01, Math.abs(vertical ? deltaY : deltaX) / 100)); + return [direction, formatAmount(amount)]; +} + +function formatAmount(value: number): string { + return value.toFixed(2).replace(/0+$/, '').replace(/\.$/, ''); +} diff --git a/src/compat/maestro/run-script.ts b/src/compat/maestro/run-script.ts new file mode 100644 index 000000000..8f810aa2b --- /dev/null +++ b/src/compat/maestro/run-script.ts @@ -0,0 +1,136 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import vm from 'node:vm'; +import { AppError } from '../../utils/errors.ts'; +import { runCmdSync } from '../../utils/exec.ts'; +import { + assertOnlyKeys, + isPlainRecord, + readEnvMap, + requireStringValue, + resolveMaestroString, +} from './support.ts'; +import type { MaestroParseContext } from './types.ts'; + +type HttpResponse = { + status: number; + body: string; + headers: Record; +}; + +const HTTP_REQUEST_SCRIPT = ` +const fs = require('node:fs'); +const input = JSON.parse(fs.readFileSync(0, 'utf8')); +fetch(input.url, { + method: input.method, + headers: input.headers, + body: input.body, +}).then(async response => { + process.stdout.write(JSON.stringify({ + status: response.status, + body: await response.text(), + headers: Object.fromEntries(response.headers.entries()), + })); +}).catch(error => { + console.error(error && error.stack ? error.stack : String(error)); + process.exit(1); +}); +`; + +export function executeRunScript(value: unknown, context: MaestroParseContext): void { + const scriptConfig = readRunScriptConfig(value, context); + const scriptPath = resolveRunScriptPath(scriptConfig.file, context); + const script = fs.readFileSync(scriptPath, 'utf8'); + const output: Record = {}; + const scriptEnv = { + ...context.env, + ...scriptConfig.env, + ...context.envOverrides, + }; + + try { + vm.runInNewContext(script, buildScriptGlobals(scriptEnv, output), { + filename: scriptPath, + timeout: 30_000, + }); + } catch (error) { + throw new AppError( + 'COMMAND_FAILED', + `Maestro runScript failed for ${scriptPath}: ${error instanceof Error ? error.message : String(error)}`, + { scriptPath }, + error instanceof Error ? error : undefined, + ); + } + + for (const [key, rawValue] of Object.entries(output)) { + context.env[`output.${key}`] = stringifyOutputValue(rawValue); + } +} + +function readRunScriptConfig( + value: unknown, + context: MaestroParseContext, +): { file: string; env: Record } { + if (typeof value === 'string') { + return { file: resolveMaestroString(value, context), env: {} }; + } + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'runScript expects a file path string or map.'); + } + assertOnlyKeys(value, 'runScript', ['file', 'env']); + const file = resolveMaestroString(requireStringValue('runScript.file', value.file), context); + const rawEnv = readEnvMap(value.env, 'runScript.env'); + const env = Object.fromEntries( + Object.entries(rawEnv).map(([key, envValue]) => [key, resolveMaestroString(envValue, context)]), + ); + return { file, env }; +} + +function resolveRunScriptPath(filePath: string, context: MaestroParseContext): string { + if (path.isAbsolute(filePath)) return filePath; + if (!context.baseDir) { + throw new AppError( + 'INVALID_ARGS', + 'runScript file paths require replay input to have a source path.', + ); + } + return path.resolve(context.baseDir, filePath); +} + +function buildScriptGlobals( + env: Record, + output: Record, +): vm.Context { + return { + ...env, + output, + json: (value: string) => JSON.parse(value) as unknown, + http: { + post: (url: string, options?: { headers?: Record; body?: string }) => + runHttpRequestSync('POST', url, options), + }, + }; +} + +function runHttpRequestSync( + method: string, + url: string, + options?: { headers?: Record; body?: string }, +): HttpResponse { + const result = runCmdSync(process.execPath, ['-e', HTTP_REQUEST_SCRIPT], { + stdin: JSON.stringify({ + method, + url, + headers: options?.headers ?? {}, + body: options?.body ?? '', + }), + timeoutMs: 30_000, + }); + return JSON.parse(result.stdout) as HttpResponse; +} + +function stringifyOutputValue(value: unknown): string { + if (typeof value === 'string') return value; + if (typeof value === 'number' || typeof value === 'boolean') return String(value); + return JSON.stringify(value); +} diff --git a/src/compat/maestro/runtime-commands.ts b/src/compat/maestro/runtime-commands.ts new file mode 100644 index 000000000..02b816a4c --- /dev/null +++ b/src/compat/maestro/runtime-commands.ts @@ -0,0 +1,7 @@ +export const MAESTRO_RUNTIME_COMMAND = { + pressEnter: '__maestroPressEnter', + runFlowWhen: '__maestroRunFlowWhen', + scrollUntilVisible: '__maestroScrollUntilVisible', + tapOn: '__maestroTapOn', + tapPointPercent: '__maestroTapPointPercent', +} as const; diff --git a/src/compat/maestro/support.ts b/src/compat/maestro/support.ts index 3bd998faf..591e23ecb 100644 --- a/src/compat/maestro/support.ts +++ b/src/compat/maestro/support.ts @@ -113,7 +113,7 @@ export function requireStringValue(command: string, value: unknown): string { } export function resolveMaestroString(value: string, context: MaestroParseContext): string { - return value.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g, (match, key: string) => { + return value.replace(/\$\{([A-Za-z_][A-Za-z0-9_.]*)\}/g, (match, key: string) => { return Object.prototype.hasOwnProperty.call(context.env, key) ? context.env[key] : match; }); } diff --git a/src/compat/maestro/types.ts b/src/compat/maestro/types.ts index 81012d5e7..8ea39be92 100644 --- a/src/compat/maestro/types.ts +++ b/src/compat/maestro/types.ts @@ -31,5 +31,3 @@ export type MaestroParseContext = { export type MaestroCommandMapperDeps = { parseRunFlowFile(filePath: string, context: MaestroParseContext): MaestroReplayFlow; }; - -export type PermissionCommand = 'grant' | 'deny' | 'reset'; diff --git a/src/core/dispatch-context.ts b/src/core/dispatch-context.ts index 37387f42b..b6e5712ec 100644 --- a/src/core/dispatch-context.ts +++ b/src/core/dispatch-context.ts @@ -12,7 +12,10 @@ export type BatchStep = { export type CommandFlags = Omit & { batchSteps?: BatchStep[]; + launchArgs?: string[]; replayBackend?: string; + allowNonHittableSelectorTap?: boolean; + maestroOptional?: boolean; }; export type DispatchContext = ScreenshotDispatchFlags & { @@ -20,6 +23,7 @@ export type DispatchContext = ScreenshotDispatchFlags & { appBundleId?: string; activity?: string; launchConsole?: string; + launchArgs?: string[]; verbose?: boolean; logPath?: string; traceLogPath?: string; @@ -40,9 +44,11 @@ export type DispatchContext = ScreenshotDispatchFlags & { pauseMs?: number; pattern?: 'one-way' | 'ping-pong'; surface?: SessionSurface; + allowNonHittableSelectorTap?: boolean; directElementSelector?: { key: 'id' | 'label' | 'text' | 'value'; value: string; raw: string; + allowNonHittableTap?: boolean; }; }; diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index f28a3c6e8..b5b84b3ec 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -217,6 +217,7 @@ async function handleOpenCommand( await interactor.open(app, { activity: context?.activity, appBundleId: context?.appBundleId, + launchArgs: context?.launchArgs, url, }); return { app, url, ...successText(`Opened: ${app}`) }; @@ -228,6 +229,7 @@ async function handleOpenCommand( activity: context?.activity, appBundleId: context?.appBundleId, launchConsole, + launchArgs: context?.launchArgs, }); return { app, ...(launchConsole ? { launchConsole } : {}), ...successText(`Opened: ${app}`) }; } diff --git a/src/core/interactor-types.ts b/src/core/interactor-types.ts index 01653a5e9..c87d19ab8 100644 --- a/src/core/interactor-types.ts +++ b/src/core/interactor-types.ts @@ -29,6 +29,7 @@ export type ScreenshotOptions = { export type ElementSelectorTapOptions = { key: 'id' | 'label' | 'text' | 'value'; value: string; + allowNonHittableTap?: boolean; }; export type SnapshotOptions = BaseSnapshotOptions & { @@ -44,7 +45,13 @@ export type SnapshotResult = Omit & export type Interactor = { open( app: string, - options?: { activity?: string; appBundleId?: string; launchConsole?: string; url?: string }, + options?: { + activity?: string; + appBundleId?: string; + launchConsole?: string; + launchArgs?: string[]; + url?: string; + }, ): Promise; openDevice(): Promise; close(app: string): Promise; diff --git a/src/core/interactors/apple.ts b/src/core/interactors/apple.ts index a841b8c99..744adcb01 100644 --- a/src/core/interactors/apple.ts +++ b/src/core/interactors/apple.ts @@ -30,6 +30,7 @@ export function createAppleInteractor( openIosApp(device, app, { appBundleId: options?.appBundleId, launchConsole: options?.launchConsole, + launchArgs: options?.launchArgs, url: options?.url, }), openDevice: () => openIosDevice(device), diff --git a/src/daemon/__tests__/context.test.ts b/src/daemon/__tests__/context.test.ts index 090314199..9909e83de 100644 --- a/src/daemon/__tests__/context.test.ts +++ b/src/daemon/__tests__/context.test.ts @@ -14,6 +14,12 @@ test('contextFromFlags forwards scroll pixels from CLI flags', () => { assert.equal(context.pixels, 240); }); +test('contextFromFlags forwards internal non-hittable selector tap flag', () => { + const flags: CommandFlags = { allowNonHittableSelectorTap: true }; + const context = contextFromFlags('/tmp/agent-device.log', flags); + assert.equal(context.allowNonHittableSelectorTap, true); +}); + test('contextFromFlags forwards screenshot flags from CLI flags', () => { const flags: CommandFlags = { screenshotFullscreen: true, diff --git a/src/daemon/context.ts b/src/daemon/context.ts index 05c0864d4..4e858d529 100644 --- a/src/daemon/context.ts +++ b/src/daemon/context.ts @@ -21,6 +21,7 @@ export function contextFromFlags( appBundleId, activity: flags?.activity, launchConsole: flags?.launchConsole, + launchArgs: flags?.launchArgs, verbose: flags?.verbose, logPath, traceLogPath, @@ -41,5 +42,6 @@ export function contextFromFlags( backMode: flags?.backMode, pauseMs: flags?.pauseMs, pattern: flags?.pattern, + allowNonHittableSelectorTap: flags?.allowNonHittableSelectorTap, }; } diff --git a/src/daemon/direct-ios-selector.ts b/src/daemon/direct-ios-selector.ts index 5063f6632..aa0e82058 100644 --- a/src/daemon/direct-ios-selector.ts +++ b/src/daemon/direct-ios-selector.ts @@ -6,6 +6,7 @@ export type DirectIosSelectorTarget = { key: 'id' | 'label' | 'text' | 'value'; value: string; raw: string; + allowNonHittableTap?: boolean; }; export function readSimpleIosSelectorTarget(params: { diff --git a/src/daemon/handlers/__tests__/interaction.test.ts b/src/daemon/handlers/__tests__/interaction.test.ts index 60e4eb7d9..116fb6f0a 100644 --- a/src/daemon/handlers/__tests__/interaction.test.ts +++ b/src/daemon/handlers/__tests__/interaction.test.ts @@ -410,6 +410,43 @@ test('click simple iOS id selector uses direct runner selector tap without snaps } }); +test('click simple iOS selector forwards Maestro non-hittable tap backdoor', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'ios-maestro-selector-fallback'; + sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); + + mockDispatch.mockResolvedValue({ + message: 'tapped via non-hittable coordinate fallback', + x: 439.5, + y: 101.5, + referenceWidth: 440, + referenceHeight: 956, + }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'click', + positionals: ['id="e2eSignInAlice"'], + flags: { allowNonHittableSelectorTap: true }, + }, + sessionName, + sessionStore, + contextFromFlags, + }); + + expect(response?.ok).toBe(true); + const pressCalls = mockDispatch.mock.calls.filter((call) => call[1] === 'press'); + expect(pressCalls.length).toBe(1); + expect((pressCalls[0]?.[4] as Record)?.directElementSelector).toEqual({ + key: 'id', + value: 'e2eSignInAlice', + raw: 'id="e2eSignInAlice"', + allowNonHittableTap: true, + }); +}); + test('click simple iOS id selector falls back to snapshot coordinates when direct tap fails', async () => { const sessionStore = makeSessionStore(); const sessionName = 'ios-direct-selector-fallback'; diff --git a/src/daemon/handlers/__tests__/session-replay-vars.test.ts b/src/daemon/handlers/__tests__/session-replay-vars.test.ts index db01e2b9c..0803a39fb 100644 --- a/src/daemon/handlers/__tests__/session-replay-vars.test.ts +++ b/src/daemon/handlers/__tests__/session-replay-vars.test.ts @@ -441,6 +441,266 @@ test('runReplayScriptFile applies CLI env overrides before Maestro compat mappin assert.deepEqual(calls[1]?.positionals, ['id="shell-button"']); }); +test('runReplayScriptFile retries Maestro scrollUntilVisible with scroll probes', async () => { + const calls: CapturedInvocation[] = []; + let waitAttempts = 0; + const { response } = await runReplayFixture({ + label: 'maestro-scroll-until-visible', + script: [ + 'appId: demo.app', + '---', + '- scrollUntilVisible:', + ' element: Discover', + ' direction: UP', + ' timeout: 1200', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'scroll') return { ok: true, data: {} }; + if (req.command === 'find') { + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'find wait timed out' }, + }; + } + waitAttempts += 1; + if (waitAttempts === 3) return { ok: true, data: { waitedMs: 1100 } }; + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'wait timed out' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['wait', ['label="Discover" || text="Discover" || id="Discover"', '500']], + ['find', ['Discover', 'wait', '500']], + ['scroll', ['down']], + ['wait', ['label="Discover" || text="Discover" || id="Discover"', '500']], + ['find', ['Discover', 'wait', '500']], + ['scroll', ['down']], + ['wait', ['label="Discover" || text="Discover" || id="Discover"', '200']], + ], + ); +}); + +test('runReplayScriptFile lets Maestro scrollUntilVisible use fuzzy visible text matching', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-scroll-until-visible-fuzzy-text', + script: ['appId: demo.app', '---', '- scrollUntilVisible:', ' element: Discover', ''].join( + '\n', + ), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'find') return { ok: true, data: { found: true } }; + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'wait timed out' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['wait', ['label="Discover" || text="Discover" || id="Discover"', '500']], + ['find', ['Discover', 'wait', '500']], + ], + ); +}); + +test('runReplayScriptFile lets Maestro tapOn use fuzzy visible text matching', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-fuzzy', + script: ['appId: demo.app', '---', '- tapOn: Discover', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'find') return { ok: true, data: { found: true } }; + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'Selector did not match' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['find', ['Discover', 'click']]], + ); +}); + +test('runReplayScriptFile resolves Maestro percentage point taps from snapshot size', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-point-percent', + script: ['appId: demo.app', '---', '- tapOn:', ' point: 20%,20%', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 0, + type: 'application', + rect: { x: 0, y: 0, width: 1000, height: 2000 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['200', '400']], + ], + ); + assert.equal(calls[0]?.flags?.noRecord, true); +}); + +test('runReplayScriptFile retries Maestro tapOn until the selector appears', async () => { + const calls: CapturedInvocation[] = []; + let clickAttempts = 0; + const { response } = await runReplayFixture({ + label: 'maestro-tap-on-retry', + script: ['appId: demo.app', '---', '- tapOn:', ' id: delayedButton', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + clickAttempts += 1; + if (clickAttempts === 3) return { ok: true, data: {} }; + return { + ok: false, + error: { code: 'ELEMENT_NOT_FOUND', message: 'element not found' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['click', ['id="delayedButton"']], + ['click', ['id="delayedButton"']], + ['click', ['id="delayedButton"']], + ], + ); +}); + +test('runReplayScriptFile recovers Maestro enter submit after iOS runner transport reset', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-press-enter-recover', + script: ['appId: demo.app', '---', '- pressKey: Enter', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') return { ok: true, data: {} }; + return { + ok: false, + error: { code: 'UNKNOWN', message: 'fetch failed' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['type', ['\n']], + ['snapshot', []], + ], + ); +}); + +test('runReplayScriptFile skips Maestro runFlow.when.visible commands when absent', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-run-flow-when-visible-skip', + script: [ + 'appId: demo.app', + '---', + '- runFlow:', + ' when:', + ' visible: Continue', + ' commands:', + ' - tapOn: Continue', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'not visible' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['is', ['visible', 'label="Continue" || text="Continue" || id="Continue"']]], + ); +}); + +test('runReplayScriptFile runs Maestro runFlow.when.visible commands when present', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-run-flow-when-visible-run', + script: [ + 'appId: demo.app', + '---', + '- runFlow:', + ' when:', + ' visible: Continue', + ' commands:', + ' - tapOn: Continue', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'is') return { ok: true, data: { pass: true } }; + if (req.command === 'click') { + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'Selector did not match' }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['is', ['visible', 'label="Continue" || text="Continue" || id="Continue"']], + ['find', ['Continue', 'click']], + ], + ); +}); + test('runReplayScriptFile reads shell env from request (client-collected), not daemon process.env', async () => { // Ensure the daemon's own process.env does NOT contain AD_VAR_APP. assert.equal(process.env.AD_VAR_APP, undefined); diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/handlers/interaction-touch.ts index 9c46354ff..abeab8f6d 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/handlers/interaction-touch.ts @@ -257,7 +257,12 @@ function readDirectIosSelectorTapTarget(params: { if (commandLabel !== 'click') return null; if (target.kind !== 'selector') return null; if (hasNonDefaultClickOptions(flags)) return null; - return readSimpleIosSelectorTarget({ session, selectorExpression: target.selector }); + const selector = readSimpleIosSelectorTarget({ session, selectorExpression: target.selector }); + if (!selector) return null; + return { + ...selector, + ...(flags?.allowNonHittableSelectorTap ? { allowNonHittableTap: true } : {}), + }; } function hasNonDefaultClickOptions(flags: CommandFlags | undefined): boolean { diff --git a/src/daemon/handlers/session-replay-maestro-runtime.ts b/src/daemon/handlers/session-replay-maestro-runtime.ts new file mode 100644 index 000000000..93a41ae67 --- /dev/null +++ b/src/daemon/handlers/session-replay-maestro-runtime.ts @@ -0,0 +1,318 @@ +import { type CommandFlags } from '../../core/dispatch.ts'; +import { MAESTRO_RUNTIME_COMMAND } from '../../compat/maestro/runtime-commands.ts'; +import type { SnapshotState } from '../../utils/snapshot.ts'; +import { sleep } from '../../utils/timeouts.ts'; +import { parseSelectorChain } from '../selectors.ts'; +import { getSnapshotReferenceFrame } from '../touch-reference-frame.ts'; +import type { DaemonRequest, DaemonResponse, SessionAction } from '../types.ts'; +import { errorResponse } from './response.ts'; + +const MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS = 500; +const MAESTRO_TAP_ON_TIMEOUT_MS = 30000; +const MAESTRO_TAP_ON_RETRY_MS = 250; + +type ReplayBaseRequest = Omit; + +type MaestroReplayInvoker = (params: { + action: SessionAction; + line: number; + step: number; +}) => Promise; + +export async function invokeMaestroRuntimeCommand(params: { + command: string; + baseReq: ReplayBaseRequest; + positionals: string[]; + batchSteps: CommandFlags['batchSteps'] | undefined; + line: number; + step: number; + invoke: (req: DaemonRequest) => Promise; + invokeReplayAction: MaestroReplayInvoker; +}): Promise { + switch (params.command) { + case MAESTRO_RUNTIME_COMMAND.scrollUntilVisible: + return await invokeMaestroScrollUntilVisible(params); + case MAESTRO_RUNTIME_COMMAND.tapOn: + return await invokeMaestroTapOn(params); + case MAESTRO_RUNTIME_COMMAND.tapPointPercent: + return await invokeMaestroTapPointPercent(params); + case MAESTRO_RUNTIME_COMMAND.runFlowWhen: + return await invokeMaestroRunFlowWhen(params); + case MAESTRO_RUNTIME_COMMAND.pressEnter: + return await invokeMaestroPressEnter(params); + default: + return undefined; + } +} + +async function invokeMaestroScrollUntilVisible(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: (req: DaemonRequest) => Promise; +}): Promise { + const [selector, timeoutValue = '5000', direction = 'down'] = params.positionals; + if (!selector) { + return errorResponse('INVALID_ARGS', 'scrollUntilVisible requires a selector.'); + } + const timeoutMs = Number(timeoutValue); + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + return errorResponse('INVALID_ARGS', 'scrollUntilVisible timeout must be a positive number.'); + } + const fuzzyTextQuery = extractMaestroVisibleTextQuery(selector); + const attempts = Math.max(1, Math.ceil(timeoutMs / MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS)); + let lastWaitResponse: DaemonResponse | undefined; + + for (let index = 0; index < attempts; index += 1) { + const probeMs = Math.min( + MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS, + Math.max(1, timeoutMs - index * MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS), + ); + const waitResponse = await params.invoke({ + ...params.baseReq, + command: 'wait', + positionals: [selector, String(probeMs)], + }); + if (waitResponse.ok) return waitResponse; + lastWaitResponse = waitResponse; + + const fuzzyResponse = fuzzyTextQuery + ? await params.invoke({ + ...params.baseReq, + command: 'find', + positionals: [fuzzyTextQuery, 'wait', String(probeMs)], + }) + : undefined; + if (fuzzyResponse?.ok) return fuzzyResponse; + lastWaitResponse = fuzzyResponse ?? lastWaitResponse; + + if (index === attempts - 1) break; + + const scrollResponse = await params.invoke({ + ...params.baseReq, + command: 'scroll', + positionals: [direction], + }); + if (!scrollResponse.ok) return scrollResponse; + } + + return withMaestroScrollTimeoutContext(lastWaitResponse, selector, timeoutMs); +} + +async function invokeMaestroTapPointPercent(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: (req: DaemonRequest) => Promise; +}): Promise { + const [xValue, yValue] = params.positionals; + const xPercent = Number(xValue); + const yPercent = Number(yValue); + if (!Number.isFinite(xPercent) || !Number.isFinite(yPercent)) { + return errorResponse('INVALID_ARGS', 'tapOn percentage point requires numeric x/y values.'); + } + + const snapshotResponse = await params.invoke({ + ...params.baseReq, + command: 'snapshot', + positionals: [], + flags: { + ...params.baseReq.flags, + noRecord: true, + snapshotRaw: true, + snapshotForceFull: true, + }, + }); + if (!snapshotResponse.ok) return snapshotResponse; + + const snapshot = readSnapshotState(snapshotResponse.data); + if (!snapshot) { + return errorResponse( + 'COMMAND_FAILED', + 'Unable to read snapshot data for Maestro percentage point tap.', + ); + } + + const frame = getSnapshotReferenceFrame(snapshot); + if (!frame) { + return errorResponse( + 'COMMAND_FAILED', + 'Unable to resolve screen size for Maestro percentage point tap.', + ); + } + + return await params.invoke({ + ...params.baseReq, + command: 'click', + positionals: [ + String(Math.round((frame.referenceWidth * xPercent) / 100)), + String(Math.round((frame.referenceHeight * yPercent) / 100)), + ], + }); +} + +function readSnapshotState(data: unknown): SnapshotState | undefined { + if ( + typeof data === 'object' && + data !== null && + Array.isArray((data as { nodes?: unknown }).nodes) + ) { + return data as SnapshotState; + } + return undefined; +} + +async function invokeMaestroTapOn(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: (req: DaemonRequest) => Promise; +}): Promise { + const [selector] = params.positionals; + if (!selector) { + return errorResponse('INVALID_ARGS', 'tapOn requires a selector.'); + } + const startedAt = Date.now(); + const fuzzyTextQuery = extractMaestroVisibleTextQuery(selector); + let lastResponse: DaemonResponse | undefined; + while (Date.now() - startedAt < MAESTRO_TAP_ON_TIMEOUT_MS) { + if (fuzzyTextQuery) { + const findResponse = await params.invoke({ + ...params.baseReq, + command: 'find', + positionals: [fuzzyTextQuery, 'click'], + }); + if (findResponse.ok) return findResponse; + lastResponse = findResponse; + } + + const clickResponse = await params.invoke({ + ...params.baseReq, + command: 'click', + positionals: [selector], + }); + if (clickResponse.ok) return clickResponse; + lastResponse = clickResponse; + await sleep(MAESTRO_TAP_ON_RETRY_MS); + } + + if (params.baseReq.flags?.maestroOptional === true) { + return { ok: true, data: { skipped: true, optional: true, selector } }; + } + return ( + lastResponse ?? errorResponse('COMMAND_FAILED', `tapOn timed out for selector: ${selector}`) + ); +} + +async function invokeMaestroRunFlowWhen(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + batchSteps: CommandFlags['batchSteps'] | undefined; + line: number; + step: number; + invoke: (req: DaemonRequest) => Promise; + invokeReplayAction: MaestroReplayInvoker; +}): Promise { + const [mode, selector] = params.positionals; + if ((mode !== 'visible' && mode !== 'notVisible') || !selector) { + return errorResponse( + 'INVALID_ARGS', + 'runFlow.when requires visible/notVisible and a selector.', + ); + } + const predicate = mode === 'visible' ? 'visible' : 'hidden'; + const conditionResponse = await params.invoke({ + ...params.baseReq, + command: 'is', + positionals: [predicate, selector], + flags: { ...params.baseReq.flags, noRecord: true }, + }); + if (!conditionResponse.ok) { + return { ok: true, data: { skipped: true, condition: mode, selector } }; + } + + const steps = (params.batchSteps ?? []).map(batchStepToSessionAction); + for (const [index, action] of steps.entries()) { + const response = await params.invokeReplayAction({ + action, + line: params.line, + step: params.step + index / 1000, + }); + if (!response.ok) return response; + } + + return { ok: true, data: { ran: steps.length, condition: mode, selector } }; +} + +async function invokeMaestroPressEnter(params: { + baseReq: ReplayBaseRequest; + invoke: (req: DaemonRequest) => Promise; +}): Promise { + const response = await params.invoke({ + ...params.baseReq, + command: 'type', + positionals: ['\n'], + }); + if (response.ok) return response; + const message = response.error.message.toLowerCase(); + if (!message.includes('fetch failed')) return response; + + const snapshotResponse = await params.invoke({ + ...params.baseReq, + command: 'snapshot', + positionals: [], + flags: { ...params.baseReq.flags, noRecord: true }, + }); + if (!snapshotResponse.ok) return response; + return { + ok: true, + data: { + recovered: true, + warning: 'Enter key submit reset the iOS runner transport; recovered after snapshot.', + }, + }; +} + +function batchStepToSessionAction( + step: NonNullable[number], +): SessionAction { + const action: SessionAction = { + ts: Date.now(), + command: step.command, + positionals: step.positionals ?? [], + flags: step.flags ?? {}, + }; + if (step.runtime && typeof step.runtime === 'object') { + action.runtime = step.runtime as SessionAction['runtime']; + } + return action; +} + +function extractMaestroVisibleTextQuery(selectorExpression: string): string | null { + const chain = parseSelectorChain(selectorExpression); + const terms = chain.selectors.flatMap((selector) => selector.terms); + if (terms.length === 0) return null; + if (!terms.some((term) => term.key === 'label' || term.key === 'text')) return null; + if (!terms.every((term) => ['label', 'text', 'id'].includes(term.key))) return null; + const values = terms.map((term) => (typeof term.value === 'string' ? term.value : '')); + const first = values[0]; + if (!first || !values.every((value) => value === first)) return null; + return first; +} + +function withMaestroScrollTimeoutContext( + response: DaemonResponse | undefined, + selector: string, + timeoutMs: number, +): DaemonResponse { + if (!response || response.ok) { + return errorResponse( + 'COMMAND_FAILED', + `scrollUntilVisible timed out after ${timeoutMs}ms for selector: ${selector}`, + ); + } + return { + ok: false, + error: { + ...response.error, + message: `scrollUntilVisible timed out after ${timeoutMs}ms for selector: ${selector}. Last wait: ${response.error.message}`, + }, + }; +} diff --git a/src/daemon/handlers/session-replay-runtime.ts b/src/daemon/handlers/session-replay-runtime.ts index 12ff2cee0..b2774457d 100644 --- a/src/daemon/handlers/session-replay-runtime.ts +++ b/src/daemon/handlers/session-replay-runtime.ts @@ -10,6 +10,7 @@ import { healReplayAction } from './session-replay-heal.ts'; import { formatScriptActionSummary } from '../../replay/script-utils.ts'; import { mergeParentFlags } from './handler-utils.ts'; import { errorResponse } from './response.ts'; +import { invokeMaestroRuntimeCommand } from './session-replay-maestro-runtime.ts'; import { buildReplayVarScope, collectReplayShellEnv, @@ -180,15 +181,41 @@ async function invokeReplayAction(params: { command: resolved.command, positionals: resolved.positionals ?? [], }); - const response = await invoke({ + const flags = buildReplayActionFlags(req.flags, resolved.flags); + const baseReq = { token: req.token, session: sessionName, - command: resolved.command, - positionals: resolved.positionals ?? [], - flags: buildReplayActionFlags(req.flags, resolved.flags), + flags, runtime: resolved.runtime, meta: req.meta, - }); + }; + const response = + (await invokeMaestroRuntimeCommand({ + command: resolved.command, + baseReq, + positionals: resolved.positionals ?? [], + batchSteps: resolved.flags?.batchSteps, + line, + step, + invoke, + invokeReplayAction: async (nested) => + await invokeReplayAction({ + req, + sessionName, + action: nested.action, + scope, + filePath, + line: nested.line, + step: nested.step, + tracePath, + invoke, + }), + })) ?? + (await invoke({ + ...baseReq, + command: resolved.command, + positionals: resolved.positionals ?? [], + })); const finishedAt = Date.now(); appendReplayTraceEvent(tracePath, { type: 'replay_action_stop', diff --git a/src/daemon/types.ts b/src/daemon/types.ts index d5f3720bd..aa5df7f59 100644 --- a/src/daemon/types.ts +++ b/src/daemon/types.ts @@ -228,6 +228,7 @@ export type SessionAction = { snapshotDepth?: number; snapshotScope?: string; snapshotRaw?: boolean; + launchArgs?: string[]; saveScript?: boolean | string; noRecord?: boolean; }; diff --git a/src/platforms/ios/apps.ts b/src/platforms/ios/apps.ts index 70bd66553..5eccf112c 100644 --- a/src/platforms/ios/apps.ts +++ b/src/platforms/ios/apps.ts @@ -127,7 +127,7 @@ export async function resolveIosApp(device: DeviceInfo, app: string): Promise { const launchConsole = options?.launchConsole?.trim(); if (launchConsole && (device.platform !== 'ios' || device.kind !== 'simulator')) { @@ -185,7 +185,10 @@ export async function openIosApp( const bundleId = options?.appBundleId ?? (await resolveIosApp(device, app)); if (device.kind === 'simulator') { - await launchIosSimulatorApp(device, bundleId, launchConsole ? { launchConsole } : undefined); + await launchIosSimulatorApp(device, bundleId, { + ...(launchConsole ? { launchConsole } : {}), + ...(options?.launchArgs ? { launchArgs: options.launchArgs } : {}), + }); return; } @@ -884,7 +887,7 @@ function isIosBiometricCapabilityMissing(stdout: string, stderr: string): boolea async function launchIosSimulatorApp( device: DeviceInfo, bundleId: string, - options?: { launchConsole?: string }, + options?: { launchConsole?: string; launchArgs?: string[] }, ): Promise { await ensureBootedSimulator(device); @@ -947,11 +950,12 @@ async function launchIosSimulatorApp( function buildIosSimulatorLaunchArgs( deviceId: string, bundleId: string, - options?: { launchConsole?: string }, + options?: { launchConsole?: string; launchArgs?: string[] }, ): string[] { const args = ['launch']; if (options?.launchConsole) args.push('--console-pty'); args.push(deviceId, bundleId); + if (options?.launchArgs) args.push(...options.launchArgs); return args; } diff --git a/src/platforms/ios/interactions.ts b/src/platforms/ios/interactions.ts index 50f91f0df..eadbd1798 100644 --- a/src/platforms/ios/interactions.ts +++ b/src/platforms/ios/interactions.ts @@ -81,6 +81,7 @@ export function iosRunnerOverrides( command: 'tap', selectorKey: selector.key, selectorValue: selector.value, + allowNonHittableSelectorTap: selector.allowNonHittableTap, appBundleId: ctx.appBundleId, }, runnerOpts, @@ -159,7 +160,7 @@ export function iosRunnerOverrides( command: 'type', text, delayMs, - textEntryMode: 'append', + textEntryMode: text === '\n' ? undefined : 'append', appBundleId: ctx.appBundleId, }, runnerOpts, diff --git a/src/platforms/ios/runner-contract.ts b/src/platforms/ios/runner-contract.ts index 598ed94f2..e87b6ccf9 100644 --- a/src/platforms/ios/runner-contract.ts +++ b/src/platforms/ios/runner-contract.ts @@ -44,6 +44,7 @@ export type RunnerCommand = { text?: string; selectorKey?: 'id' | 'label' | 'text' | 'value'; selectorValue?: string; + allowNonHittableSelectorTap?: boolean; delayMs?: number; textEntryMode?: 'append' | 'replace'; action?: 'get' | 'accept' | 'dismiss'; diff --git a/src/utils/__tests__/args.test.ts b/src/utils/__tests__/args.test.ts index 742e5f8dd..baab1af9d 100644 --- a/src/utils/__tests__/args.test.ts +++ b/src/utils/__tests__/args.test.ts @@ -917,10 +917,9 @@ test('usageForCommand includes Maestro replay flag', () => { assert.match(help, /--maestro/); assert.match(help, /doubleTapOn/); assert.match(help, /pasteText/); - assert.match(help, /setPermissions/); - assert.match(help, /startRecording\/stopRecording/); assert.match(help, /runFlow file\/inline/); assert.match(help, /repeat\.times/); + assert.match(help, /stopApp/); assert.match(help, /Unsupported syntax fails loudly/); assert.match(help, /issues\/558/); }); diff --git a/src/utils/__tests__/video.test.ts b/src/utils/__tests__/video.test.ts index 4e56c6b9e..cc1dc6b1c 100644 --- a/src/utils/__tests__/video.test.ts +++ b/src/utils/__tests__/video.test.ts @@ -3,6 +3,8 @@ import assert from 'node:assert/strict'; import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; +import { AppError } from '../errors.ts'; +import { withCommandExecutorOverride } from '../exec.ts'; import { isPlayableVideo } from '../video.ts'; function makeAtom(type: string, payload = Buffer.alloc(0)): Buffer { @@ -23,7 +25,7 @@ test('isPlayableVideo falls back to MP4 container validation when swift is unava process.env.AGENT_DEVICE_SWIFT_CACHE_DIR = path.join(tmpDir, 'swift-cache'); try { - assert.equal(await isPlayableVideo(videoPath), true); + assert.equal(await withUnavailableSwift(() => isPlayableVideo(videoPath)), true); } finally { process.env.PATH = previousPath; restoreEnv('AGENT_DEVICE_SWIFT_CACHE_DIR', previousSwiftCacheDir); @@ -42,7 +44,7 @@ test('isPlayableVideo fallback rejects files without playable MP4 atoms', async process.env.AGENT_DEVICE_SWIFT_CACHE_DIR = path.join(tmpDir, 'swift-cache'); try { - assert.equal(await isPlayableVideo(videoPath), false); + assert.equal(await withUnavailableSwift(() => isPlayableVideo(videoPath)), false); } finally { process.env.PATH = previousPath; restoreEnv('AGENT_DEVICE_SWIFT_CACHE_DIR', previousSwiftCacheDir); @@ -57,3 +59,9 @@ function restoreEnv(name: string, value: string | undefined): void { } process.env[name] = value; } + +async function withUnavailableSwift(fn: () => Promise): Promise { + return await withCommandExecutorOverride(() => { + throw new AppError('TOOL_MISSING', 'swift unavailable for test'); + }, fn); +} diff --git a/src/utils/command-schema.ts b/src/utils/command-schema.ts index 28710d73c..ab5f29997 100644 --- a/src/utils/command-schema.ts +++ b/src/utils/command-schema.ts @@ -1261,7 +1261,7 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ type: 'boolean', usageLabel: '--maestro', usageDescription: - 'Replay: treat input as a Maestro YAML compatibility flow. Supported subset: launchApp without state-reset side effects, runFlow file/inline with when.platform, onFlowStart/onFlowComplete, deterministic repeat.times, tapOn, doubleTapOn, longPressOn, inputText, pasteText, openLink, assertVisible, assertNotVisible, assertTrue literal true/false, extendedWaitUntil, scroll, absolute/percentage swipe, takeScreenshot, hideKeyboard, pressKey back/enter/home, back, waitForAnimationToEnd, stopApp/killApp, setAirplaneMode, setLocation, setOrientation, supported setPermissions targets, and startRecording/stopRecording. ' + + 'Replay: treat input as a Maestro YAML compatibility flow. Supported subset: launchApp with launch arguments but without state-reset side effects, runFlow file/inline with when.platform/visible/notVisible, runScript file/env with http.post/json/output variables, onFlowStart/onFlowComplete, deterministic repeat.times, tapOn including optional and absolute/percentage point taps, doubleTapOn, longPressOn, inputText, pasteText, openLink, assertVisible, assertNotVisible, extendedWaitUntil, scroll, scrollUntilVisible, absolute/percentage swipe, takeScreenshot, hideKeyboard, pressKey back/enter/home, back, waitForAnimationToEnd, and stopApp. ' + 'Unsupported syntax fails loudly with a link to https://github.com/callstackincubator/agent-device/issues/558', }, { diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index 675266000..24b264501 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -61,11 +61,11 @@ Maestro compatibility translates supported YAML commands into Agent Device repla - Supported and unsupported capabilities: https://github.com/callstackincubator/agent-device/issues/558 - New focused compatibility request: https://github.com/callstackincubator/agent-device/issues/new -Currently supported areas include app launch without state-reset side effects, file and inline `runFlow` with `when.platform`, `onFlowStart` / `onFlowComplete`, deterministic `repeat.times`, `tapOn`, `doubleTapOn`, `longPressOn`, `inputText`, `pasteText`, `openLink`, visibility assertions, literal `assertTrue`, `extendedWaitUntil`, `scroll`, absolute/percentage `swipe`, screenshots, keyboard dismiss, basic `pressKey`, `back`, animation waits, `stopApp` / `killApp`, airplane mode, mock location, orientation, supported permission targets, and screen recording. +Currently supported areas include app launch with launch arguments but without state-reset side effects, file and inline `runFlow` with `when.platform`, `when.visible`, and `when.notVisible`, `onFlowStart` / `onFlowComplete`, deterministic `repeat.times`, `tapOn` including `optional` and absolute/percentage point taps, `doubleTapOn`, `longPressOn`, `inputText`, `pasteText`, `openLink`, visibility assertions, `extendedWaitUntil`, `scroll`, `scrollUntilVisible`, absolute/percentage `swipe`, screenshots, keyboard dismiss, basic `pressKey`, `back`, animation waits, and `stopApp`. `runScript` is supported only as a Maestro compatibility feature for file/env scripts that use `http.post`, `json`, and `output` variables; it is not a native `.ad` command. Maestro `env` values use the same replay precedence as `.ad` files: flow `env` is the default, shell `AD_VAR_*` values override it, and CLI `-e KEY=VALUE` wins over both. -Runtime-dependent Maestro features such as `scrollUntilVisible`, `repeat.while`, `runFlow.when.visible`, `runScript`, `evalScript`, text clearing, and app state reset are tracked separately because they require neutral Agent Device runtime or device capabilities before they can be mapped safely. +Unsupported Maestro features such as `repeat.while`, `runFlow.when.true`, full expression predicates, `evalScript`, text clearing, selector relations such as `index` / `childOf`, device utility commands, and app state reset are tracked separately because they require neutral Agent Device runtime or device capabilities before they can be mapped safely. ## Run a lightweight `.ad` suite From d1875b3b2ec3ad820e70f6a3398c7219cb4a8f52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 22 May 2026 09:50:25 +0200 Subject: [PATCH 2/6] fix: harden Maestro replay compatibility --- .../RunnerTests+CommandExecution.swift | 1 + .../RunnerTests+Interaction.swift | 29 +++ .../maestro/__tests__/replay-flow.test.ts | 1 + src/compat/maestro/device-actions.ts | 4 +- src/compat/maestro/interactions.ts | 22 +- src/compat/maestro/support.ts | 22 -- src/core/dispatch-context.ts | 2 + src/core/dispatch.ts | 17 +- src/daemon/__tests__/context.test.ts | 6 + src/daemon/context.ts | 4 + src/daemon/handlers/__tests__/find.test.ts | 36 ++++ .../__tests__/session-replay-vars.test.ts | 65 ++++++ src/daemon/handlers/find.ts | 204 +++++++++++++----- .../session-replay-maestro-runtime.ts | 122 +++++++---- src/platforms/ios/apps.ts | 47 ++++ src/utils/command-schema.ts | 2 +- 16 files changed, 455 insertions(+), 129 deletions(-) diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index b71f570cb..c7dd5cf96 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -282,6 +282,7 @@ extension RunnerTests { if let response = unsupportedResponse(for: outcome) { return response } + waitForTextEntryReadinessAfterTap(app: activeApp, element: element) return Response( ok: true, data: DataPayload( diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 138117c8d..707f69386 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -809,6 +809,35 @@ extension RunnerTests { #endif } + func waitForTextEntryReadinessAfterTap(app: XCUIApplication, element: XCUIElement) { +#if os(iOS) + switch element.elementType { + case .textField, .secureTextField, .searchField, .textView: + if waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout) != nil { + return + } + let frame = element.frame + if !frame.isEmpty { + _ = tapAt(app: app, x: frame.midX, y: frame.midY) + _ = waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout) + } + default: + return + } +#endif + } + + private func waitForFocusedTextInput(app: XCUIApplication, timeout: TimeInterval) -> XCUIElement? { + let deadline = Date().addingTimeInterval(timeout) + while Date() < deadline { + if let focused = focusedTextInput(app: app) { + return focused + } + sleepFor(TextEntryTiming.pollInterval) + } + return focusedTextInput(app: app) + } + private func textEntryRefreshPoint(for element: XCUIElement?) -> CGPoint? { guard let element else { return nil diff --git a/src/compat/maestro/__tests__/replay-flow.test.ts b/src/compat/maestro/__tests__/replay-flow.test.ts index f50fb6c3f..3425baa77 100644 --- a/src/compat/maestro/__tests__/replay-flow.test.ts +++ b/src/compat/maestro/__tests__/replay-flow.test.ts @@ -345,6 +345,7 @@ test('parseMaestroReplayFlow accepts launchApp reset options without state-reset 'open', ['com.callstack.agentdevicelab'], { + maestroClearState: true, relaunch: true, launchArgs: ['-EXDevMenuIsOnboardingFinished', 'true', '-Example', 'ignored'], }, diff --git a/src/compat/maestro/device-actions.ts b/src/compat/maestro/device-actions.ts index 8069b2fab..8dcaa62b0 100644 --- a/src/compat/maestro/device-actions.ts +++ b/src/compat/maestro/device-actions.ts @@ -37,9 +37,11 @@ export function convertLaunchApp( context, ); const launchArgs = readLaunchArgs(value, context); - const shouldRelaunch = value.stopApp === true || launchArgs.length > 0; + const shouldClearState = value.clearState === true; + const shouldRelaunch = value.stopApp === true || shouldClearState || launchArgs.length > 0; return action('open', [appId], { relaunch: shouldRelaunch, + ...(shouldClearState ? { maestroClearState: true } : {}), ...(launchArgs.length > 0 ? { launchArgs } : {}), }); } diff --git a/src/compat/maestro/interactions.ts b/src/compat/maestro/interactions.ts index 8fa5b1b13..81153af5e 100644 --- a/src/compat/maestro/interactions.ts +++ b/src/compat/maestro/interactions.ts @@ -251,15 +251,11 @@ function selectorTerm(key: string, value: string): string { function tapFlags(value: unknown): SessionAction['flags'] | undefined { if (!isPlainRecord(value)) return undefined; const flags: SessionAction['flags'] = {}; - if (typeof value.repeat === 'number' && Number.isInteger(value.repeat) && value.repeat > 1) { - flags.count = value.repeat; - } - if (typeof value.delay === 'number' && Number.isInteger(value.delay) && value.delay >= 0) { - flags.intervalMs = value.delay; - } - if (value.optional === true) { - flags.maestroOptional = true; - } + const repeat = positiveInteger(value.repeat); + const delay = nonNegativeInteger(value.delay); + if (repeat && repeat > 1) flags.count = repeat; + if (delay !== undefined) flags.intervalMs = delay; + if (value.optional === true) flags.maestroOptional = true; return Object.keys(flags).length > 0 ? flags : undefined; } @@ -270,3 +266,11 @@ function doubleTapFlags(value: unknown): SessionAction['flags'] { } return flags; } + +function positiveInteger(value: unknown): number | undefined { + return typeof value === 'number' && Number.isInteger(value) && value > 0 ? value : undefined; +} + +function nonNegativeInteger(value: unknown): number | undefined { + return typeof value === 'number' && Number.isInteger(value) && value >= 0 ? value : undefined; +} diff --git a/src/compat/maestro/support.ts b/src/compat/maestro/support.ts index 591e23ecb..997a5c70d 100644 --- a/src/compat/maestro/support.ts +++ b/src/compat/maestro/support.ts @@ -63,24 +63,6 @@ export function normalizePlatformValue(value: unknown, name: string): 'android' return platform; } -export function normalizeToken(value: string): string { - return value - .trim() - .replace(/([a-z0-9])([A-Z])/g, '$1-$2') - .replace(/[\s_]+/g, '-') - .toLowerCase(); -} - -export function readBooleanLiteral(value: unknown, command: string): boolean { - if (typeof value === 'boolean') return value; - if (typeof value === 'string') { - const normalized = normalizeToken(value); - if (normalized === 'true') return true; - if (normalized === 'false') return false; - } - throw new AppError('INVALID_ARGS', `${command} expects a boolean value.`); -} - export function readEnvMap(value: unknown, name: string): Record { if (value === undefined || value === null) return {}; if (!isPlainRecord(value)) { @@ -118,10 +100,6 @@ export function resolveMaestroString(value: string, context: MaestroParseContext }); } -export function resolveMaybeMaestroString(value: unknown, context: MaestroParseContext): unknown { - return typeof value === 'string' ? resolveMaestroString(value, context) : value; -} - export function unsupportedCommand(command: string): never { throw unsupportedMaestroSyntax(`Maestro command "${command}" is not supported yet.`); } diff --git a/src/core/dispatch-context.ts b/src/core/dispatch-context.ts index b6e5712ec..9bf959b8b 100644 --- a/src/core/dispatch-context.ts +++ b/src/core/dispatch-context.ts @@ -16,6 +16,7 @@ export type CommandFlags = Omit & { replayBackend?: string; allowNonHittableSelectorTap?: boolean; maestroOptional?: boolean; + maestroClearState?: boolean; }; export type DispatchContext = ScreenshotDispatchFlags & { @@ -24,6 +25,7 @@ export type DispatchContext = ScreenshotDispatchFlags & { activity?: string; launchConsole?: string; launchArgs?: string[]; + maestroClearState?: boolean; verbose?: boolean; logPath?: string; traceLogPath?: string; diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index b5b84b3ec..f4eea78d1 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -10,7 +10,7 @@ import { pushAndroidNotification } from '../platforms/android/notifications.ts'; import { getInteractor } from './interactors.ts'; import type { Interactor, RunnerContext } from './interactor-types.ts'; import { runIosRunnerCommand } from '../platforms/ios/runner-client.ts'; -import { pushIosNotification } from '../platforms/ios/apps.ts'; +import { clearIosSimulatorAppState, pushIosNotification } from '../platforms/ios/apps.ts'; import { isDeepLinkTarget } from './open-target.ts'; import { parseTriggerAppEventArgs, resolveAppEventUrl } from './app-events.ts'; import { @@ -225,6 +225,21 @@ async function handleOpenCommand( if (launchConsole && isDeepLinkTarget(app)) { throw new AppError('INVALID_ARGS', LAUNCH_CONSOLE_DIRECT_APP_ONLY_MESSAGE); } + if (context?.maestroClearState) { + if (isDeepLinkTarget(app)) { + throw new AppError( + 'INVALID_ARGS', + 'Maestro launchApp.clearState requires an app target, not a deep link.', + ); + } + if (device.platform !== 'ios' || device.kind !== 'simulator') { + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'Maestro launchApp.clearState is currently supported only on iOS simulators.', + ); + } + await clearIosSimulatorAppState(device, app); + } await interactor.open(app, { activity: context?.activity, appBundleId: context?.appBundleId, diff --git a/src/daemon/__tests__/context.test.ts b/src/daemon/__tests__/context.test.ts index 9909e83de..bf117b083 100644 --- a/src/daemon/__tests__/context.test.ts +++ b/src/daemon/__tests__/context.test.ts @@ -20,6 +20,12 @@ test('contextFromFlags forwards internal non-hittable selector tap flag', () => assert.equal(context.allowNonHittableSelectorTap, true); }); +test('contextFromFlags forwards Maestro clearState launch compatibility flag', () => { + const flags: CommandFlags = { maestroClearState: true }; + const context = contextFromFlags('/tmp/agent-device.log', flags); + assert.equal(context.maestroClearState, true); +}); + test('contextFromFlags forwards screenshot flags from CLI flags', () => { const flags: CommandFlags = { screenshotFullscreen: true, diff --git a/src/daemon/context.ts b/src/daemon/context.ts index 4e858d529..a3fd128c7 100644 --- a/src/daemon/context.ts +++ b/src/daemon/context.ts @@ -8,6 +8,9 @@ import { getDiagnosticsMeta } from '../utils/diagnostics.ts'; export type DaemonCommandContext = DispatchContext & ScreenshotRuntimeFlags; +// Flat compatibility mapper: keeping each CLI flag visible here makes request +// context drift easier to spot than splitting the same optional fields apart. +// fallow-ignore-next-line complexity export function contextFromFlags( logPath: string, flags: CommandFlags | undefined, @@ -22,6 +25,7 @@ export function contextFromFlags( activity: flags?.activity, launchConsole: flags?.launchConsole, launchArgs: flags?.launchArgs, + maestroClearState: flags?.maestroClearState, verbose: flags?.verbose, logPath, traceLogPath, diff --git a/src/daemon/handlers/__tests__/find.test.ts b/src/daemon/handlers/__tests__/find.test.ts index 7efbca153..9b7d165ba 100644 --- a/src/daemon/handlers/__tests__/find.test.ts +++ b/src/daemon/handlers/__tests__/find.test.ts @@ -121,6 +121,42 @@ test('handleFindCommands click returns deterministic metadata across locator var } }); +test('handleFindCommands click prefers on-screen duplicate text matches', async () => { + const { response, invokeCalls } = await runFindClickScenario({ + positionals: ['Sign in', 'click'], + nodes: [ + { + index: 0, + ref: 'e1', + type: 'Application', + hittable: true, + rect: { x: 0, y: 0, width: 440, height: 956 }, + }, + { + index: 1, + ref: 'e2', + type: 'Button', + label: 'Sign in', + hittable: false, + rect: { x: -199, y: 186, width: 70, height: 33 }, + parentIndex: 0, + }, + { + index: 2, + ref: 'e3', + type: 'Button', + label: 'Sign in', + hittable: false, + rect: { x: 40, y: 870, width: 360, height: 44 }, + parentIndex: 0, + }, + ], + }); + + expect(response.ok).toBe(true); + expect(invokeCalls[0].positionals?.[0]).toBe('@e3'); +}); + test('handleFindCommands wait bypasses snapshot cache while Android freshness recovery is active', async () => { const sessionName = 'android-find-wait'; const session: SessionState = { diff --git a/src/daemon/handlers/__tests__/session-replay-vars.test.ts b/src/daemon/handlers/__tests__/session-replay-vars.test.ts index 0803a39fb..2da6f3151 100644 --- a/src/daemon/handlers/__tests__/session-replay-vars.test.ts +++ b/src/daemon/handlers/__tests__/session-replay-vars.test.ts @@ -540,6 +540,71 @@ test('runReplayScriptFile lets Maestro tapOn use fuzzy visible text matching', a ); }); +test('runReplayScriptFile retries Maestro fuzzy tapOn without raw selector fallback', async () => { + const calls: CapturedInvocation[] = []; + let findAttempts = 0; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-fuzzy-retry', + script: ['appId: demo.app', '---', '- tapOn: Discover', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'find') { + findAttempts += 1; + if (findAttempts === 2) return { ok: true, data: { found: true } }; + } + return { + ok: false, + error: { code: 'ELEMENT_NOT_FOUND', message: 'element not found' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['find', ['Discover', 'click']], + ['find', ['Discover', 'click']], + ], + ); +}); + +test('runReplayScriptFile lets optional Maestro fuzzy tapOn hit native alert labels', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-optional-native-label', + script: [ + 'appId: demo.app', + '---', + '- tapOn:', + ' text: Not Now', + ' optional: true', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'click' && req.positionals?.[0] === 'label="Not Now"') { + return { ok: true, data: { dismissed: true } }; + } + return { + ok: false, + error: { code: 'ELEMENT_NOT_FOUND', message: 'element not found' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['find', ['Not Now', 'click']], + ['click', ['label="Not Now"']], + ], + ); +}); + test('runReplayScriptFile resolves Maestro percentage point taps from snapshot size', async () => { const calls: CapturedInvocation[] = []; const { response } = await runReplayFixture({ diff --git a/src/daemon/handlers/find.ts b/src/daemon/handlers/find.ts index 42f2c9104..66363d85b 100644 --- a/src/daemon/handlers/find.ts +++ b/src/daemon/handlers/find.ts @@ -37,6 +37,10 @@ type ResolvedMatch = { actionFlags: Record; }; +type FindMatchResult = + | { ok: true; node: SnapshotState['nodes'][number] } + | { ok: false; response: DaemonResponse }; + export async function handleFindCommands(params: { req: DaemonRequest; sessionName: string; @@ -67,8 +71,7 @@ export async function handleFindCommands(params: { }); if (runtimeResponse) return runtimeResponse; const session = sessionStore.get(sessionName); - const isReadOnly = - action === 'exists' || action === 'wait' || action === 'get_text' || action === 'get_attrs'; + const isReadOnly = isReadOnlyFindAction(action); if (!session && !isReadOnly) { return errorResponse('SESSION_NOT_FOUND', 'No active session. Run open first.'); } @@ -76,9 +79,10 @@ export async function handleFindCommands(params: { if (!session) { await ensureDeviceReady(device); } - const scope = shouldScopeFind(locator) ? query : undefined; - const requiresRect = - action === 'click' || action === 'focus' || action === 'fill' || action === 'type'; + const requiresRect = findActionRequiresRect(action); + // Interaction targets need the full compact tree so duplicate labels can be + // resolved against viewport visibility before an off-screen subtree wins. + const scope = shouldScopeFind(locator) && !requiresRect ? query : undefined; const interactiveOnly = requiresRect; let lastSnapshotAt = 0; let lastNodes: SnapshotState['nodes'] | null = null; @@ -134,29 +138,16 @@ export async function handleFindCommands(params: { } const { nodes } = await fetchNodes(); - const bestMatches = findBestMatchesByLocator(nodes, locator, query, { - requireRect: requiresRect, + const matchResult = resolveFindMatch({ + nodes, + locator, + query, + requiresRect, + flags: req.flags, }); - - if (requiresRect && bestMatches.matches.length > 1) { - if (req.flags?.findFirst) { - bestMatches.matches = [bestMatches.matches[0]]; - } else if (req.flags?.findLast) { - bestMatches.matches = [bestMatches.matches[bestMatches.matches.length - 1]]; - } else { - return buildAmbiguousMatchError(bestMatches.matches, locator, query); - } - } - - const node = bestMatches.matches[0] ?? null; - if (!node) { - return errorResponse('COMMAND_FAILED', 'find did not match any element'); - } - - const resolvedNode = - action === 'click' || action === 'focus' || action === 'fill' || action === 'type' - ? (findNearestHittableAncestor(nodes, node) ?? node) - : node; + if (!matchResult.ok) return matchResult.response; + const node = matchResult.node; + const resolvedNode = requiresRect ? resolveInteractiveMatchNode(nodes, node) : node; const ref = `@${resolvedNode.ref}`; const actionFlags = { ...(req.flags ?? {}), noRecord: true }; const match: ResolvedMatch = { node, resolvedNode, ref, nodes, actionFlags }; @@ -177,6 +168,97 @@ export async function handleFindCommands(params: { // --- Per-action handlers --- +function isReadOnlyFindAction(action: string): boolean { + return ( + action === 'exists' || action === 'wait' || action === 'get_text' || action === 'get_attrs' + ); +} + +function findActionRequiresRect(action: string): boolean { + return action === 'click' || action === 'focus' || action === 'fill' || action === 'type'; +} + +function resolveFindMatch(params: { + nodes: SnapshotState['nodes']; + locator: FindLocator; + query: string; + requiresRect: boolean; + flags: DaemonRequest['flags']; +}): FindMatchResult { + const { nodes, locator, query, requiresRect, flags } = params; + const bestMatches = findBestMatchesByLocator(nodes, locator, query, { + requireRect: requiresRect, + }); + if (requiresRect) { + bestMatches.matches = preferOnscreenMatches(bestMatches.matches, nodes); + } + + if (requiresRect && bestMatches.matches.length > 1) { + if (flags?.findFirst) { + bestMatches.matches = [bestMatches.matches[0]]; + } else if (flags?.findLast) { + bestMatches.matches = [bestMatches.matches[bestMatches.matches.length - 1]]; + } else { + return { ok: false, response: buildAmbiguousMatchError(bestMatches.matches, locator, query) }; + } + } + + const node = bestMatches.matches[0] ?? null; + if (!node) { + return { + ok: false, + response: errorResponse('COMMAND_FAILED', 'find did not match any element'), + }; + } + return { ok: true, node }; +} + +function preferOnscreenMatches( + matches: SnapshotState['nodes'], + nodes: SnapshotState['nodes'], +): SnapshotState['nodes'] { + const viewport = nodes[0]?.rect; + if (!viewport) return matches; + const onscreen = matches.filter((node) => { + if (!node.rect) return false; + const center = centerOfRect(node.rect); + return ( + center.x >= viewport.x && + center.x <= viewport.x + viewport.width && + center.y >= viewport.y && + center.y <= viewport.y + viewport.height + ); + }); + return onscreen.length > 0 ? onscreen : matches; +} + +function resolveInteractiveMatchNode( + nodes: SnapshotState['nodes'], + node: SnapshotState['nodes'][number], +): SnapshotState['nodes'][number] { + const ancestor = findNearestHittableAncestor(nodes, node); + if (!ancestor) return node; + if (node.rect && isRootInteractionContainer(ancestor, nodes[0])) { + return node; + } + return ancestor; +} + +function isRootInteractionContainer( + node: SnapshotState['nodes'][number], + root: SnapshotState['nodes'][number] | undefined, +): boolean { + if (!root?.rect || !node.rect) return false; + const type = node.type?.toLowerCase() ?? ''; + if (!type.includes('application') && !type.includes('window')) return false; + return ( + node.rect.x === root.rect.x && + node.rect.y === root.rect.y && + node.rect.width === root.rect.width && + node.rect.height === root.rect.height + ); +} + async function handleFindWait( ctx: FindContext, fetchNodes: () => Promise<{ nodes: SnapshotState['nodes'] }>, @@ -266,7 +348,11 @@ async function handleFindClick(ctx: FindContext, match: ResolvedMatch): Promise< flags: match.actionFlags, }); if (!response.ok) return response; - const matchCoords = match.resolvedNode.rect ? centerOfRect(match.resolvedNode.rect) : null; + const matchCoords = match.resolvedNode.rect + ? centerOfRect(match.resolvedNode.rect) + : match.node.rect + ? centerOfRect(match.node.rect) + : null; const matchData: Record = { ref: match.ref, locator, query }; if (matchCoords) { matchData.x = matchCoords.x; @@ -312,7 +398,35 @@ async function handleFindFill( } async function handleFindFocus(ctx: FindContext, match: ResolvedMatch): Promise { - const { req, sessionStore, session, device, command, logPath } = ctx; + const response = await dispatchFocusForFindMatch(ctx, match); + if (!response.ok) return response; + recordFindAction(ctx, match, 'focus'); + return response; +} + +async function handleFindType( + ctx: FindContext, + match: ResolvedMatch, + value: string | undefined, +): Promise { + const { req, device, logPath, session } = ctx; + if (!value) { + return errorResponse('INVALID_ARGS', 'find type requires text'); + } + const focusResponse = await dispatchFocusForFindMatch(ctx, match); + if (!focusResponse.ok) return focusResponse; + const response = await dispatchCommand(device, 'type', [value], req.flags?.out, { + ...contextFromFlags(logPath, req.flags, session?.appBundleId, session?.trace?.outPath), + }); + recordFindAction(ctx, match, 'type'); + return { ok: true, data: response ?? { ref: match.ref } }; +} + +async function dispatchFocusForFindMatch( + ctx: FindContext, + match: ResolvedMatch, +): Promise { + const { req, device, logPath, session } = ctx; const coords = match.node.rect ? centerOfRect(match.node.rect) : null; if (!coords) { return errorResponse('COMMAND_FAILED', 'matched element has no bounds'); @@ -326,45 +440,19 @@ async function handleFindFocus(ctx: FindContext, match: ResolvedMatch): Promise< ...contextFromFlags(logPath, req.flags, session?.appBundleId, session?.trace?.outPath), }, ); - if (session) { - sessionStore.recordAction(session, { - command, - positionals: req.positionals ?? [], - flags: req.flags ?? {}, - result: { ref: match.ref, action: 'focus' }, - }); - } return { ok: true, data: response ?? { ref: match.ref } }; } -async function handleFindType( - ctx: FindContext, - match: ResolvedMatch, - value: string | undefined, -): Promise { - const { req, sessionStore, session, device, command, logPath } = ctx; - if (!value) { - return errorResponse('INVALID_ARGS', 'find type requires text'); - } - const coords = match.node.rect ? centerOfRect(match.node.rect) : null; - if (!coords) { - return errorResponse('COMMAND_FAILED', 'matched element has no bounds'); - } - await dispatchCommand(device, 'focus', [String(coords.x), String(coords.y)], req.flags?.out, { - ...contextFromFlags(logPath, req.flags, session?.appBundleId, session?.trace?.outPath), - }); - const response = await dispatchCommand(device, 'type', [value], req.flags?.out, { - ...contextFromFlags(logPath, req.flags, session?.appBundleId, session?.trace?.outPath), - }); +function recordFindAction(ctx: FindContext, match: ResolvedMatch, action: string): void { + const { req, sessionStore, session, command } = ctx; if (session) { sessionStore.recordAction(session, { command, positionals: req.positionals ?? [], flags: req.flags ?? {}, - result: { ref: match.ref, action: 'type' }, + result: { ref: match.ref, action }, }); } - return { ok: true, data: response ?? { ref: match.ref } }; } // --- Helpers --- diff --git a/src/daemon/handlers/session-replay-maestro-runtime.ts b/src/daemon/handlers/session-replay-maestro-runtime.ts index 93a41ae67..10b7d1db3 100644 --- a/src/daemon/handlers/session-replay-maestro-runtime.ts +++ b/src/daemon/handlers/session-replay-maestro-runtime.ts @@ -19,6 +19,20 @@ type MaestroReplayInvoker = (params: { step: number; }) => Promise; +type MaestroRuntimeInvoke = (req: DaemonRequest) => Promise; + +type MaestroScrollUntilVisibleParams = { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; +}; + +type MaestroTapOnParams = { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; +}; + export async function invokeMaestroRuntimeCommand(params: { command: string; baseReq: ReplayBaseRequest; @@ -45,11 +59,9 @@ export async function invokeMaestroRuntimeCommand(params: { } } -async function invokeMaestroScrollUntilVisible(params: { - baseReq: ReplayBaseRequest; - positionals: string[]; - invoke: (req: DaemonRequest) => Promise; -}): Promise { +async function invokeMaestroScrollUntilVisible( + params: MaestroScrollUntilVisibleParams, +): Promise { const [selector, timeoutValue = '5000', direction = 'down'] = params.positionals; if (!selector) { return errorResponse('INVALID_ARGS', 'scrollUntilVisible requires a selector.'); @@ -63,27 +75,14 @@ async function invokeMaestroScrollUntilVisible(params: { let lastWaitResponse: DaemonResponse | undefined; for (let index = 0; index < attempts; index += 1) { - const probeMs = Math.min( - MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS, - Math.max(1, timeoutMs - index * MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS), + const probe = await probeMaestroScrollVisibility( + params, + selector, + fuzzyTextQuery, + scrollProbeMs(timeoutMs, index), ); - const waitResponse = await params.invoke({ - ...params.baseReq, - command: 'wait', - positionals: [selector, String(probeMs)], - }); - if (waitResponse.ok) return waitResponse; - lastWaitResponse = waitResponse; - - const fuzzyResponse = fuzzyTextQuery - ? await params.invoke({ - ...params.baseReq, - command: 'find', - positionals: [fuzzyTextQuery, 'wait', String(probeMs)], - }) - : undefined; - if (fuzzyResponse?.ok) return fuzzyResponse; - lastWaitResponse = fuzzyResponse ?? lastWaitResponse; + if (probe.visible) return probe.response; + lastWaitResponse = probe.response; if (index === attempts - 1) break; @@ -98,6 +97,35 @@ async function invokeMaestroScrollUntilVisible(params: { return withMaestroScrollTimeoutContext(lastWaitResponse, selector, timeoutMs); } +async function probeMaestroScrollVisibility( + params: MaestroScrollUntilVisibleParams, + selector: string, + fuzzyTextQuery: string | null, + probeMs: number, +): Promise<{ visible: boolean; response: DaemonResponse }> { + const waitResponse = await params.invoke({ + ...params.baseReq, + command: 'wait', + positionals: [selector, String(probeMs)], + }); + if (waitResponse.ok) return { visible: true, response: waitResponse }; + if (!fuzzyTextQuery) return { visible: false, response: waitResponse }; + + const fuzzyResponse = await params.invoke({ + ...params.baseReq, + command: 'find', + positionals: [fuzzyTextQuery, 'wait', String(probeMs)], + }); + return { visible: fuzzyResponse.ok, response: fuzzyResponse }; +} + +function scrollProbeMs(timeoutMs: number, index: number): number { + return Math.min( + MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS, + Math.max(1, timeoutMs - index * MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS), + ); +} + async function invokeMaestroTapPointPercent(params: { baseReq: ReplayBaseRequest; positionals: string[]; @@ -160,11 +188,7 @@ function readSnapshotState(data: unknown): SnapshotState | undefined { return undefined; } -async function invokeMaestroTapOn(params: { - baseReq: ReplayBaseRequest; - positionals: string[]; - invoke: (req: DaemonRequest) => Promise; -}): Promise { +async function invokeMaestroTapOn(params: MaestroTapOnParams): Promise { const [selector] = params.positionals; if (!selector) { return errorResponse('INVALID_ARGS', 'tapOn requires a selector.'); @@ -174,13 +198,11 @@ async function invokeMaestroTapOn(params: { let lastResponse: DaemonResponse | undefined; while (Date.now() - startedAt < MAESTRO_TAP_ON_TIMEOUT_MS) { if (fuzzyTextQuery) { - const findResponse = await params.invoke({ - ...params.baseReq, - command: 'find', - positionals: [fuzzyTextQuery, 'click'], - }); - if (findResponse.ok) return findResponse; - lastResponse = findResponse; + const attempt = await invokeMaestroFuzzyTapOn(params, fuzzyTextQuery); + if (!attempt.retry) return attempt.response; + lastResponse = attempt.response; + await sleep(MAESTRO_TAP_ON_RETRY_MS); + continue; } const clickResponse = await params.invoke({ @@ -201,6 +223,28 @@ async function invokeMaestroTapOn(params: { ); } +async function invokeMaestroFuzzyTapOn( + params: MaestroTapOnParams, + query: string, +): Promise<{ retry: boolean; response: DaemonResponse }> { + const findResponse = await params.invoke({ + ...params.baseReq, + command: 'find', + positionals: [query, 'click'], + }); + if (findResponse.ok) return { retry: false, response: findResponse }; + if (params.baseReq.flags?.maestroOptional !== true) { + return { retry: true, response: findResponse }; + } + + const nativeLabelResponse = await params.invoke({ + ...params.baseReq, + command: 'click', + positionals: [simpleLabelSelector(query)], + }); + return { retry: !nativeLabelResponse.ok, response: nativeLabelResponse }; +} + async function invokeMaestroRunFlowWhen(params: { baseReq: ReplayBaseRequest; positionals: string[]; @@ -297,6 +341,10 @@ function extractMaestroVisibleTextQuery(selectorExpression: string): string | nu return first; } +function simpleLabelSelector(value: string): string { + return `label=${JSON.stringify(value)}`; +} + function withMaestroScrollTimeoutContext( response: DaemonResponse | undefined, selector: string, diff --git a/src/platforms/ios/apps.ts b/src/platforms/ios/apps.ts index 5eccf112c..6f091f1f1 100644 --- a/src/platforms/ios/apps.ts +++ b/src/platforms/ios/apps.ts @@ -238,6 +238,53 @@ export async function closeIosApp(device: DeviceInfo, app: string): Promise { + if (device.platform !== 'ios' || device.kind !== 'simulator') { + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'Maestro launchApp.clearState is currently supported only on iOS simulators.', + ); + } + + const bundleId = await resolveIosApp(device, app); + await ensureBootedSimulator(device); + await closeIosApp(device, bundleId); + + const result = await runSimctl(device, ['get_app_container', device.id, bundleId, 'data'], { + allowFailure: true, + }); + if (result.exitCode !== 0) { + throw new AppError('COMMAND_FAILED', `simctl get_app_container failed for ${bundleId}`, { + stdout: result.stdout, + stderr: result.stderr, + exitCode: result.exitCode, + }); + } + + const containerPath = result.stdout.trim(); + if (!containerPath) { + throw new AppError( + 'COMMAND_FAILED', + `simctl get_app_container returned an empty data container path for ${bundleId}`, + ); + } + + const entries = await fs.readdir(containerPath); + await Promise.all( + entries.map((entry) => + fs.rm(path.join(containerPath, entry), { + recursive: true, + force: true, + }), + ), + ); + + return { bundleId, containerPath }; +} + export async function uninstallIosApp( device: DeviceInfo, app: string, diff --git a/src/utils/command-schema.ts b/src/utils/command-schema.ts index ab5f29997..6b52c6c24 100644 --- a/src/utils/command-schema.ts +++ b/src/utils/command-schema.ts @@ -1261,7 +1261,7 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ type: 'boolean', usageLabel: '--maestro', usageDescription: - 'Replay: treat input as a Maestro YAML compatibility flow. Supported subset: launchApp with launch arguments but without state-reset side effects, runFlow file/inline with when.platform/visible/notVisible, runScript file/env with http.post/json/output variables, onFlowStart/onFlowComplete, deterministic repeat.times, tapOn including optional and absolute/percentage point taps, doubleTapOn, longPressOn, inputText, pasteText, openLink, assertVisible, assertNotVisible, extendedWaitUntil, scroll, scrollUntilVisible, absolute/percentage swipe, takeScreenshot, hideKeyboard, pressKey back/enter/home, back, waitForAnimationToEnd, and stopApp. ' + + 'Replay: treat input as a Maestro YAML compatibility flow. Supported subset: launchApp with launch arguments and iOS simulator clearState, runFlow file/inline with when.platform/visible/notVisible, runScript file/env with http.post/json/output variables, onFlowStart/onFlowComplete, deterministic repeat.times, tapOn including optional and absolute/percentage point taps, doubleTapOn, longPressOn, inputText, pasteText, openLink, assertVisible, assertNotVisible, extendedWaitUntil, scroll, scrollUntilVisible, absolute/percentage swipe, takeScreenshot, hideKeyboard, pressKey back/enter/home, back, waitForAnimationToEnd, and stopApp. ' + 'Unsupported syntax fails loudly with a link to https://github.com/callstackincubator/agent-device/issues/558', }, { From 78be55c2d70b5abc4066963b9f8866e4f5d8c60c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 22 May 2026 13:54:10 +0200 Subject: [PATCH 3/6] fix: address Maestro replay review feedback --- .../RunnerTests+Interaction.swift | 5 +- .../maestro/__tests__/replay-flow.test.ts | 27 +++++- src/compat/maestro/interactions.ts | 18 +++- src/compat/maestro/run-script.ts | 36 ++++++- src/core/__tests__/dispatch-open.test.ts | 23 +++++ src/core/dispatch.ts | 6 ++ .../__tests__/session-replay-vars.test.ts | 95 ++++++++++++++++++- .../session-replay-maestro-runtime.ts | 72 +++++++++++--- src/utils/command-schema.ts | 2 +- website/docs/docs/replay-e2e.md | 4 +- 10 files changed, 262 insertions(+), 26 deletions(-) diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 707f69386..c78c68537 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -236,7 +236,10 @@ extension RunnerTests { return false } let appFrame = app.frame - return appFrame.isEmpty || appFrame.intersects(frame) + if appFrame.isEmpty { + return true + } + return appFrame.contains(CGPoint(x: frame.midX, y: frame.midY)) } func queryElement(app: XCUIApplication, selectorKey: String, selectorValue: String) -> Response { diff --git a/src/compat/maestro/__tests__/replay-flow.test.ts b/src/compat/maestro/__tests__/replay-flow.test.ts index 3425baa77..f54d670af 100644 --- a/src/compat/maestro/__tests__/replay-flow.test.ts +++ b/src/compat/maestro/__tests__/replay-flow.test.ts @@ -69,7 +69,7 @@ env: ['scroll', ['right']], [ '__maestroScrollUntilVisible', - ['label="Discover" || text="Discover" || id="Discover"', '5000', 'down'], + ['label="Discover" || text="Discover" || id="Discover"', '5000', 'up'], ], ['screenshot', ['./screens/form.png']], ['keyboard', ['dismiss']], @@ -128,6 +128,29 @@ output.result = SERVER_PATH + ':' + json(res.body).appviewDid ); }); +test('parseMaestroReplayFlow reports runScript http failures with command context', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-maestro-runscript-fail-')); + const scriptPath = path.join(root, 'setup.js'); + const flowPath = path.join(root, 'flow.yml'); + fs.writeFileSync(scriptPath, `output.result = http.post('http://127.0.0.1:1').body`); + + assert.throws( + () => + parseMaestroReplayFlow( + `appId: com.callstack.agentdevicelab +--- +- runScript: ./setup.js +`, + { sourcePath: flowPath }, + ), + (error) => + error instanceof AppError && + error.code === 'COMMAND_FAILED' && + /runScript failed/.test(error.message) && + /http\.post failed/.test(error.message), + ); +}); + test('parseMaestroReplayFlow rejects unsupported Maestro commands', () => { assert.throws( () => parseMaestroReplayFlow('---\n- travelThroughTime: Save\n'), @@ -325,7 +348,7 @@ test('parseMaestroReplayFlow keeps visible-gated runFlow commands for runtime ev ]); }); -test('parseMaestroReplayFlow accepts launchApp reset options without state-reset side effects', () => { +test('parseMaestroReplayFlow accepts launchApp reset options', () => { const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab --- - launchApp: diff --git a/src/compat/maestro/interactions.ts b/src/compat/maestro/interactions.ts index 81153af5e..012e85226 100644 --- a/src/compat/maestro/interactions.ts +++ b/src/compat/maestro/interactions.ts @@ -142,9 +142,7 @@ export function convertScrollUntilVisible( assertOnlyKeys(value, 'scrollUntilVisible', ['element', 'direction', 'timeout']); const selector = maestroSelector(value.element, 'scrollUntilVisible.element', [], context); const direction = - typeof value.direction === 'string' - ? readScrollPositionalsFromDirectionSwipe(value.direction)[0] - : 'down'; + typeof value.direction === 'string' ? readScrollUntilVisibleDirection(value.direction) : 'down'; const timeoutMs = String(readTimeoutMs(value, 5000)); return [action(MAESTRO_RUNTIME_COMMAND.scrollUntilVisible, [selector, timeoutMs, direction])]; } @@ -198,6 +196,20 @@ function readScrollPositionalsFromDirectionSwipe(direction: string): string[] { } } +function readScrollUntilVisibleDirection(direction: string): string { + switch (direction.toLowerCase()) { + case 'up': + case 'down': + case 'left': + case 'right': + return direction.toLowerCase(); + default: + throw unsupportedMaestroSyntax( + 'Maestro scrollUntilVisible.direction must be UP, DOWN, LEFT, or RIGHT.', + ); + } +} + export function convertPressKey(value: unknown): SessionAction { const key = requireStringValue('pressKey', value).toLowerCase(); if (key === 'back') return action('back'); diff --git a/src/compat/maestro/run-script.ts b/src/compat/maestro/run-script.ts index 8f810aa2b..9e036568e 100644 --- a/src/compat/maestro/run-script.ts +++ b/src/compat/maestro/run-script.ts @@ -12,6 +12,8 @@ import { } from './support.ts'; import type { MaestroParseContext } from './types.ts'; +const RUN_SCRIPT_TIMEOUT_MS = 30_000; + type HttpResponse = { status: number; body: string; @@ -51,7 +53,7 @@ export function executeRunScript(value: unknown, context: MaestroParseContext): try { vm.runInNewContext(script, buildScriptGlobals(scriptEnv, output), { filename: scriptPath, - timeout: 30_000, + timeout: RUN_SCRIPT_TIMEOUT_MS, }); } catch (error) { throw new AppError( @@ -124,9 +126,32 @@ function runHttpRequestSync( headers: options?.headers ?? {}, body: options?.body ?? '', }), - timeoutMs: 30_000, + timeoutMs: RUN_SCRIPT_TIMEOUT_MS, + allowFailure: true, }); - return JSON.parse(result.stdout) as HttpResponse; + if (result.exitCode !== 0) { + throw new AppError( + 'COMMAND_FAILED', + `Maestro runScript http.${method.toLowerCase()} failed for ${url}: ${trimHttpErrorOutput(result.stderr)}`, + { + exitCode: result.exitCode, + stderr: result.stderr, + }, + ); + } + try { + return JSON.parse(result.stdout) as HttpResponse; + } catch (error) { + throw new AppError( + 'COMMAND_FAILED', + `Maestro runScript http.${method.toLowerCase()} returned invalid JSON for ${url}`, + { + stdout: result.stdout.slice(0, 1000), + stderr: result.stderr.slice(0, 1000), + }, + error instanceof Error ? error : undefined, + ); + } } function stringifyOutputValue(value: unknown): string { @@ -134,3 +159,8 @@ function stringifyOutputValue(value: unknown): string { if (typeof value === 'number' || typeof value === 'boolean') return String(value); return JSON.stringify(value); } + +function trimHttpErrorOutput(stderr: string): string { + const trimmed = stderr.trim(); + return trimmed.length > 0 ? trimmed.slice(0, 1000) : 'request process exited without stderr'; +} diff --git a/src/core/__tests__/dispatch-open.test.ts b/src/core/__tests__/dispatch-open.test.ts index c55cbbe68..807122348 100644 --- a/src/core/__tests__/dispatch-open.test.ts +++ b/src/core/__tests__/dispatch-open.test.ts @@ -23,3 +23,26 @@ test('dispatch open rejects URL as first argument when second URL is provided', }, ); }); + +test('dispatch open rejects Android launch arguments instead of dropping them', async () => { + const device: DeviceInfo = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel', + kind: 'emulator', + booted: true, + }; + + await assert.rejects( + () => + dispatchCommand(device, 'open', ['com.example.app'], undefined, { + launchArgs: ['--fixture', 'demo'], + }), + (error: unknown) => { + assert.equal(error instanceof AppError, true); + assert.equal((error as AppError).code, 'UNSUPPORTED_OPERATION'); + assert.match((error as AppError).message, /Apple platforms/i); + return true; + }, + ); +}); diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index f4eea78d1..1e5b12397 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -225,6 +225,12 @@ async function handleOpenCommand( if (launchConsole && isDeepLinkTarget(app)) { throw new AppError('INVALID_ARGS', LAUNCH_CONSOLE_DIRECT_APP_ONLY_MESSAGE); } + if (device.platform === 'android' && context?.launchArgs && context.launchArgs.length > 0) { + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'Launch arguments are currently supported only on Apple platforms.', + ); + } if (context?.maestroClearState) { if (isDeepLinkTarget(app)) { throw new AppError( diff --git a/src/daemon/handlers/__tests__/session-replay-vars.test.ts b/src/daemon/handlers/__tests__/session-replay-vars.test.ts index 2da6f3151..72cea0e02 100644 --- a/src/daemon/handlers/__tests__/session-replay-vars.test.ts +++ b/src/daemon/handlers/__tests__/session-replay-vars.test.ts @@ -480,10 +480,10 @@ test('runReplayScriptFile retries Maestro scrollUntilVisible with scroll probes' [ ['wait', ['label="Discover" || text="Discover" || id="Discover"', '500']], ['find', ['Discover', 'wait', '500']], - ['scroll', ['down']], + ['scroll', ['up']], ['wait', ['label="Discover" || text="Discover" || id="Discover"', '500']], ['find', ['Discover', 'wait', '500']], - ['scroll', ['down']], + ['scroll', ['up']], ['wait', ['label="Discover" || text="Discover" || id="Discover"', '200']], ], ); @@ -728,6 +728,61 @@ test('runReplayScriptFile skips Maestro runFlow.when.visible commands when absen ); }); +test('runReplayScriptFile skips Maestro runFlow.when.visible commands on false predicate', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-run-flow-when-visible-false-skip', + script: [ + 'appId: demo.app', + '---', + '- runFlow:', + ' when:', + ' visible: Continue', + ' commands:', + ' - tapOn: Continue', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + return { ok: true, data: { pass: false } }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['is', ['visible', 'label="Continue" || text="Continue" || id="Continue"']]], + ); +}); + +test('runReplayScriptFile propagates Maestro runFlow.when runtime errors', async () => { + const { response } = await runReplayFixture({ + label: 'maestro-run-flow-when-visible-runtime-error', + script: [ + 'appId: demo.app', + '---', + '- runFlow:', + ' when:', + ' visible: Continue', + ' commands:', + ' - tapOn: Continue', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async () => ({ + ok: false, + error: { code: 'UNKNOWN', message: 'fetch failed' }, + }), + }); + + assert.equal(response.ok, false); + if (!response.ok) { + assert.equal(response.error.code, 'UNKNOWN'); + assert.match(response.error.message, /fetch failed/); + } +}); + test('runReplayScriptFile runs Maestro runFlow.when.visible commands when present', async () => { const calls: CapturedInvocation[] = []; const { response } = await runReplayFixture({ @@ -766,6 +821,42 @@ test('runReplayScriptFile runs Maestro runFlow.when.visible commands when presen ); }); +test('runReplayScriptFile runs nested Maestro runtime commands inside runFlow.when', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-run-flow-when-nested-runtime', + script: [ + 'appId: demo.app', + '---', + '- runFlow:', + ' when:', + ' visible: Feed', + ' commands:', + ' - scrollUntilVisible:', + ' element: Done', + ' direction: DOWN', + ' timeout: 500', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'is') return { ok: true, data: { pass: true } }; + if (req.command === 'wait') return { ok: true, data: { found: true } }; + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['is', ['visible', 'label="Feed" || text="Feed" || id="Feed"']], + ['wait', ['label="Done" || text="Done" || id="Done"', '500']], + ], + ); +}); + test('runReplayScriptFile reads shell env from request (client-collected), not daemon process.env', async () => { // Ensure the daemon's own process.env does NOT contain AD_VAR_APP. assert.equal(process.env.AD_VAR_APP, undefined); diff --git a/src/daemon/handlers/session-replay-maestro-runtime.ts b/src/daemon/handlers/session-replay-maestro-runtime.ts index 10b7d1db3..56cac5e6a 100644 --- a/src/daemon/handlers/session-replay-maestro-runtime.ts +++ b/src/daemon/handlers/session-replay-maestro-runtime.ts @@ -33,6 +33,10 @@ type MaestroTapOnParams = { invoke: MaestroRuntimeInvoke; }; +type MaestroRunFlowWhenCondition = + | { ok: true; mode: string; predicate: string; selector: string } + | { ok: false; response: DaemonResponse }; + export async function invokeMaestroRuntimeCommand(params: { command: string; baseReq: ReplayBaseRequest; @@ -254,26 +258,56 @@ async function invokeMaestroRunFlowWhen(params: { invoke: (req: DaemonRequest) => Promise; invokeReplayAction: MaestroReplayInvoker; }): Promise { - const [mode, selector] = params.positionals; - if ((mode !== 'visible' && mode !== 'notVisible') || !selector) { - return errorResponse( - 'INVALID_ARGS', - 'runFlow.when requires visible/notVisible and a selector.', - ); - } - const predicate = mode === 'visible' ? 'visible' : 'hidden'; + const condition = readMaestroRunFlowWhenCondition(params.positionals); + if (!condition.ok) return condition.response; const conditionResponse = await params.invoke({ ...params.baseReq, command: 'is', - positionals: [predicate, selector], + positionals: [condition.predicate, condition.selector], flags: { ...params.baseReq.flags, noRecord: true }, }); - if (!conditionResponse.ok) { - return { ok: true, data: { skipped: true, condition: mode, selector } }; + if (isMaestroWhenConditionMiss(conditionResponse)) { + return { + ok: true, + data: { skipped: true, condition: condition.mode, selector: condition.selector }, + }; } + if (!conditionResponse.ok) return conditionResponse; + return await invokeMaestroRunFlowWhenSteps(params, condition); +} +function readMaestroRunFlowWhenCondition(positionals: string[]): MaestroRunFlowWhenCondition { + const [mode, selector] = positionals; + if ((mode !== 'visible' && mode !== 'notVisible') || !selector) { + return { + ok: false, + response: errorResponse( + 'INVALID_ARGS', + 'runFlow.when requires visible/notVisible and a selector.', + ), + }; + } + return { + ok: true, + mode, + predicate: mode === 'visible' ? 'visible' : 'hidden', + selector, + }; +} + +async function invokeMaestroRunFlowWhenSteps( + params: { + batchSteps: CommandFlags['batchSteps'] | undefined; + line: number; + step: number; + invokeReplayAction: MaestroReplayInvoker; + }, + condition: Extract, +): Promise { const steps = (params.batchSteps ?? []).map(batchStepToSessionAction); for (const [index, action] of steps.entries()) { + // Preserve stable parent-step ordering for nested runtime commands while + // keeping the substep distinguishable in traces. const response = await params.invokeReplayAction({ action, line: params.line, @@ -282,7 +316,16 @@ async function invokeMaestroRunFlowWhen(params: { if (!response.ok) return response; } - return { ok: true, data: { ran: steps.length, condition: mode, selector } }; + return { + ok: true, + data: { ran: steps.length, condition: condition.mode, selector: condition.selector }, + }; +} + +function isMaestroWhenConditionMiss(response: DaemonResponse): boolean { + if (response.ok) return response.data?.pass === false; + if (response.error.code !== 'COMMAND_FAILED') return false; + return response.error.details?.blockedBy !== 'android_foreground_surface'; } async function invokeMaestroPressEnter(params: { @@ -298,6 +341,9 @@ async function invokeMaestroPressEnter(params: { const message = response.error.message.toLowerCase(); if (!message.includes('fetch failed')) return response; + // Maestro compatibility: some iOS apps submit on Enter and immediately reset + // the runner transport. Treat this as recovered only after a fresh snapshot + // proves the runner connection is usable again; it does not assert UI state. const snapshotResponse = await params.invoke({ ...params.baseReq, command: 'snapshot', @@ -333,6 +379,8 @@ function extractMaestroVisibleTextQuery(selectorExpression: string): string | nu const chain = parseSelectorChain(selectorExpression); const terms = chain.selectors.flatMap((selector) => selector.terms); if (terms.length === 0) return null; + // Mixed selectors may encode more than a visible-text lookup, so they keep + // the exact selector path instead of fuzzy text fallback. if (!terms.some((term) => term.key === 'label' || term.key === 'text')) return null; if (!terms.every((term) => ['label', 'text', 'id'].includes(term.key))) return null; const values = terms.map((term) => (typeof term.value === 'string' ? term.value : '')); diff --git a/src/utils/command-schema.ts b/src/utils/command-schema.ts index 6b52c6c24..f0bfd1993 100644 --- a/src/utils/command-schema.ts +++ b/src/utils/command-schema.ts @@ -1261,7 +1261,7 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ type: 'boolean', usageLabel: '--maestro', usageDescription: - 'Replay: treat input as a Maestro YAML compatibility flow. Supported subset: launchApp with launch arguments and iOS simulator clearState, runFlow file/inline with when.platform/visible/notVisible, runScript file/env with http.post/json/output variables, onFlowStart/onFlowComplete, deterministic repeat.times, tapOn including optional and absolute/percentage point taps, doubleTapOn, longPressOn, inputText, pasteText, openLink, assertVisible, assertNotVisible, extendedWaitUntil, scroll, scrollUntilVisible, absolute/percentage swipe, takeScreenshot, hideKeyboard, pressKey back/enter/home, back, waitForAnimationToEnd, and stopApp. ' + + 'Replay: treat input as a Maestro YAML compatibility flow. Supported subset: launchApp with Apple-platform launch arguments and iOS simulator clearState, runFlow file/inline with when.platform/visible/notVisible, runScript file/env with parse-time http.post/json/output variables, onFlowStart/onFlowComplete, deterministic repeat.times, tapOn including optional and absolute/percentage point taps, doubleTapOn, longPressOn, inputText, pasteText, openLink, assertVisible, assertNotVisible, extendedWaitUntil, scroll, scrollUntilVisible, absolute/percentage swipe, takeScreenshot, hideKeyboard, pressKey back/enter/home, back, waitForAnimationToEnd, and stopApp. ' + 'Unsupported syntax fails loudly with a link to https://github.com/callstackincubator/agent-device/issues/558', }, { diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index 24b264501..7846bf503 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -61,11 +61,11 @@ Maestro compatibility translates supported YAML commands into Agent Device repla - Supported and unsupported capabilities: https://github.com/callstackincubator/agent-device/issues/558 - New focused compatibility request: https://github.com/callstackincubator/agent-device/issues/new -Currently supported areas include app launch with launch arguments but without state-reset side effects, file and inline `runFlow` with `when.platform`, `when.visible`, and `when.notVisible`, `onFlowStart` / `onFlowComplete`, deterministic `repeat.times`, `tapOn` including `optional` and absolute/percentage point taps, `doubleTapOn`, `longPressOn`, `inputText`, `pasteText`, `openLink`, visibility assertions, `extendedWaitUntil`, `scroll`, `scrollUntilVisible`, absolute/percentage `swipe`, screenshots, keyboard dismiss, basic `pressKey`, `back`, animation waits, and `stopApp`. `runScript` is supported only as a Maestro compatibility feature for file/env scripts that use `http.post`, `json`, and `output` variables; it is not a native `.ad` command. +Currently supported areas include app launch with Apple-platform launch arguments and iOS simulator `clearState`, file and inline `runFlow` with `when.platform`, `when.visible`, and `when.notVisible`, `onFlowStart` / `onFlowComplete`, deterministic `repeat.times`, `tapOn` including `optional` and absolute/percentage point taps, `doubleTapOn`, `longPressOn`, `inputText`, `pasteText`, `openLink`, visibility assertions, `extendedWaitUntil`, `scroll`, `scrollUntilVisible`, absolute/percentage `swipe`, screenshots, keyboard dismiss, basic `pressKey`, `back`, animation waits, and `stopApp`. `runScript` is supported only as a Maestro compatibility feature for file/env scripts that use `http.post`, `json`, and `output` variables; it executes during flow parsing, can make network requests, and is not a native `.ad` command or security sandbox. Maestro `env` values use the same replay precedence as `.ad` files: flow `env` is the default, shell `AD_VAR_*` values override it, and CLI `-e KEY=VALUE` wins over both. -Unsupported Maestro features such as `repeat.while`, `runFlow.when.true`, full expression predicates, `evalScript`, text clearing, selector relations such as `index` / `childOf`, device utility commands, and app state reset are tracked separately because they require neutral Agent Device runtime or device capabilities before they can be mapped safely. +Unsupported Maestro features such as `repeat.while`, `runFlow.when.true`, full expression predicates, `evalScript`, text clearing, selector relations such as `index` / `childOf`, device utility commands, Android app launch arguments, and Android app state reset are tracked separately because they require neutral Agent Device runtime or device capabilities before they can be mapped safely. ## Run a lightweight `.ad` suite From ed3e970eff3192fdad162719cc028a827370b210 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 26 May 2026 09:07:03 +0200 Subject: [PATCH 4/6] fix: finalize Maestro replay compatibility --- .../RunnerTests+CommandExecution.swift | 54 +- .../RunnerTests+Interaction.swift | 218 +++++- .../RunnerTests+Models.swift | 1 + src/__tests__/client.test.ts | 15 + src/backend.ts | 2 +- src/cli/commands/client-command.ts | 13 +- src/cli/commands/generic.ts | 1 + src/client-types.ts | 5 +- src/commands/selector-read.ts | 14 +- src/commands/system.ts | 31 +- .../maestro/__tests__/replay-flow.test.ts | 175 ++++- src/compat/maestro/command-mapper.ts | 21 +- src/compat/maestro/device-actions.ts | 7 +- src/compat/maestro/flow-control.ts | 5 +- src/compat/maestro/interactions.ts | 113 ++- src/compat/maestro/replay-flow.ts | 78 ++- src/compat/maestro/run-script.ts | 54 +- src/compat/maestro/runtime-commands.ts | 6 +- src/core/__tests__/dispatch-keyboard.test.ts | 48 ++ src/core/__tests__/dispatch-open.test.ts | 45 +- src/core/capabilities.ts | 2 +- src/core/dispatch-context.ts | 14 +- src/core/dispatch-interactions.ts | 31 + src/core/dispatch.ts | 42 +- src/core/interactor-types.ts | 5 + src/daemon-client.ts | 10 +- src/daemon/__tests__/context.test.ts | 12 +- src/daemon/context.ts | 3 +- src/daemon/handlers/__tests__/find.test.ts | 60 +- .../interaction-touch-targets.test.ts | 13 + .../handlers/__tests__/interaction.test.ts | 45 +- .../__tests__/session-replay-vars.test.ts | 657 +++++++++++++++++- .../handlers/__tests__/session-replay.test.ts | 3 +- src/daemon/handlers/find.ts | 51 +- .../handlers/interaction-touch-targets.ts | 5 +- src/daemon/handlers/interaction-touch.ts | 84 ++- .../handlers/session-replay-action-runtime.ts | 148 ++++ .../session-replay-maestro-runtime.ts | 623 +++++++++++++++-- src/daemon/handlers/session-replay-runtime.ts | 112 +-- src/daemon/handlers/session.ts | 6 +- src/platforms/android/input-actions.ts | 4 + .../ios/__tests__/runner-client.test.ts | 2 + src/platforms/ios/interactions.ts | 17 + src/platforms/ios/runner-contract.ts | 1 + src/replay/vars.ts | 32 +- src/utils/__tests__/args.test.ts | 15 +- src/utils/command-schema.ts | 13 +- website/docs/docs/replay-e2e.md | 4 +- 48 files changed, 2563 insertions(+), 357 deletions(-) create mode 100644 src/core/__tests__/dispatch-keyboard.test.ts create mode 100644 src/daemon/handlers/session-replay-action-runtime.ts diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index c7dd5cf96..5556202a3 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -742,6 +742,25 @@ extension RunnerTests { dismissed: result.dismissed ) ) + case .keyboardReturn: + let result = pressKeyboardReturn(app: activeApp) + if !result.pressed { + return Response( + ok: false, + error: ErrorPayload( + code: "UNSUPPORTED_OPERATION", + message: "Unable to press the iOS keyboard return key" + ) + ) + } + return Response( + ok: true, + data: DataPayload( + message: "keyboardReturn", + visible: result.visible, + wasVisible: result.wasVisible + ) + ) case .alert: let action = (command.action ?? "get").lowercased() guard let alert = resolveAlert(app: activeApp) else { @@ -852,7 +871,27 @@ extension RunnerTests { } let delaySeconds = Double(max(command.delayMs ?? 0, 0)) / 1000.0 let textEntryMode = resolveTextEntryMode(command) - let target = focusTextInputForTextEntry(app: activeApp, x: command.x, y: command.y) + let target: TextEntryTarget + if let selectorKey = command.selectorKey, let selectorValue = command.selectorValue { + let match = findElement( + app: activeApp, + selectorKey: selectorKey, + selectorValue: selectorValue, + allowNonHittableFallback: command.allowNonHittableSelectorTap == true + ) + if match.isAmbiguous { + return Response(ok: false, error: ErrorPayload(code: "AMBIGUOUS_MATCH", message: "selector matched multiple elements")) + } + guard let element = match.element else { + return Response(ok: false, error: ErrorPayload(code: "NO_MATCH", message: "selector did not match an element")) + } + guard isTextEntryElement(element) else { + return Response(ok: false, error: ErrorPayload(code: "INVALID_TARGET", message: "selector did not match a text input")) + } + target = focusTextInputForTextEntry(app: activeApp, element: element) + } else { + target = focusTextInputForTextEntry(app: activeApp, x: command.x, y: command.y) + } if textEntryMode == .replacement { guard target.element != nil else { let message = @@ -880,6 +919,17 @@ extension RunnerTests { ) ) } - return Response(ok: true, data: DataPayload(message: textResult.repaired ? "typed after repair" : "typed")) + let point = target.refreshPoint + let frame = activeApp.frame + return Response( + ok: true, + data: DataPayload( + message: textResult.repaired ? "typed after repair" : "typed", + x: point.map { Double($0.x) }, + y: point.map { Double($0.y) }, + referenceWidth: frame.isEmpty ? nil : Double(frame.width), + referenceHeight: frame.isEmpty ? nil : Double(frame.height) + ) + ) } } diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index c78c68537..5a87c01a4 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -335,7 +335,7 @@ extension RunnerTests { switch element.elementType { case .textField, .secureTextField, .searchField, .textView: let frame = element.frame - return !frame.isEmpty && frame.contains(point) + return !frame.isEmpty && frameContainsPoint(frame, point, tolerance: 2) default: return false } @@ -366,20 +366,31 @@ extension RunnerTests { return matched } + private func frameContainsPoint(_ frame: CGRect, _ point: CGPoint, tolerance: CGFloat) -> Bool { + point.x >= frame.minX - tolerance + && point.x <= frame.maxX + tolerance + && point.y >= frame.minY - tolerance + && point.y <= frame.maxY + tolerance + } + func focusedTextInput(app: XCUIApplication) -> XCUIElement? { +#if os(iOS) + return nil +#else var focused: XCUIElement? let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ - let candidate = app + let candidates = app .descendants(matching: .any) .matching(NSPredicate(format: "hasKeyboardFocus == 1")) - .firstMatch - guard candidate.exists else { return } - - switch candidate.elementType { - case .textField, .secureTextField, .searchField, .textView: - focused = candidate - default: - return + .allElementsBoundByIndex + for candidate in candidates where candidate.exists { + switch candidate.elementType { + case .textField, .secureTextField, .searchField, .textView: + focused = candidate + return + default: + continue + } } }) if let exceptionMessage { @@ -390,6 +401,7 @@ extension RunnerTests { return nil } return focused +#endif } func stabilizeTextInputBeforeTyping(app: XCUIApplication, target: XCUIElement?) -> XCUIElement? { @@ -449,6 +461,36 @@ extension RunnerTests { ) } + func focusTextInputForTextEntry(app: XCUIApplication, element: XCUIElement) -> TextEntryTarget { + let point = textEntryRefreshPoint(for: element) + if let point { + _ = tapAt(app: app, x: point.x, y: point.y) + } + let stabilized = stabilizeTextInputBeforeTyping(app: app, target: element) + let resolved = waitForTextEntryReadiness( + app: app, + target: TextEntryTarget( + element: stabilized ?? element, + refreshPoint: point, + prefersFocusedElement: false + ) + ) ?? stabilized ?? element + return TextEntryTarget( + element: resolved, + refreshPoint: textEntryRefreshPoint(for: resolved) ?? point, + prefersFocusedElement: false + ) + } + + func isTextEntryElement(_ element: XCUIElement) -> Bool { + switch element.elementType { + case .textField, .secureTextField, .searchField, .textView: + return true + default: + return false + } + } + func resolveTextEntryMode(_ command: Command) -> TextTypingRepairMode { switch command.textEntryMode { case "append": @@ -629,7 +671,7 @@ extension RunnerTests { guard let observedText = editableTextValue(for: targetElement) else { return TextEntryResult(verified: nil, repaired: repaired, expectedText: expectedText, observedText: nil) } - guard observedText == expectedText else { + guard textEntryValueMatchesExpected(targetElement, observedText: observedText, expectedText: expectedText) else { return TextEntryResult( verified: false, repaired: repaired, @@ -645,7 +687,11 @@ extension RunnerTests { return TextEntryResult(verified: nil, repaired: repaired, expectedText: expectedText, observedText: nil) } latestObservedText = nextObservedText - guard nextObservedText == expectedText else { + guard textEntryValueMatchesExpected( + resolveTextEntryElement(app: app, target: target), + observedText: nextObservedText, + expectedText: expectedText + ) else { return TextEntryResult( verified: false, repaired: repaired, @@ -662,6 +708,28 @@ extension RunnerTests { ) } + private func textEntryValueMatchesExpected( + _ element: XCUIElement?, + observedText: String, + expectedText: String + ) -> Bool { + if observedText == expectedText { + return true + } + guard hasTextEntrySubmitSuffix(expectedText), element?.elementType != .textView else { + return false + } + var submittedText = expectedText + while hasTextEntrySubmitSuffix(submittedText) { + submittedText.removeLast() + } + return observedText == submittedText + } + + private func hasTextEntrySubmitSuffix(_ text: String) -> Bool { + text.hasSuffix("\n") || text.hasSuffix("\r") + } + private func expectedTextEntryValue( typedText: String, mode: TextTypingRepairMode, @@ -693,7 +761,11 @@ extension RunnerTests { guard let observedText = editableTextValue(for: resolveTextEntryElement(app: app, target: target)) else { return false } - if observedText == expectedText { + if textEntryValueMatchesExpected( + resolveTextEntryElement(app: app, target: target), + observedText: observedText, + expectedText: expectedText + ) { return false } latestObservedText = observedText @@ -710,7 +782,11 @@ extension RunnerTests { guard let latestObservedText else { return false } - guard latestObservedText != expectedText else { + guard !textEntryValueMatchesExpected( + resolveTextEntryElement(app: app, target: target), + observedText: latestObservedText, + expectedText: expectedText + ) else { return false } return isRepairableTextEntryMismatch( @@ -904,6 +980,85 @@ extension RunnerTests { #endif } + func pressKeyboardReturn(app: XCUIApplication) -> (wasVisible: Bool, pressed: Bool, visible: Bool) { +#if os(tvOS) + return (wasVisible: false, pressed: pressTvRemote(.select), visible: false) +#elseif os(iOS) + let wasVisible = isKeyboardVisible(app: app) + if tapKeyboardReturnControl(app: app) { + sleepFor(0.2) + return (wasVisible: wasVisible, pressed: true, visible: isKeyboardVisible(app: app)) + } + + var typed = false + let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + app.typeText(XCUIKeyboardKey.return.rawValue) + typed = true + }) + if let exceptionMessage { + NSLog( + "AGENT_DEVICE_RUNNER_KEYBOARD_RETURN_IGNORED_EXCEPTION=%@", + exceptionMessage + ) + if let singleTarget = singleTextEntryElement(app: app) { + return pressKeyboardReturn(on: singleTarget, app: app, wasVisible: wasVisible) + } + return (wasVisible: wasVisible, pressed: false, visible: isKeyboardVisible(app: app)) + } + sleepFor(0.2) + return (wasVisible: wasVisible, pressed: typed, visible: isKeyboardVisible(app: app)) +#else + return (wasVisible: false, pressed: false, visible: false) +#endif + } + + private func pressKeyboardReturn( + on element: XCUIElement, + app: XCUIApplication, + wasVisible: Bool + ) -> (wasVisible: Bool, pressed: Bool, visible: Bool) { + let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + element.tap() + element.typeText(XCUIKeyboardKey.return.rawValue) + }) + if let exceptionMessage { + NSLog( + "AGENT_DEVICE_RUNNER_KEYBOARD_RETURN_TARGET_IGNORED_EXCEPTION=%@", + exceptionMessage + ) + return (wasVisible: wasVisible, pressed: false, visible: isKeyboardVisible(app: app)) + } + sleepFor(0.2) + return (wasVisible: wasVisible, pressed: true, visible: isKeyboardVisible(app: app)) + } + + private func singleTextEntryElement(app: XCUIApplication) -> XCUIElement? { +#if os(iOS) + var matches: [XCUIElement] = [] + let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + matches = app.descendants(matching: .any).allElementsBoundByIndex.filter { element in + guard element.exists else { return false } + switch element.elementType { + case .textField, .secureTextField, .searchField, .textView: + return true + default: + return false + } + } + }) + if let exceptionMessage { + NSLog( + "AGENT_DEVICE_RUNNER_KEYBOARD_RETURN_TEXT_ENTRY_QUERY_IGNORED_EXCEPTION=%@", + exceptionMessage + ) + return nil + } + return matches.count == 1 ? matches[0] : nil +#else + return nil +#endif + } + private func tapKeyboardDismissControl(app: XCUIApplication) -> Bool { #if os(tvOS) return false @@ -941,6 +1096,22 @@ extension RunnerTests { #endif } + private func tapKeyboardReturnControl(app: XCUIApplication) -> Bool { +#if os(iOS) + for label in ["return", "Return", "Enter", "Go", "Search", "Next", "Done", "Send", "Join"] { + let candidates = [ + app.keyboards.buttons[label], + app.keyboards.keys[label], + ] + if let hittable = candidates.first(where: { $0.exists && $0.isHittable }) { + hittable.tap() + return true + } + } +#endif + return false + } + private func isKeyboardAccessoryControl(_ element: XCUIElement, keyboardFrame: CGRect) -> Bool { let frame = element.frame guard !frame.isEmpty && !keyboardFrame.isEmpty else { @@ -1003,11 +1174,24 @@ extension RunnerTests { guard !normalizedValue.isEmpty else { return false } - guard let placeholder = element.placeholderValue?.trimmingCharacters(in: .whitespacesAndNewlines), - !placeholder.isEmpty else { + let placeholder = element.placeholderValue?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + if !placeholder.isEmpty && normalizedValue == placeholder { + return true + } + if isGenericTextInputLabel(normalizedValue) { + return true + } + let normalizedLabel = element.label.trimmingCharacters(in: .whitespacesAndNewlines) + return normalizedLabel == normalizedValue && isGenericTextInputLabel(normalizedLabel) + } + + private func isGenericTextInputLabel(_ value: String) -> Bool { + switch value { + case "Text input field": + return true + default: return false } - return normalizedValue == placeholder } private func readableText(for element: XCUIElement) -> String? { diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index 13a295e69..0020cc81f 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -23,6 +23,7 @@ enum CommandType: String, Codable { case rotate case appSwitcher case keyboardDismiss + case keyboardReturn case alert case pinch case rotateGesture diff --git a/src/__tests__/client.test.ts b/src/__tests__/client.test.ts index 1ed0a2174..79abe7822 100644 --- a/src/__tests__/client.test.ts +++ b/src/__tests__/client.test.ts @@ -492,6 +492,21 @@ test('replay.run keeps deprecated maestro option as backend alias', async () => assert.equal(setup.calls[0]?.flags?.replayBackend, 'maestro'); }); +test('replay.run forwards timeout budget', async () => { + const setup = createTransport(async () => ({ ok: true, data: {} })); + const client = createAgentDeviceClient(setup.config, { transport: setup.transport }); + + await client.replay.run({ + path: './flows/mod-lists.yaml', + backend: 'maestro', + timeoutMs: 240_000, + }); + + assert.equal(setup.calls.length, 1); + assert.equal(setup.calls[0]?.command, 'replay'); + assert.equal(setup.calls[0]?.flags?.timeoutMs, 240_000); +}); + test('client.command.wait prepares selector options and rejects invalid selectors', async () => { const setup = createTransport(async () => ({ ok: true, diff --git a/src/backend.ts b/src/backend.ts index eae5c551f..9cd46589d 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -92,7 +92,7 @@ export type BackendBackOptions = { }; export type BackendKeyboardOptions = { - action: 'status' | 'get' | 'dismiss'; + action: 'status' | 'get' | 'dismiss' | 'enter' | 'return'; }; export type BackendKeyboardResult = { diff --git a/src/cli/commands/client-command.ts b/src/cli/commands/client-command.ts index 828846300..e52bddbf9 100644 --- a/src/cli/commands/client-command.ts +++ b/src/cli/commands/client-command.ts @@ -173,10 +173,19 @@ function readKeyboardAction( ): KeyboardCommandOptions['action'] | undefined { const action = value?.toLowerCase(); if (action === 'get') return 'status'; - if (action === undefined || action === 'status' || action === 'dismiss') { + if ( + action === undefined || + action === 'status' || + action === 'dismiss' || + action === 'enter' || + action === 'return' + ) { return action; } - throw new AppError('INVALID_ARGS', 'keyboard action must be status, get, or dismiss.'); + throw new AppError( + 'INVALID_ARGS', + 'keyboard action must be status, get, dismiss, enter, or return.', + ); } function readFiniteNumber(value: string | undefined, label: string): number | undefined { diff --git a/src/cli/commands/generic.ts b/src/cli/commands/generic.ts index b75902085..8fa93ad0e 100644 --- a/src/cli/commands/generic.ts +++ b/src/cli/commands/generic.ts @@ -61,6 +61,7 @@ const genericClientCommandRunners = { update: flags.replayUpdate, backend: flags.replayMaestro ? 'maestro' : undefined, env: flags.replayEnv, + timeoutMs: flags.timeoutMs, }), test: ({ client, positionals, flags }) => { announceReplayTestRun({ json: flags.json }); diff --git a/src/client-types.ts b/src/client-types.ts index 2a44be877..8d28aab89 100644 --- a/src/client-types.ts +++ b/src/client-types.ts @@ -389,7 +389,7 @@ export type RotateCommandOptions = DeviceCommandBaseOptions & { export type AppSwitcherCommandOptions = DeviceCommandBaseOptions; export type KeyboardCommandOptions = DeviceCommandBaseOptions & { - action?: 'status' | 'dismiss'; + action?: 'status' | 'dismiss' | 'enter' | 'return'; }; export type ClipboardCommandOptions = @@ -449,7 +449,7 @@ export type AppSwitcherCommandResult = CommandActionResult<'app-switcher'>; export type KeyboardCommandResult = DaemonResponseData & { platform?: 'android' | 'ios'; - action?: 'status' | 'dismiss'; + action?: 'status' | 'dismiss' | 'enter'; visible?: boolean; inputType?: string | null; inputMethodPackage?: string | null; @@ -681,6 +681,7 @@ export type ReplayRunOptions = AgentDeviceRequestOverrides & { maestro?: boolean; backend?: string; env?: string[]; + timeoutMs?: number; }; export type ReplayTestOptions = AgentDeviceRequestOverrides & diff --git a/src/commands/selector-read.ts b/src/commands/selector-read.ts index b8443d27f..526ec7d39 100644 --- a/src/commands/selector-read.ts +++ b/src/commands/selector-read.ts @@ -303,7 +303,12 @@ export const isCommand: RuntimeCommand = asyn disambiguateAmbiguous: false, }); if (!resolved) { - throw new AppError('COMMAND_FAILED', formatSelectorFailure(chain, [], { unique: true })); + throw new AppError('COMMAND_FAILED', formatSelectorFailure(chain, [], { unique: true }), { + command: 'is', + reason: 'selector_not_found', + predicate: options.predicate, + selector: chain.raw, + }); } const result = evaluateIsPredicate({ predicate: options.predicate, @@ -316,6 +321,13 @@ export const isCommand: RuntimeCommand = asyn throw new AppError( 'COMMAND_FAILED', `is ${options.predicate} failed for selector ${resolved.selector.raw}: ${result.details}`, + { + command: 'is', + reason: 'predicate_failed', + predicate: options.predicate, + selector: resolved.selector.raw, + predicateDetails: result.details, + }, ); } return { diff --git a/src/commands/system.ts b/src/commands/system.ts index adbfe7f01..20d234b90 100644 --- a/src/commands/system.ts +++ b/src/commands/system.ts @@ -44,7 +44,7 @@ export type SystemRotateCommandResult = { }; export type SystemKeyboardCommandOptions = CommandContext & { - action?: 'status' | 'get' | 'dismiss'; + action?: 'status' | 'get' | 'dismiss' | 'enter' | 'return'; }; export type SystemKeyboardCommandResult = @@ -60,6 +60,13 @@ export type SystemKeyboardCommandResult = state: BackendKeyboardResult; backendResult?: Record; message?: string; + } + | { + kind: 'keyboardEnterPressed'; + action: 'enter'; + state: BackendKeyboardResult; + backendResult?: Record; + message?: string; }; export type SystemClipboardCommandOptions = @@ -200,11 +207,29 @@ export const keyboardCommand: RuntimeCommand< throw new AppError('UNSUPPORTED_OPERATION', 'system.keyboard is not supported by this backend'); } const action = options.action ?? 'status'; - if (action !== 'status' && action !== 'get' && action !== 'dismiss') { - throw new AppError('INVALID_ARGS', 'system.keyboard action must be status, get, or dismiss'); + if ( + action !== 'status' && + action !== 'get' && + action !== 'dismiss' && + action !== 'enter' && + action !== 'return' + ) { + throw new AppError( + 'INVALID_ARGS', + 'system.keyboard action must be status, get, dismiss, enter, or return', + ); } const state = await runtime.backend.setKeyboard(toBackendContext(runtime, options), { action }); const formattedBackendResult = toBackendResult(state); + if (action === 'enter' || action === 'return') { + return { + kind: 'keyboardEnterPressed', + action: 'enter', + state: isKeyboardResult(state) ? state : {}, + ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), + ...successText('Keyboard enter pressed'), + }; + } if (action === 'dismiss') { const dismissed = isKeyboardResult(state) ? state.dismissed : undefined; return { diff --git a/src/compat/maestro/__tests__/replay-flow.test.ts b/src/compat/maestro/__tests__/replay-flow.test.ts index f54d670af..27db41a1b 100644 --- a/src/compat/maestro/__tests__/replay-flow.test.ts +++ b/src/compat/maestro/__tests__/replay-flow.test.ts @@ -16,6 +16,7 @@ env: id: home-open-form - tapOn: point: 20%,20% + label: Dismiss save password prompt - doubleTapOn: id: release-notice delay: 150 @@ -62,7 +63,7 @@ env: ['__maestroTapOn', ['label="Full name" || text="Full name" || id="Full name"']], ['type', ['Ada Lovelace']], ['wait', ['label="Checkout form"', '5000']], - ['is', ['hidden', 'label="Missing banner"']], + ['__maestroAssertNotVisible', ['label="Missing banner"']], ['wait', ['id="submit-order"', '7000']], ['scroll', ['down']], ['scroll', ['down', '0.4']], @@ -79,8 +80,8 @@ env: assert.equal(parsed.actions[3]?.flags.doubleTap, true); assert.equal(parsed.actions[3]?.flags.intervalMs, 150); assert.equal(parsed.actions[4]?.flags.holdMs, 3000); - assert.equal(parsed.actions[1]?.flags.allowNonHittableSelectorTap, true); - assert.equal(parsed.actions[6]?.flags?.allowNonHittableSelectorTap, undefined); + assert.equal(parsed.actions[1]?.flags.maestro?.allowNonHittableSelectorTap, true); + assert.equal(parsed.actions[6]?.flags?.maestro?.allowNonHittableSelectorTap, undefined); }); test('parseMaestroReplayFlow maps iOS openLink through the app id when available', () => { @@ -98,17 +99,50 @@ test('parseMaestroReplayFlow maps iOS openLink through the app id when available ); }); -test('parseMaestroReplayFlow executes runScript and exposes output variables', () => { +test('parseMaestroReplayFlow converts Bluesky Maestro selector compatibility syntax', () => { + const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- eraseText +- eraseText: 12 +- tapOn: + id: likeBtn + childOf: + id: postThreadItem-by-bob.test +- tapOn: + id: postDropdownBtn + index: 0 +- tapOn: + label: Display name metadata + text: Display name +- swipe: + label: Drag feed down + from: + id: feed-drag-handle + direction: UP + duration: 350 +`); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [ + ['type', ['\b'.repeat(50)]], + ['type', ['\b'.repeat(12)]], + [ + '__maestroTapOn', + ['id="likeBtn"', JSON.stringify({ childOf: 'id="postThreadItem-by-bob.test"' })], + ], + ['__maestroTapOn', ['id="postDropdownBtn"', JSON.stringify({ index: 0 })]], + ['__maestroTapOn', ['label="Display name"']], + ['__maestroSwipeOn', ['id="feed-drag-handle"', 'up', '350']], + ], + ); +}); + +test('parseMaestroReplayFlow preserves runScript as an ordered runtime action', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-maestro-runscript-')); const scriptPath = path.join(root, 'setup.js'); const flowPath = path.join(root, 'flow.yml'); - fs.writeFileSync( - scriptPath, - ` -var res = {body: '{"appviewDid":"did:plc:test"}'} -output.result = SERVER_PATH + ':' + json(res.body).appviewDid -`, - ); + fs.writeFileSync(scriptPath, `output.result = SERVER_PATH`); const parsed = parseMaestroReplayFlow( `appId: com.callstack.agentdevicelab @@ -124,30 +158,83 @@ output.result = SERVER_PATH + ':' + json(res.body).appviewDid assert.deepEqual( parsed.actions.map((entry) => [entry.command, entry.positionals]), - [['type', ['local:did:plc:test']]], + [ + ['__maestroRunScript', [scriptPath]], + ['type', ['${output.result}']], + ], ); + assert.deepEqual(parsed.actions[0]?.flags.maestro?.runScriptEnv, { SERVER_PATH: 'local' }); }); -test('parseMaestroReplayFlow reports runScript http failures with command context', () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-maestro-runscript-fail-')); - const scriptPath = path.join(root, 'setup.js'); - const flowPath = path.join(root, 'flow.yml'); - fs.writeFileSync(scriptPath, `output.result = http.post('http://127.0.0.1:1').body`); +test('parseMaestroReplayFlow keeps focused inputText and pressKey Enter as separate actions', () => { + const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- inputText: hello +- pressKey: Enter +- inputText: world +`); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [ + ['type', ['hello']], + ['__maestroPressEnter', []], + ['type', ['world']], + ], + ); + assert.deepEqual(parsed.actionLines, [3, 4, 5]); +}); + +test('parseMaestroReplayFlow marks tapOn before inputText for snapshot tap focus', () => { + const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- tapOn: + id: editListNameInput +- inputText: Muted Users +`); + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [ + ['__maestroTapOn', ['id="editListNameInput"']], + ['type', ['Muted Users']], + ], + ); + assert.equal(parsed.actions[0]?.flags?.maestro?.allowNonHittableSelectorTap, undefined); +}); + +test('parseMaestroReplayFlow coalesces tapOn inputText while preserving pressKey Enter submit', () => { + const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- tapOn: + id: e2eProxyHeaderInput +- inputText: \${output.result} +- pressKey: Enter +`); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [ + ['wait', ['id="e2eProxyHeaderInput"', '30000']], + ['fill', ['id="e2eProxyHeaderInput"', '${output.result}']], + ['__maestroPressEnter', []], + ], + ); + assert.deepEqual(parsed.actionLines, [3, 3, 6]); + assert.equal(parsed.actions[1]?.flags?.maestro?.allowNonHittableSelectorTap, true); +}); + +test('parseMaestroReplayFlow rejects relative runScript paths without source path', () => { assert.throws( () => - parseMaestroReplayFlow( - `appId: com.callstack.agentdevicelab + parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab --- - runScript: ./setup.js -`, - { sourcePath: flowPath }, - ), +`), (error) => error instanceof AppError && - error.code === 'COMMAND_FAILED' && - /runScript failed/.test(error.message) && - /http\.post failed/.test(error.message), + error.code === 'INVALID_ARGS' && + /runScript file paths/.test(error.message), ); }); @@ -343,7 +430,7 @@ test('parseMaestroReplayFlow keeps visible-gated runFlow commands for runtime ev { command: '__maestroTapOn', positionals: ['label="Continue" || text="Continue" || id="Continue"'], - flags: {}, + flags: { maestro: { allowNonHittableSelectorTap: true } }, }, ]); }); @@ -353,7 +440,6 @@ test('parseMaestroReplayFlow accepts launchApp reset options', () => { --- - launchApp: clearState: true - clearKeychain: true arguments: "-EXDevMenuIsOnboardingFinished": true launchArguments: @@ -368,8 +454,7 @@ test('parseMaestroReplayFlow accepts launchApp reset options', () => { 'open', ['com.callstack.agentdevicelab'], { - maestroClearState: true, - relaunch: true, + maestro: { clearState: true }, launchArgs: ['-EXDevMenuIsOnboardingFinished', 'true', '-Example', 'ignored'], }, ], @@ -377,6 +462,38 @@ test('parseMaestroReplayFlow accepts launchApp reset options', () => { ); }); +test('parseMaestroReplayFlow rejects clearKeychain instead of ignoring it', () => { + assert.throws( + () => + parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- launchApp: + clearKeychain: true +`), + (error) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + /clearKeychain/.test(error.message), + ); +}); + +test('parseMaestroReplayFlow relaunches launchApp only when clearState is absent', () => { + const withLaunchArgs = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- launchApp: + arguments: + "-Example": "value" +`); + const withStopApp = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- launchApp: + stopApp: true +`); + + assert.equal(withLaunchArgs.actions[0]?.flags.relaunch, true); + assert.equal(withStopApp.actions[0]?.flags.relaunch, true); +}); + test('parseMaestroReplayFlow rejects unsupported runtime-dependent flow control', () => { assert.throws( () => diff --git a/src/compat/maestro/command-mapper.ts b/src/compat/maestro/command-mapper.ts index 0d1f12c81..30e36d60d 100644 --- a/src/compat/maestro/command-mapper.ts +++ b/src/compat/maestro/command-mapper.ts @@ -3,6 +3,7 @@ import { AppError } from '../../utils/errors.ts'; import { convertLaunchApp, convertStopApp } from './device-actions.ts'; import { convertDoubleTapOn, + convertEraseText, convertExtendedWaitUntil, convertLongPressOn, convertPressKey, @@ -22,7 +23,8 @@ import { unsupportedCommand, } from './support.ts'; import { convertRepeat, convertRunFlow } from './flow-control.ts'; -import { executeRunScript } from './run-script.ts'; +import { convertRunScript } from './run-script.ts'; +import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; import type { MaestroCommand, MaestroCommandMapperDeps, @@ -46,6 +48,7 @@ const MAP_COMMAND_HANDLERS: Record = { inputText: ({ value, context }) => [ action('type', [resolveMaestroString(readInputText(value), context)]), ], + eraseText: ({ value }) => [convertEraseText(value)], pasteText: ({ value, context, name }) => [ action('type', [resolveMaestroString(requireStringValue(name, value), context)]), ], @@ -54,7 +57,7 @@ const MAP_COMMAND_HANDLERS: Record = { action('wait', [maestroSelector(value, name, [], context), '5000']), ], assertNotVisible: ({ value, context, name }) => [ - action('is', ['hidden', maestroSelector(value, name, [], context)]), + action(MAESTRO_RUNTIME_COMMAND.assertNotVisible, [maestroSelector(value, name, [], context)]), ], extendedWaitUntil: ({ value, context }) => convertExtendedWaitUntil(value, context), takeScreenshot: ({ value, context, name }) => [ @@ -62,16 +65,15 @@ const MAP_COMMAND_HANDLERS: Record = { ], scroll: ({ value }) => [convertScroll(value)], scrollUntilVisible: ({ value, context }) => convertScrollUntilVisible(value, context), - swipe: ({ value }) => [convertSwipe(value)], + swipe: ({ value, context }) => [convertSwipe(value, context)], hideKeyboard: () => [action('keyboard', ['dismiss'])], pressKey: ({ value }) => [convertPressKey(value)], back: () => [action('back')], - waitForAnimationToEnd: ({ value }) => [action('wait', [String(readTimeoutMs(value, 250))])], + waitForAnimationToEnd: ({ value }) => [ + action(MAESTRO_RUNTIME_COMMAND.waitForAnimationToEnd, [String(readTimeoutMs(value, 15000))]), + ], stopApp: ({ value, config, context }) => [convertStopApp(value, config, context)], - runScript: ({ value, context }) => { - executeRunScript(value, context); - return []; - }, + runScript: ({ value, context }) => [convertRunScript(value, context)], runFlow: ({ value, config, context, deps }) => convertRunFlow(value, config, context, deps, convertCommandList), repeat: ({ value, config, context, deps }) => @@ -85,8 +87,9 @@ const SCALAR_COMMAND_HANDLERS: Record< launchApp: (config, context) => [convertLaunchApp(undefined, config, context)], scroll: () => [action('scroll', ['down'])], hideKeyboard: () => [action('keyboard', ['dismiss'])], + eraseText: () => [convertEraseText(undefined)], back: () => [action('back')], - waitForAnimationToEnd: () => [action('wait', ['250'])], + waitForAnimationToEnd: () => [action(MAESTRO_RUNTIME_COMMAND.waitForAnimationToEnd, ['15000'])], stopApp: (config, context) => [convertStopApp(undefined, config, context)], }; diff --git a/src/compat/maestro/device-actions.ts b/src/compat/maestro/device-actions.ts index 8dcaa62b0..b873acab2 100644 --- a/src/compat/maestro/device-actions.ts +++ b/src/compat/maestro/device-actions.ts @@ -32,16 +32,17 @@ export function convertLaunchApp( 'launchArguments', ]); rejectUnsupportedLaunchOption(value, 'permissions'); + rejectUnsupportedLaunchOption(value, 'clearKeychain'); const appId = resolveMaestroString( typeof value.appId === 'string' ? value.appId : requireAppId(config, 'launchApp'), context, ); const launchArgs = readLaunchArgs(value, context); const shouldClearState = value.clearState === true; - const shouldRelaunch = value.stopApp === true || shouldClearState || launchArgs.length > 0; + const shouldRelaunch = !shouldClearState && (value.stopApp === true || launchArgs.length > 0); return action('open', [appId], { - relaunch: shouldRelaunch, - ...(shouldClearState ? { maestroClearState: true } : {}), + ...(shouldRelaunch ? { relaunch: true } : {}), + ...(shouldClearState ? { maestro: { clearState: true } } : {}), ...(launchArgs.length > 0 ? { launchArgs } : {}), }); } diff --git a/src/compat/maestro/flow-control.ts b/src/compat/maestro/flow-control.ts index d11e8af87..513d0a1cb 100644 --- a/src/compat/maestro/flow-control.ts +++ b/src/compat/maestro/flow-control.ts @@ -20,7 +20,10 @@ import type { MaestroParseContext, } from './types.ts'; -const MAX_REPEAT_EXPANSIONS = 100; +// repeat.times is expanded at parse time for deterministic replay traces. Keep +// a guardrail until repeat can execute as a runtime loop without materializing +// every child action. +const MAX_REPEAT_EXPANSIONS = 1000; type ConvertCommandList = ( commands: MaestroCommand[], diff --git a/src/compat/maestro/interactions.ts b/src/compat/maestro/interactions.ts index 012e85226..eeb9f9695 100644 --- a/src/compat/maestro/interactions.ts +++ b/src/compat/maestro/interactions.ts @@ -17,14 +17,18 @@ import { import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; import type { MaestroParseContext } from './types.ts'; +type SwipeDirection = 'up' | 'down' | 'left' | 'right'; + export function convertTapOn(value: unknown, context: MaestroParseContext): SessionAction { if (typeof value === 'string') { - return action(MAESTRO_RUNTIME_COMMAND.tapOn, [ - visibleTextSelector(resolveMaestroString(value, context)), - ]); + return action( + MAESTRO_RUNTIME_COMMAND.tapOn, + [visibleTextSelector(resolveMaestroString(value, context))], + maestroTapOnFlags(value), + ); } if (isPlainRecord(value) && typeof value.point === 'string') { - assertOnlyKeys(value, 'tapOn', ['point', 'repeat', 'delay']); + assertOnlyKeys(value, 'tapOn', ['point', 'repeat', 'delay', 'optional', 'label']); const point = parseMaestroPoint(value.point); if (point.kind === 'percent') { return action( @@ -39,7 +43,9 @@ export function convertTapOn(value: unknown, context: MaestroParseContext): Sess assertOnlyKeys(value, 'tapOn', [ 'id', 'text', + 'childOf', 'enabled', + 'index', 'selected', 'repeat', 'delay', @@ -47,10 +53,19 @@ export function convertTapOn(value: unknown, context: MaestroParseContext): Sess 'label', ]); } + const flags = maestroTapOnFlags(value); return action( MAESTRO_RUNTIME_COMMAND.tapOn, - [maestroSelector(value, 'tapOn', ['repeat', 'delay', 'optional', 'label'], context)], - { ...tapFlags(value), allowNonHittableSelectorTap: true }, + [ + maestroSelector( + value, + 'tapOn', + ['repeat', 'delay', 'optional', 'label', 'index', 'childOf'], + context, + ), + ...maestroTapOnRuntimeOptions(value, context), + ], + flags, ); } @@ -94,6 +109,26 @@ export function readInputText(value: unknown): string { return value.text; } +export function convertEraseText(value: unknown): SessionAction { + if (value === null || value === undefined) return action('type', ['\b'.repeat(50)]); + if (typeof value === 'number' && Number.isInteger(value) && value > 0) { + return action('type', ['\b'.repeat(value)]); + } + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'eraseText expects empty, a positive count, or a map.'); + } + assertOnlyKeys(value, 'eraseText', ['charactersToErase']); + if (value.charactersToErase === undefined) return action('type', ['\b'.repeat(50)]); + if ( + typeof value.charactersToErase !== 'number' || + !Number.isInteger(value.charactersToErase) || + value.charactersToErase <= 0 + ) { + throw new AppError('INVALID_ARGS', 'eraseText.charactersToErase must be a positive integer.'); + } + return action('type', ['\b'.repeat(value.charactersToErase)]); +} + export function convertExtendedWaitUntil( value: unknown, context: MaestroParseContext, @@ -147,11 +182,22 @@ export function convertScrollUntilVisible( return [action(MAESTRO_RUNTIME_COMMAND.scrollUntilVisible, [selector, timeoutMs, direction])]; } -export function convertSwipe(value: unknown): SessionAction { +export function convertSwipe(value: unknown, context: MaestroParseContext): SessionAction { if (!isPlainRecord(value)) { throw new AppError('INVALID_ARGS', 'swipe expects a map.'); } - assertOnlyKeys(value, 'swipe', ['start', 'end', 'direction', 'duration']); + assertOnlyKeys(value, 'swipe', ['start', 'end', 'direction', 'duration', 'from', 'label']); + const from = value.from ?? (typeof value.label === 'string' ? value.label : undefined); + if (from !== undefined) { + const direction = readSwipeDirection( + typeof value.direction === 'string' ? value.direction : 'up', + ); + return action(MAESTRO_RUNTIME_COMMAND.swipeOn, [ + maestroSelector(from, 'swipe.from', [], context), + direction, + ...swipeDurationPositionals(value), + ]); + } if (typeof value.direction === 'string') { return action('scroll', readScrollPositionalsFromDirectionSwipe(value.direction)); } @@ -182,7 +228,7 @@ export function convertSwipe(value: unknown): SessionAction { } function readScrollPositionalsFromDirectionSwipe(direction: string): string[] { - switch (direction.toLowerCase()) { + switch (readSwipeDirection(direction)) { case 'up': return ['down']; case 'down': @@ -191,6 +237,17 @@ function readScrollPositionalsFromDirectionSwipe(direction: string): string[] { return ['right']; case 'right': return ['left']; + } +} + +function readSwipeDirection(direction: string): SwipeDirection { + const normalized = direction.toLowerCase(); + switch (normalized) { + case 'up': + case 'down': + case 'left': + case 'right': + return normalized; default: throw unsupportedMaestroSyntax('Maestro swipe direction must be UP, DOWN, LEFT, or RIGHT.'); } @@ -235,6 +292,8 @@ export function maestroSelector( terms.push(selectorTerm('id', resolveMaestroString(value.id, context))); if (typeof value.text === 'string') terms.push(selectorTerm('label', resolveMaestroString(value.text, context))); + if (typeof value.label === 'string' && terms.length === 0) + terms.push(selectorTerm('label', resolveMaestroString(value.label, context))); if (typeof value.enabled === 'boolean') terms.push(selectorTerm('enabled', String(value.enabled))); if (typeof value.selected === 'boolean') @@ -242,7 +301,7 @@ export function maestroSelector( if (terms.length === 0) { throw new AppError( 'INVALID_ARGS', - `${command} selector map must include one of id, text, enabled, or selected.`, + `${command} selector map must include one of id, text, label, enabled, or selected.`, ); } return terms.join(' '); @@ -256,6 +315,27 @@ function visibleTextSelector(value: string): string { ].join(' || '); } +function maestroTapOnRuntimeOptions(value: unknown, context: MaestroParseContext): string[] { + if (!isPlainRecord(value)) return []; + const options: { index?: number; childOf?: string } = {}; + if (value.index !== undefined) { + if (typeof value.index !== 'number' || !Number.isInteger(value.index) || value.index < 0) { + throw new AppError('INVALID_ARGS', 'tapOn.index must be a non-negative integer.'); + } + options.index = value.index; + } + if (value.childOf !== undefined) { + options.childOf = maestroSelector(value.childOf, 'tapOn.childOf', [], context); + } + return Object.keys(options).length > 0 ? [JSON.stringify(options)] : []; +} + +function swipeDurationPositionals(value: Record): string[] { + return typeof value.duration === 'number' && Number.isFinite(value.duration) + ? [String(Math.max(16, Math.floor(value.duration)))] + : []; +} + function selectorTerm(key: string, value: string): string { return `${key}=${JSON.stringify(value)}`; } @@ -267,10 +347,21 @@ function tapFlags(value: unknown): SessionAction['flags'] | undefined { const delay = nonNegativeInteger(value.delay); if (repeat && repeat > 1) flags.count = repeat; if (delay !== undefined) flags.intervalMs = delay; - if (value.optional === true) flags.maestroOptional = true; + if (value.optional === true) flags.maestro = { optional: true }; return Object.keys(flags).length > 0 ? flags : undefined; } +function maestroTapOnFlags(value: unknown): SessionAction['flags'] { + const flags = tapFlags(value) ?? {}; + return { + ...flags, + maestro: { + ...(flags.maestro ?? {}), + allowNonHittableSelectorTap: true, + }, + }; +} + function doubleTapFlags(value: unknown): SessionAction['flags'] { const flags: SessionAction['flags'] = { doubleTap: true }; if (isPlainRecord(value) && typeof value.delay === 'number' && Number.isInteger(value.delay)) { diff --git a/src/compat/maestro/replay-flow.ts b/src/compat/maestro/replay-flow.ts index e6b28cdaa..c1b0f669c 100644 --- a/src/compat/maestro/replay-flow.ts +++ b/src/compat/maestro/replay-flow.ts @@ -4,6 +4,7 @@ import { parseAllDocuments } from 'yaml'; import type { SessionAction } from '../../daemon/types.ts'; import { AppError } from '../../utils/errors.ts'; import { convertMaestroCommandWithLine } from './command-mapper.ts'; +import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; import { isPlainRecord, normalizeCommandList, normalizePlatform, readEnvMap } from './support.ts'; import type { MaestroCommand, @@ -74,7 +75,82 @@ function convertRootCommands(params: { actions.push(...converted); converted.forEach(() => actionLines.push(line)); } - return { actions, actionLines }; + return optimizeInputTextActions(actions, actionLines); +} + +function optimizeInputTextActions( + actions: SessionAction[], + actionLines: number[], +): { actions: SessionAction[]; actionLines: number[] } { + const maestroTapTimeoutMs = '30000'; + const mergedActions: SessionAction[] = []; + const mergedLines: number[] = []; + for (let index = 0; index < actions.length; index += 1) { + const action = actions[index]; + const nextAction = actions[index + 1]; + const typedAfterTap = readPlainTypeText(nextAction); + if (typedAfterTap !== null) { + const tapSelector = readPlainMaestroTapSelector(action); + const pressEnterAfterType = + actions[index + 2]?.command === MAESTRO_RUNTIME_COMMAND.pressEnter; + if (tapSelector !== null && pressEnterAfterType) { + mergedActions.push({ + ...action, + command: 'wait', + positionals: [tapSelector, maestroTapTimeoutMs], + }); + mergedLines.push(actionLines[index] ?? 1); + mergedActions.push({ + ...nextAction, + command: 'fill', + positionals: [tapSelector, typedAfterTap], + flags: action.flags, + }); + mergedLines.push(actionLines[index] ?? 1); + mergedActions.push(actions[index + 2] as SessionAction); + mergedLines.push(actionLines[index + 2] ?? actionLines[index] ?? 1); + index += 2; + continue; + } + if (tapSelector !== null) { + mergedActions.push(clearMaestroNonHittableTap(action)); + mergedLines.push(actionLines[index] ?? 1); + continue; + } + } + mergedActions.push(action); + mergedLines.push(actionLines[index] ?? 1); + } + return { actions: mergedActions, actionLines: mergedLines }; +} + +function clearMaestroNonHittableTap(action: SessionAction): SessionAction { + const maestro = { ...(action.flags?.maestro ?? {}) }; + delete maestro.allowNonHittableSelectorTap; + return { + ...action, + flags: { + ...(action.flags ?? {}), + maestro: { + ...maestro, + }, + }, + }; +} + +function readPlainMaestroTapSelector(action: SessionAction | undefined): string | null { + if (action?.command !== MAESTRO_RUNTIME_COMMAND.tapOn) return null; + const [selector, ...rest] = action.positionals ?? []; + if (rest.length > 0 || typeof selector !== 'string') return null; + return selector; +} + +function readPlainTypeText(action: SessionAction | undefined): string | null { + if (action?.command !== 'type') return null; + if (action.flags && Object.keys(action.flags).length > 0) return null; + const [text, ...rest] = action.positionals ?? []; + if (rest.length > 0 || typeof text !== 'string') return null; + return text; } function parseYamlDocuments(script: string): unknown[] { diff --git a/src/compat/maestro/run-script.ts b/src/compat/maestro/run-script.ts index 9e036568e..49f9f242e 100644 --- a/src/compat/maestro/run-script.ts +++ b/src/compat/maestro/run-script.ts @@ -1,9 +1,12 @@ import fs from 'node:fs'; import path from 'node:path'; import vm from 'node:vm'; +import type { SessionAction } from '../../daemon/types.ts'; import { AppError } from '../../utils/errors.ts'; import { runCmdSync } from '../../utils/exec.ts'; +import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; import { + action, assertOnlyKeys, isPlainRecord, readEnvMap, @@ -23,6 +26,10 @@ type HttpResponse = { const HTTP_REQUEST_SCRIPT = ` const fs = require('node:fs'); const input = JSON.parse(fs.readFileSync(0, 'utf8')); +if (typeof fetch !== 'function') { + console.error('global fetch is required for Maestro runScript http helpers'); + process.exit(1); +} fetch(input.url, { method: input.method, headers: input.headers, @@ -39,19 +46,30 @@ fetch(input.url, { }); `; -export function executeRunScript(value: unknown, context: MaestroParseContext): void { +export function convertRunScript(value: unknown, context: MaestroParseContext): SessionAction { const scriptConfig = readRunScriptConfig(value, context); const scriptPath = resolveRunScriptPath(scriptConfig.file, context); + return action(MAESTRO_RUNTIME_COMMAND.runScript, [scriptPath], { + ...(Object.keys(scriptConfig.env).length > 0 + ? { maestro: { runScriptEnv: scriptConfig.env } } + : {}), + }); +} + +export function executeRunScriptFile(params: { + scriptPath: string; + env: Record; +}): Record { + const { scriptPath, env } = params; const script = fs.readFileSync(scriptPath, 'utf8'); const output: Record = {}; - const scriptEnv = { - ...context.env, - ...scriptConfig.env, - ...context.envOverrides, - }; try { - vm.runInNewContext(script, buildScriptGlobals(scriptEnv, output), { + // Compatibility note: node:vm is not a security sandbox. Maestro runScript + // files are trusted flow-local setup code; the timeout only bounds + // synchronous script execution. Async http.post work is bounded separately + // by the child process timeout in runHttpRequestSync. + vm.runInNewContext(script, buildScriptGlobals(env, output), { filename: scriptPath, timeout: RUN_SCRIPT_TIMEOUT_MS, }); @@ -64,9 +82,13 @@ export function executeRunScript(value: unknown, context: MaestroParseContext): ); } - for (const [key, rawValue] of Object.entries(output)) { - context.env[`output.${key}`] = stringifyOutputValue(rawValue); - } + validateOutputKeys(output, scriptPath); + return Object.fromEntries( + Object.entries(output).map(([key, rawValue]) => [ + `output.${key}`, + stringifyOutputValue(rawValue), + ]), + ); } function readRunScriptConfig( @@ -119,6 +141,8 @@ function runHttpRequestSync( url: string, options?: { headers?: Record; body?: string }, ): HttpResponse { + // Keep http.post synchronous from the flow author's point of view while the + // network request remains timeout-bounded independently from node:vm. const result = runCmdSync(process.execPath, ['-e', HTTP_REQUEST_SCRIPT], { stdin: JSON.stringify({ method, @@ -154,6 +178,16 @@ function runHttpRequestSync( } } +function validateOutputKeys(output: Record, scriptPath: string): void { + for (const key of Object.keys(output)) { + if (!key.includes('.')) continue; + throw new AppError('INVALID_ARGS', `Maestro runScript output key cannot contain ".": ${key}`, { + scriptPath, + key, + }); + } +} + function stringifyOutputValue(value: unknown): string { if (typeof value === 'string') return value; if (typeof value === 'number' || typeof value === 'boolean') return String(value); diff --git a/src/compat/maestro/runtime-commands.ts b/src/compat/maestro/runtime-commands.ts index 02b816a4c..8e354f362 100644 --- a/src/compat/maestro/runtime-commands.ts +++ b/src/compat/maestro/runtime-commands.ts @@ -1,7 +1,11 @@ export const MAESTRO_RUNTIME_COMMAND = { - pressEnter: '__maestroPressEnter', runFlowWhen: '__maestroRunFlowWhen', + runScript: '__maestroRunScript', + assertNotVisible: '__maestroAssertNotVisible', + pressEnter: '__maestroPressEnter', + waitForAnimationToEnd: '__maestroWaitForAnimationToEnd', scrollUntilVisible: '__maestroScrollUntilVisible', + swipeOn: '__maestroSwipeOn', tapOn: '__maestroTapOn', tapPointPercent: '__maestroTapPointPercent', } as const; diff --git a/src/core/__tests__/dispatch-keyboard.test.ts b/src/core/__tests__/dispatch-keyboard.test.ts new file mode 100644 index 000000000..8cd0a8c2e --- /dev/null +++ b/src/core/__tests__/dispatch-keyboard.test.ts @@ -0,0 +1,48 @@ +import { beforeEach, test, vi } from 'vitest'; +import assert from 'node:assert/strict'; +import { promises as fs } from 'node:fs'; + +vi.mock('../../platforms/ios/runner-client.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, runIosRunnerCommand: vi.fn() }; +}); + +import { dispatchCommand } from '../dispatch.ts'; +import { runIosRunnerCommand } from '../../platforms/ios/runner-client.ts'; +import { ANDROID_EMULATOR, IOS_DEVICE } from '../../__tests__/test-utils/device-fixtures.ts'; +import { withMockedAdb } from '../../__tests__/test-utils/mocked-binaries.ts'; + +const mockRunIosRunnerCommand = vi.mocked(runIosRunnerCommand); + +beforeEach(() => { + vi.resetAllMocks(); + mockRunIosRunnerCommand.mockResolvedValue({ + message: 'keyboardReturn', + wasVisible: true, + visible: false, + }); +}); + +test('dispatch keyboard enter sends Android ENTER keyevent', async () => { + await withMockedAdb('agent-device-dispatch-keyboard-enter-', async (argsLogPath) => { + const result = await dispatchCommand(ANDROID_EMULATOR, 'keyboard', ['enter']); + + assert.equal(result?.action, 'enter'); + const logged = await fs.readFile(argsLogPath, 'utf8'); + assert.match(logged, /shell\ninput\nkeyevent\nENTER/); + }); +}); + +test('dispatch keyboard enter sends native iOS keyboard return command', async () => { + const result = await dispatchCommand(IOS_DEVICE, 'keyboard', ['return'], undefined, { + appBundleId: 'com.example.app', + }); + + assert.equal(result?.action, 'enter'); + assert.equal(result?.wasVisible, true); + assert.equal(mockRunIosRunnerCommand.mock.calls.length, 1); + assert.deepEqual(mockRunIosRunnerCommand.mock.calls[0]?.[1], { + command: 'keyboardReturn', + appBundleId: 'com.example.app', + }); +}); diff --git a/src/core/__tests__/dispatch-open.test.ts b/src/core/__tests__/dispatch-open.test.ts index 807122348..1cf399a73 100644 --- a/src/core/__tests__/dispatch-open.test.ts +++ b/src/core/__tests__/dispatch-open.test.ts @@ -1,8 +1,30 @@ -import { test } from 'vitest'; +import { beforeEach, test, vi } from 'vitest'; import assert from 'node:assert/strict'; import { dispatchCommand } from '../dispatch.ts'; import { AppError } from '../../utils/errors.ts'; import type { DeviceInfo } from '../../utils/device.ts'; +import { clearIosSimulatorAppState, openIosApp } from '../../platforms/ios/apps.ts'; +import { IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; + +vi.mock('../../platforms/ios/apps.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + clearIosSimulatorAppState: vi.fn(async () => ({ + bundleId: 'com.example.app', + containerPath: '/tmp/com.example.app', + })), + openIosApp: vi.fn(async () => {}), + }; +}); + +const mockClearIosSimulatorAppState = vi.mocked(clearIosSimulatorAppState); +const mockOpenIosApp = vi.mocked(openIosApp); + +beforeEach(() => { + mockClearIosSimulatorAppState.mockClear(); + mockOpenIosApp.mockClear(); +}); test('dispatch open rejects URL as first argument when second URL is provided', async () => { const device: DeviceInfo = { @@ -46,3 +68,24 @@ test('dispatch open rejects Android launch arguments instead of dropping them', }, ); }); + +test('dispatch open clears Maestro iOS simulator state and launches once', async () => { + const result = await dispatchCommand(IOS_SIMULATOR, 'open', ['com.example.app'], undefined, { + clearAppState: true, + launchArgs: ['-EXDevMenuIsOnboardingFinished', 'true'], + }); + + assert.equal(result?.app, 'com.example.app'); + assert.equal(mockClearIosSimulatorAppState.mock.calls.length, 1); + assert.deepEqual(mockClearIosSimulatorAppState.mock.calls[0]?.slice(0, 2), [ + IOS_SIMULATOR, + 'com.example.app', + ]); + assert.equal(mockOpenIosApp.mock.calls.length, 1); + assert.equal(mockOpenIosApp.mock.calls[0]?.[0], IOS_SIMULATOR); + assert.equal(mockOpenIosApp.mock.calls[0]?.[1], 'com.example.app'); + assert.deepEqual(mockOpenIosApp.mock.calls[0]?.[2]?.launchArgs, [ + '-EXDevMenuIsOnboardingFinished', + 'true', + ]); +}); diff --git a/src/core/capabilities.ts b/src/core/capabilities.ts index b96573010..8ff7bfeb7 100644 --- a/src/core/capabilities.ts +++ b/src/core/capabilities.ts @@ -96,7 +96,7 @@ const COMMAND_CAPABILITY_MATRIX: Record = { device.kind === 'simulator', }, keyboard: { - // iOS only supports keyboard dismiss; status/get remains Android-only. + // iOS only supports keyboard dismiss/enter; status/get remains Android-only. apple: { simulator: true, device: true }, android: { emulator: true, device: true, unknown: true }, linux: LINUX_NONE, diff --git a/src/core/dispatch-context.ts b/src/core/dispatch-context.ts index 9bf959b8b..bb8952881 100644 --- a/src/core/dispatch-context.ts +++ b/src/core/dispatch-context.ts @@ -10,13 +10,18 @@ export type BatchStep = { runtime?: unknown; }; +export type MaestroRuntimeFlags = { + allowNonHittableSelectorTap?: boolean; + clearState?: boolean; + optional?: boolean; + runScriptEnv?: Record; +}; + export type CommandFlags = Omit & { batchSteps?: BatchStep[]; launchArgs?: string[]; + maestro?: MaestroRuntimeFlags; replayBackend?: string; - allowNonHittableSelectorTap?: boolean; - maestroOptional?: boolean; - maestroClearState?: boolean; }; export type DispatchContext = ScreenshotDispatchFlags & { @@ -25,7 +30,7 @@ export type DispatchContext = ScreenshotDispatchFlags & { activity?: string; launchConsole?: string; launchArgs?: string[]; - maestroClearState?: boolean; + clearAppState?: boolean; verbose?: boolean; logPath?: string; traceLogPath?: string; @@ -46,7 +51,6 @@ export type DispatchContext = ScreenshotDispatchFlags & { pauseMs?: number; pattern?: 'one-way' | 'ping-pong'; surface?: SessionSurface; - allowNonHittableSelectorTap?: boolean; directElementSelector?: { key: 'id' | 'label' | 'text' | 'value'; value: string; diff --git a/src/core/dispatch-interactions.ts b/src/core/dispatch-interactions.ts index 530862e7f..76b78fa64 100644 --- a/src/core/dispatch-interactions.ts +++ b/src/core/dispatch-interactions.ts @@ -86,6 +86,15 @@ export async function handleFillCommand( positionals: string[], context: DispatchContext | undefined, ): Promise> { + if (context?.directElementSelector) { + return await handleDirectElementSelectorFill( + interactor, + context.directElementSelector, + positionals, + context, + ); + } + const x = Number(positionals[0]); const y = Number(positionals[1]); const text = positionals.slice(2).join(' '); @@ -97,6 +106,28 @@ export async function handleFillCommand( return { x, y, text, delayMs, ...successText(formatTextLengthMessage('Filled', text)) }; } +async function handleDirectElementSelectorFill( + interactor: Interactor, + selector: NonNullable, + positionals: string[], + context: DispatchContext, +): Promise> { + if (!interactor.fillElementSelector) { + throw new AppError('UNSUPPORTED_OPERATION', 'direct element selector fill is not supported'); + } + const text = positionals.join(' '); + if (!text) throw new AppError('INVALID_ARGS', 'fill requires text'); + const delayMs = requireIntInRange(context.delayMs ?? 0, 'delay-ms', 0, 10_000); + const result = await interactor.fillElementSelector(selector, text, delayMs); + return { + selector: selector.raw, + text, + delayMs, + ...(result ?? {}), + ...successText(formatTextLengthMessage('Filled', text)), + }; +} + export async function handlePressCommand( device: DeviceInfo, interactor: Interactor, diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index 1e5b12397..61a57b554 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -6,6 +6,7 @@ import { dismissAndroidKeyboard, getAndroidKeyboardState, } from '../platforms/android/device-input-state.ts'; +import { pressAndroidEnter } from '../platforms/android/input-actions.ts'; import { pushAndroidNotification } from '../platforms/android/notifications.ts'; import { getInteractor } from './interactors.ts'; import type { Interactor, RunnerContext } from './interactor-types.ts'; @@ -231,7 +232,7 @@ async function handleOpenCommand( 'Launch arguments are currently supported only on Apple platforms.', ); } - if (context?.maestroClearState) { + if (context?.clearAppState) { if (isDeepLinkTarget(app)) { throw new AppError( 'INVALID_ARGS', @@ -289,13 +290,30 @@ async function handleKeyboardCommand( runnerCtx: RunnerContext, ): Promise> { const action = (positionals[0] ?? 'status').toLowerCase(); - if (action !== 'status' && action !== 'get' && action !== 'dismiss') { - throw new AppError('INVALID_ARGS', 'keyboard requires a subcommand: status, get, or dismiss'); + if ( + action !== 'status' && + action !== 'get' && + action !== 'dismiss' && + action !== 'enter' && + action !== 'return' + ) { + throw new AppError( + 'INVALID_ARGS', + 'keyboard requires a subcommand: status, get, dismiss, enter, or return', + ); } if (positionals.length > 1) { throw new AppError('INVALID_ARGS', 'keyboard accepts at most one subcommand argument'); } if (device.platform === 'android') { + if (action === 'enter' || action === 'return') { + await pressAndroidEnter(device); + return { + platform: 'android', + action: 'enter', + ...successText('Keyboard enter pressed'), + }; + } if (action === 'dismiss') { const result = await dismissAndroidKeyboard(device); return { @@ -327,12 +345,26 @@ async function handleKeyboardCommand( }; } if (device.platform === 'ios') { - if (action !== 'dismiss') { + if (action !== 'dismiss' && action !== 'enter' && action !== 'return') { throw new AppError( 'UNSUPPORTED_OPERATION', - 'keyboard status/get is currently supported only on Android; use keyboard dismiss on iOS', + 'keyboard status/get is currently supported only on Android; use keyboard dismiss or enter on iOS', ); } + if (action === 'enter' || action === 'return') { + const result = await runIosRunnerCommand( + device, + { command: 'keyboardReturn', appBundleId: context?.appBundleId }, + runnerCtx, + ); + return { + platform: 'ios', + action: 'enter', + visible: result.visible, + wasVisible: result.wasVisible, + ...successText('Keyboard enter pressed'), + }; + } const result = await runIosRunnerCommand( device, { command: 'keyboardDismiss', appBundleId: context?.appBundleId }, diff --git a/src/core/interactor-types.ts b/src/core/interactor-types.ts index c87d19ab8..6b5abfa36 100644 --- a/src/core/interactor-types.ts +++ b/src/core/interactor-types.ts @@ -82,6 +82,11 @@ export type Interactor = { longPress(x: number, y: number, durationMs?: number): Promise | void>; focus(x: number, y: number): Promise | void>; type(text: string, delayMs?: number): Promise; + fillElementSelector?( + selector: ElementSelectorTapOptions, + text: string, + delayMs?: number, + ): Promise | void>; fill( x: number, y: number, diff --git a/src/daemon-client.ts b/src/daemon-client.ts index ba17f166e..2faf868c8 100644 --- a/src/daemon-client.ts +++ b/src/daemon-client.ts @@ -117,7 +117,7 @@ export async function sendToDaemon(req: Omit): Promise await ensureDaemon(settings), @@ -168,6 +168,14 @@ export async function sendToDaemon(req: Omit): Promise): number | undefined { + if (req.command === 'test') return undefined; + if (req.command === 'replay' && typeof req.flags?.timeoutMs === 'number') { + return req.flags.timeoutMs; + } + return REQUEST_TIMEOUT_MS; +} + export async function openApp(options: OpenAppOptions = {}): Promise { const { session = 'default', diff --git a/src/daemon/__tests__/context.test.ts b/src/daemon/__tests__/context.test.ts index bf117b083..35559226d 100644 --- a/src/daemon/__tests__/context.test.ts +++ b/src/daemon/__tests__/context.test.ts @@ -14,16 +14,10 @@ test('contextFromFlags forwards scroll pixels from CLI flags', () => { assert.equal(context.pixels, 240); }); -test('contextFromFlags forwards internal non-hittable selector tap flag', () => { - const flags: CommandFlags = { allowNonHittableSelectorTap: true }; +test('contextFromFlags maps Maestro clearState to generic app-state clearing', () => { + const flags: CommandFlags = { maestro: { clearState: true } }; const context = contextFromFlags('/tmp/agent-device.log', flags); - assert.equal(context.allowNonHittableSelectorTap, true); -}); - -test('contextFromFlags forwards Maestro clearState launch compatibility flag', () => { - const flags: CommandFlags = { maestroClearState: true }; - const context = contextFromFlags('/tmp/agent-device.log', flags); - assert.equal(context.maestroClearState, true); + assert.equal(context.clearAppState, true); }); test('contextFromFlags forwards screenshot flags from CLI flags', () => { diff --git a/src/daemon/context.ts b/src/daemon/context.ts index a3fd128c7..8234a521b 100644 --- a/src/daemon/context.ts +++ b/src/daemon/context.ts @@ -25,7 +25,7 @@ export function contextFromFlags( activity: flags?.activity, launchConsole: flags?.launchConsole, launchArgs: flags?.launchArgs, - maestroClearState: flags?.maestroClearState, + clearAppState: flags?.maestro?.clearState, verbose: flags?.verbose, logPath, traceLogPath, @@ -46,6 +46,5 @@ export function contextFromFlags( backMode: flags?.backMode, pauseMs: flags?.pauseMs, pattern: flags?.pattern, - allowNonHittableSelectorTap: flags?.allowNonHittableSelectorTap, }; } diff --git a/src/daemon/handlers/__tests__/find.test.ts b/src/daemon/handlers/__tests__/find.test.ts index 9b7d165ba..8c9b92c0d 100644 --- a/src/daemon/handlers/__tests__/find.test.ts +++ b/src/daemon/handlers/__tests__/find.test.ts @@ -95,6 +95,7 @@ test('handleFindCommands click returns deterministic metadata across locator var expectedLocator: 'any', expectedQuery: 'Increment', expectedCoordinates: { x: 100, y: 50 }, + expectedRef: '@e2', }, ]; @@ -104,7 +105,7 @@ test('handleFindCommands click returns deterministic metadata across locator var if (!response.ok) return; const data = response.data as Record; expect(Object.keys(data).sort()).toEqual(scenario.expectedKeys); - expect(data.ref).toBe('@e1'); + expect(data.ref).toBe(scenario.expectedRef); expect(data.locator).toBe(scenario.expectedLocator); expect(data.query).toBe(scenario.expectedQuery); @@ -117,7 +118,7 @@ test('handleFindCommands click returns deterministic metadata across locator var } expect(invokeCalls.length).toBe(1); - expect(invokeCalls[0].positionals?.[0]).toBe('@e1'); + expect(invokeCalls[0].positionals?.[0]).toBe(scenario.expectedRef); } }); @@ -157,6 +158,61 @@ test('handleFindCommands click prefers on-screen duplicate text matches', async expect(invokeCalls[0].positionals?.[0]).toBe('@e3'); }); +test('handleFindCommands click prefers semantic controls over matching containers', async () => { + const { response, invokeCalls } = await runFindClickScenario({ + positionals: ['Later', 'click'], + flags: { findFirst: true }, + nodes: [ + { + index: 0, + ref: 'e1', + type: 'Application', + hittable: true, + rect: { x: 0, y: 0, width: 440, height: 956 }, + }, + { + index: 1, + ref: 'e2', + type: 'Element(5)', + label: 'Dialog', + hittable: true, + rect: { x: 60, y: 356, width: 320, height: 272 }, + parentIndex: 0, + }, + { + index: 2, + ref: 'e3', + type: 'ScrollView', + label: 'Later', + hittable: false, + rect: { x: 60, y: 548, width: 320, height: 80 }, + parentIndex: 1, + }, + { + index: 3, + ref: 'e4', + type: 'Other', + label: 'Later', + hittable: false, + rect: { x: 76, y: 564, width: 288, height: 48 }, + parentIndex: 2, + }, + { + index: 4, + ref: 'e5', + type: 'Button', + label: 'Later', + hittable: false, + rect: { x: 76, y: 564, width: 140, height: 48 }, + parentIndex: 3, + }, + ], + }); + + expect(response.ok).toBe(true); + expect(invokeCalls[0].positionals?.[0]).toBe('@e5'); +}); + test('handleFindCommands wait bypasses snapshot cache while Android freshness recovery is active', async () => { const sessionName = 'android-find-wait'; const session: SessionState = { diff --git a/src/daemon/handlers/__tests__/interaction-touch-targets.test.ts b/src/daemon/handlers/__tests__/interaction-touch-targets.test.ts index eed6d7af5..fc29098f4 100644 --- a/src/daemon/handlers/__tests__/interaction-touch-targets.test.ts +++ b/src/daemon/handlers/__tests__/interaction-touch-targets.test.ts @@ -52,6 +52,19 @@ test('parseFillTarget reads selector text through shared fill codec', () => { }); }); +test('parseFillTarget preserves selector text whitespace', () => { + const parsed = parseFillTarget(['label="Command"', 'submit\n']); + + expect(parsed).toEqual({ + ok: true, + target: { + kind: 'selector', + selector: 'label="Command"', + }, + text: 'submit\n', + }); +}); + test('parseFillTarget rejects invalid coordinates instead of treating them as a point', () => { const parsed = parseFillTarget(['10', 'not-y', 'text']); diff --git a/src/daemon/handlers/__tests__/interaction.test.ts b/src/daemon/handlers/__tests__/interaction.test.ts index 116fb6f0a..2801ca469 100644 --- a/src/daemon/handlers/__tests__/interaction.test.ts +++ b/src/daemon/handlers/__tests__/interaction.test.ts @@ -410,6 +410,49 @@ test('click simple iOS id selector uses direct runner selector tap without snaps } }); +test('fill simple iOS id selector uses direct runner selector fill without snapshot coordinates', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'ios-direct-selector-fill'; + sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); + + mockDispatch.mockResolvedValue({ + message: 'filled', + x: 439.5, + y: 100.5, + referenceWidth: 440, + referenceHeight: 956, + }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'fill', + positionals: ['id="email"', 'ada@example.com'], + flags: { delayMs: 25 }, + }, + sessionName, + sessionStore, + contextFromFlags, + }); + + expect(response?.ok).toBe(true); + expect(mockDispatch).toHaveBeenCalledTimes(1); + expect(mockDispatch.mock.calls[0]?.[1]).toBe('fill'); + expect(mockDispatch.mock.calls[0]?.[2]).toEqual(['ada@example.com']); + const context = mockDispatch.mock.calls[0]?.[4] as Record; + expect(context.directElementSelector).toEqual({ + key: 'id', + value: 'email', + raw: 'id="email"', + }); + expect(context.delayMs).toBe(25); + if (response?.ok) { + expect(response.data?.selector).toBe('id="email"'); + expect(response.data?.text).toBe('ada@example.com'); + } +}); + test('click simple iOS selector forwards Maestro non-hittable tap backdoor', async () => { const sessionStore = makeSessionStore(); const sessionName = 'ios-maestro-selector-fallback'; @@ -429,7 +472,7 @@ test('click simple iOS selector forwards Maestro non-hittable tap backdoor', asy session: sessionName, command: 'click', positionals: ['id="e2eSignInAlice"'], - flags: { allowNonHittableSelectorTap: true }, + flags: { maestro: { allowNonHittableSelectorTap: true } }, }, sessionName, sessionStore, diff --git a/src/daemon/handlers/__tests__/session-replay-vars.test.ts b/src/daemon/handlers/__tests__/session-replay-vars.test.ts index 72cea0e02..20b71eb70 100644 --- a/src/daemon/handlers/__tests__/session-replay-vars.test.ts +++ b/src/daemon/handlers/__tests__/session-replay-vars.test.ts @@ -32,6 +32,7 @@ type CapturedInvocation = { async function runReplayFixture(params: { label: string; script: string; + files?: Record; flags?: CommandFlags; invoke?: (req: DaemonRequest) => Promise; }): Promise<{ @@ -41,11 +42,15 @@ async function runReplayFixture(params: { scriptPath: string; }> { const root = fs.mkdtempSync(path.join(os.tmpdir(), `agent-device-replay-${params.label}-`)); + for (const [name, contents] of Object.entries(params.files ?? {})) { + fs.writeFileSync(path.join(root, name), contents); + } const scriptPath = path.join(root, 'flow.ad'); fs.writeFileSync(scriptPath, params.script); const calls: CapturedInvocation[] = []; - const defaultInvoke = async (req: DaemonRequest): Promise => { + const invoke = async (req: DaemonRequest): Promise => { calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (params.invoke) return await params.invoke(req); return { ok: true, data: {} }; }; const response = await runReplayScriptFile({ @@ -60,7 +65,7 @@ async function runReplayFixture(params: { sessionName: 's', logPath: path.join(root, 'log'), sessionStore: new SessionStore(path.join(root, 'state')), - invoke: params.invoke ?? defaultInvoke, + invoke, }); return { response, calls, root, scriptPath }; } @@ -434,11 +439,105 @@ test('runReplayScriptFile applies CLI env overrides before Maestro compat mappin replayShellEnv: { AD_VAR_BUTTON_ID: 'shell-button' }, replayEnv: ['APP_ID=cli-app'], }, + invoke: async (req) => { + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + identifier: 'shell-button', + rect: { x: 20, y: 40, width: 120, height: 44 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, }); assert.equal(response.ok, true); assert.deepEqual(calls[0]?.positionals, ['cli-app']); - assert.deepEqual(calls[1]?.positionals, ['id="shell-button"']); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['open', ['cli-app']], + ['snapshot', []], + ['click', ['80', '62']], + ], + ); +}); + +test('runReplayScriptFile runs Maestro runScript in replay order and exposes output variables', async () => { + const { response, calls } = await runReplayFixture({ + label: 'maestro-runscript-runtime', + files: { + 'setup.js': ` +var res = {body: '{"appviewDid":"did:plc:test"}'} +output.result = SERVER_PATH + ':' + json(res.body).appviewDid +`, + }, + script: [ + 'appId: demo.app', + '---', + '- runScript:', + ' file: ./setup.js', + ' env:', + ' SERVER_PATH: local', + '- inputText: ${output.result}', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['type', ['local:did:plc:test']]], + ); +}); + +test('runReplayScriptFile reports Maestro runScript failures at the runScript step', async () => { + const { response, calls } = await runReplayFixture({ + label: 'maestro-runscript-fail', + files: { + 'setup.js': `output.result = http.post('http://127.0.0.1:1').body`, + }, + script: ['appId: demo.app', '---', '- runScript: ./setup.js', '- inputText: never', ''].join( + '\n', + ), + flags: { replayBackend: 'maestro' }, + }); + + assert.equal(response.ok, false); + if (!response.ok) { + assert.match(response.error.message, /Replay failed at step 1/); + assert.match(response.error.message, /runScript failed/); + assert.match(response.error.message, /http\.post failed/); + } + assert.equal(calls.length, 0); +}); + +test('runReplayScriptFile rejects Maestro runScript output keys containing dots', async () => { + const { response, calls } = await runReplayFixture({ + label: 'maestro-runscript-dotted-output', + files: { + 'setup.js': `output['nested.value'] = 'ambiguous'`, + }, + script: ['appId: demo.app', '---', '- runScript: ./setup.js', '- inputText: never', ''].join( + '\n', + ), + flags: { replayBackend: 'maestro' }, + }); + + assert.equal(response.ok, false); + if (!response.ok) { + assert.match(response.error.message, /Replay failed at step 1/); + assert.match(response.error.message, /output key cannot contain/); + } + assert.equal(calls.length, 0); }); test('runReplayScriptFile retries Maestro scrollUntilVisible with scroll probes', async () => { @@ -536,8 +635,208 @@ test('runReplayScriptFile lets Maestro tapOn use fuzzy visible text matching', a assert.equal(response.ok, true); assert.deepEqual( calls.map((call) => [call.command, call.positionals]), - [['find', ['Discover', 'click']]], + [ + ['snapshot', []], + ['find', ['Discover', 'click']], + ], + ); + assert.equal(calls[0]?.flags?.noRecord, true); + assert.equal(calls[1]?.flags?.findFirst, true); +}); + +test('runReplayScriptFile lets exact Maestro text tapOn win before fuzzy matching', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-exact-before-fuzzy', + script: ['appId: demo.app', '---', '- tapOn: Mute accounts', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + label: 'Block accounts', + rect: { x: 10, y: 600, width: 240, height: 44 }, + }, + { + index: 2, + label: 'Mute accounts', + rect: { x: 10, y: 540, width: 240, height: 44 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['130', '562']], + ], + ); +}); + +test('runReplayScriptFile prefers on-screen Maestro text tapOn matches', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-onscreen', + script: ['appId: demo.app', '---', '- tapOn: Sign in', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + type: 'Button', + label: 'Sign in', + rect: { x: -328, y: 182, width: 328, height: 42 }, + }, + { + index: 2, + type: 'Button', + label: 'Sign in', + rect: { x: 56, y: 842, width: 328, height: 56 }, + }, + ], + metadata: { referenceWidth: 440, referenceHeight: 956 }, + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['220', '870']], + ], + ); +}); + +test('runReplayScriptFile taps Maestro text near the label in large action containers', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-action-container', + script: ['appId: demo.app', '---', '- tapOn: Mute accounts', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + type: 'ScrollView', + label: 'Mute accounts', + rect: { x: 8, y: 805, width: 424, height: 93 }, + }, + { + index: 2, + parentIndex: 1, + type: 'Other', + label: 'Block accounts', + rect: { x: 31, y: 835, width: 377, height: 42 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['92', '829']], + ], + ); +}); + +test('runReplayScriptFile prefers actionable Maestro tapOn matches over broad ancestors', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-prefers-actionable-match', + script: ['appId: demo.app', '---', '- tapOn: New list', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + type: 'Other', + label: 'New list', + rect: { x: 0, y: 0, width: 440, height: 956 }, + }, + { + index: 2, + type: 'Button', + label: 'New list', + rect: { x: 349, y: 67, width: 75, height: 33 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['387', '84']], + ], + ); +}); + +test('runReplayScriptFile treats absent Maestro assertNotVisible targets as passing', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-assert-not-visible-absent', + script: ['appId: demo.app', '---', '- assertNotVisible: Feeds ✨', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + return { + ok: false, + error: { code: 'ELEMENT_NOT_FOUND', message: 'Selector did not match' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['is', ['visible', 'label="Feeds ✨" || text="Feeds ✨" || id="Feeds ✨"']]], ); + assert.equal(calls[0]?.flags?.noRecord, true); }); test('runReplayScriptFile retries Maestro fuzzy tapOn without raw selector fallback', async () => { @@ -564,33 +863,35 @@ test('runReplayScriptFile retries Maestro fuzzy tapOn without raw selector fallb assert.deepEqual( calls.map((call) => [call.command, call.positionals]), [ + ['snapshot', []], ['find', ['Discover', 'click']], + ['snapshot', []], ['find', ['Discover', 'click']], ], ); }); -test('runReplayScriptFile lets optional Maestro fuzzy tapOn hit native alert labels', async () => { +test('runReplayScriptFile lets optional Maestro fuzzy tapOn click first visible match', async () => { const calls: CapturedInvocation[] = []; const { response } = await runReplayFixture({ - label: 'maestro-tap-visible-text-optional-native-label', + label: 'maestro-tap-visible-text-optional-first-match', script: [ 'appId: demo.app', '---', '- tapOn:', - ' text: Not Now', + ' text: Later', ' optional: true', '', ].join('\n'), flags: { replayBackend: 'maestro' }, invoke: async (req) => { calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); - if (req.command === 'click' && req.positionals?.[0] === 'label="Not Now"') { - return { ok: true, data: { dismissed: true } }; + if (req.command === 'find' && req.flags?.findFirst === true) { + return { ok: true, data: { ref: '@e4', x: 220, y: 720 } }; } return { ok: false, - error: { code: 'ELEMENT_NOT_FOUND', message: 'element not found' }, + error: { code: 'AMBIGUOUS_MATCH', message: 'matched multiple elements' }, }; }, }); @@ -599,10 +900,11 @@ test('runReplayScriptFile lets optional Maestro fuzzy tapOn hit native alert lab assert.deepEqual( calls.map((call) => [call.command, call.positionals]), [ - ['find', ['Not Now', 'click']], - ['click', ['label="Not Now"']], + ['snapshot', []], + ['find', ['Later', 'click']], ], ); + assert.equal(calls[1]?.flags?.findFirst, true); }); test('runReplayScriptFile resolves Maestro percentage point taps from snapshot size', async () => { @@ -642,17 +944,37 @@ test('runReplayScriptFile resolves Maestro percentage point taps from snapshot s assert.equal(calls[0]?.flags?.noRecord, true); }); -test('runReplayScriptFile retries Maestro tapOn until the selector appears', async () => { +test('runReplayScriptFile retries Maestro id tapOn through snapshot coordinates', async () => { const calls: CapturedInvocation[] = []; - let clickAttempts = 0; + let snapshotAttempts = 0; const { response } = await runReplayFixture({ label: 'maestro-tap-on-retry', script: ['appId: demo.app', '---', '- tapOn:', ' id: delayedButton', ''].join('\n'), flags: { replayBackend: 'maestro' }, invoke: async (req) => { calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); - clickAttempts += 1; - if (clickAttempts === 3) return { ok: true, data: {} }; + if (req.command === 'snapshot') { + snapshotAttempts += 1; + if (snapshotAttempts === 3) { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + identifier: 'delayedButton', + rect: { x: 20, y: 40, width: 120, height: 44 }, + }, + ], + }, + }; + } + return { + ok: false, + error: { code: 'ELEMENT_NOT_FOUND', message: 'element not found' }, + }; + } + if (req.command === 'click') return { ok: true, data: {} }; return { ok: false, error: { code: 'ELEMENT_NOT_FOUND', message: 'element not found' }, @@ -664,25 +986,249 @@ test('runReplayScriptFile retries Maestro tapOn until the selector appears', asy assert.deepEqual( calls.map((call) => [call.command, call.positionals]), [ - ['click', ['id="delayedButton"']], - ['click', ['id="delayedButton"']], - ['click', ['id="delayedButton"']], + ['snapshot', []], + ['snapshot', []], + ['snapshot', []], + ['click', ['80', '62']], + ], + ); +}); + +test('runReplayScriptFile resolves Maestro tapOn index and childOf from snapshots', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-index-childof', + script: [ + 'appId: demo.app', + '---', + '- tapOn:', + ' id: likeBtn', + ' childOf:', + ' id: postThreadItem-by-bob.test', + '- tapOn:', + ' id: postDropdownBtn', + ' index: 1', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { index: 1, identifier: 'postThreadItem-by-alice.test' }, + { + index: 2, + parentIndex: 1, + identifier: 'likeBtn', + rect: { x: 10, y: 10, width: 40, height: 20 }, + }, + { index: 10, identifier: 'postThreadItem-by-bob.test' }, + { + index: 11, + parentIndex: 10, + identifier: 'likeBtn', + rect: { x: 20, y: 120, width: 40, height: 20 }, + }, + { + index: 20, + identifier: 'postDropdownBtn', + rect: { x: 100, y: 200, width: 40, height: 20 }, + }, + { + index: 21, + identifier: 'postDropdownBtn', + rect: { x: 200, y: 300, width: 40, height: 20 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['40', '130']], + ['snapshot', []], + ['click', ['220', '310']], ], ); + assert.equal(calls[0]?.flags?.noRecord, true); }); -test('runReplayScriptFile recovers Maestro enter submit after iOS runner transport reset', async () => { +test('runReplayScriptFile lets snapshot id tap handle Maestro one-point edge controls', async () => { const calls: CapturedInvocation[] = []; const { response } = await runReplayFixture({ - label: 'maestro-press-enter-recover', + label: 'maestro-tap-edge-rect', + script: ['appId: demo.app', '---', '- tapOn:', ' id: e2eSignInAlice', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + identifier: 'e2eSignInAlice', + rect: { x: 0, y: 0, width: 1, height: 1 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['0', '0']], + ], + ); +}); + +test('runReplayScriptFile keeps Maestro text-entry tapOn on the snapshot path', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-input-text-snapshot', + script: [ + 'appId: demo.app', + '---', + '- tapOn:', + ' id: editListNameInput', + '- inputText: Muted Users', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + identifier: 'editListNameInput', + rect: { x: 20, y: 100, width: 200, height: 40 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['120', '120']], + ['type', ['Muted Users']], + ], + ); + assert.equal(calls[0]?.flags?.noRecord, true); +}); + +test('runReplayScriptFile resolves Maestro swipe.label from a labeled element rect', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-swipe-label', + script: [ + 'appId: demo.app', + '---', + '- swipe:', + ' label: Thread body', + ' direction: UP', + ' duration: 400', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + label: 'Thread body', + rect: { x: 10, y: 100, width: 200, height: 300 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['swipe', ['110', '250', '110', '8', '400']], + ], + ); +}); + +test('runReplayScriptFile maps Maestro enter to keyboard enter', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-press-enter', script: ['appId: demo.app', '---', '- pressKey: Enter', ''].join('\n'), flags: { replayBackend: 'maestro' }, invoke: async (req) => { calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); - if (req.command === 'snapshot') return { ok: true, data: {} }; + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['keyboard', ['enter']]], + ); +}); + +test('runReplayScriptFile waits for Maestro animation snapshots to stabilize', async () => { + const calls: CapturedInvocation[] = []; + let snapshots = 0; + const { response } = await runReplayFixture({ + label: 'maestro-wait-animation-stable', + script: ['appId: demo.app', '---', '- waitForAnimationToEnd', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + snapshots += 1; + const y = snapshots === 1 ? 100 : 120; return { - ok: false, - error: { code: 'UNKNOWN', message: 'fetch failed' }, + ok: true, + data: { + nodes: [ + { + index: 1, + label: 'Animating', + rect: { x: 10, y, width: 100, height: 40 }, + }, + ], + }, }; }, }); @@ -691,10 +1237,37 @@ test('runReplayScriptFile recovers Maestro enter submit after iOS runner transpo assert.deepEqual( calls.map((call) => [call.command, call.positionals]), [ - ['type', ['\n']], + ['snapshot', []], + ['snapshot', []], ['snapshot', []], ], ); + assert.equal(calls[0]?.flags?.noRecord, true); +}); + +test('runReplayScriptFile falls back to newline type when keyboard enter is unsupported', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-press-enter-fallback', + script: ['appId: demo.app', '---', '- pressKey: Enter', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'keyboard') { + return { ok: false, error: { code: 'UNSUPPORTED_OPERATION', message: 'unsupported' } }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['keyboard', ['enter']], + ['type', ['\n']], + ], + ); }); test('runReplayScriptFile skips Maestro runFlow.when.visible commands when absent', async () => { @@ -716,7 +1289,11 @@ test('runReplayScriptFile skips Maestro runFlow.when.visible commands when absen calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); return { ok: false, - error: { code: 'COMMAND_FAILED', message: 'not visible' }, + error: { + code: 'COMMAND_FAILED', + message: 'not visible', + details: { command: 'is', reason: 'selector_not_found' }, + }, }; }, }); @@ -783,6 +1360,33 @@ test('runReplayScriptFile propagates Maestro runFlow.when runtime errors', async } }); +test('runReplayScriptFile propagates Maestro runFlow.when COMMAND_FAILED errors without condition-miss details', async () => { + const { response } = await runReplayFixture({ + label: 'maestro-run-flow-when-visible-command-error', + script: [ + 'appId: demo.app', + '---', + '- runFlow:', + ' when:', + ' visible: Continue', + ' commands:', + ' - tapOn: Continue', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async () => ({ + ok: false, + error: { code: 'COMMAND_FAILED', message: 'snapshot failed' }, + }), + }); + + assert.equal(response.ok, false); + if (!response.ok) { + assert.equal(response.error.code, 'COMMAND_FAILED'); + assert.match(response.error.message, /snapshot failed/); + } +}); + test('runReplayScriptFile runs Maestro runFlow.when.visible commands when present', async () => { const calls: CapturedInvocation[] = []; const { response } = await runReplayFixture({ @@ -816,6 +1420,7 @@ test('runReplayScriptFile runs Maestro runFlow.when.visible commands when presen calls.map((call) => [call.command, call.positionals]), [ ['is', ['visible', 'label="Continue" || text="Continue" || id="Continue"']], + ['snapshot', []], ['find', ['Continue', 'click']], ], ); diff --git a/src/daemon/handlers/__tests__/session-replay.test.ts b/src/daemon/handlers/__tests__/session-replay.test.ts index ae98d0939..74e656d03 100644 --- a/src/daemon/handlers/__tests__/session-replay.test.ts +++ b/src/daemon/handlers/__tests__/session-replay.test.ts @@ -1,6 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; -import { buildReplayActionFlags, withReplayFailureContext } from '../session-replay-runtime.ts'; +import { buildReplayActionFlags } from '../session-replay-action-runtime.ts'; +import { withReplayFailureContext } from '../session-replay-runtime.ts'; import { buildNestedReplayFlags } from '../session-replay.ts'; test('buildReplayActionFlags keeps allowed parent flags only', () => { diff --git a/src/daemon/handlers/find.ts b/src/daemon/handlers/find.ts index 66363d85b..7efc4988d 100644 --- a/src/daemon/handlers/find.ts +++ b/src/daemon/handlers/find.ts @@ -6,7 +6,11 @@ import type { DaemonRequest, DaemonResponse } from '../types.ts'; import { SessionStore } from '../session-store.ts'; import { contextFromFlags } from '../context.ts'; import { ensureDeviceReady } from '../device-ready.ts'; -import { extractNodeText, findNearestHittableAncestor } from '../snapshot-processing.ts'; +import { extractNodeText } from '../snapshot-processing.ts'; +import { + resolveActionableTouchNode, + resolveActionableTouchResolution, +} from '../../commands/interaction-targeting.ts'; import { readTextForNode } from './interaction-read.ts'; import { captureSnapshot } from './snapshot-capture.ts'; import { setSessionSnapshot } from '../session-snapshot.ts'; @@ -229,19 +233,50 @@ function preferOnscreenMatches( center.y <= viewport.y + viewport.height ); }); - return onscreen.length > 0 ? onscreen : matches; + return rankInteractiveMatches(onscreen.length > 0 ? onscreen : matches, nodes); +} + +function rankInteractiveMatches( + matches: SnapshotState['nodes'], + nodes: SnapshotState['nodes'], +): SnapshotState['nodes'] { + if (matches.length < 2) return matches; + return matches + .map((node, index) => ({ node, index, score: interactiveMatchScore(node, nodes) })) + .sort((left, right) => { + if (right.score !== left.score) return right.score - left.score; + return rectArea(left.node) - rectArea(right.node) || left.index - right.index; + }) + .map((entry) => entry.node); +} + +function interactiveMatchScore( + node: SnapshotState['nodes'][number], + nodes: SnapshotState['nodes'], +): number { + const resolution = resolveActionableTouchResolution(nodes, node); + if (resolution.reason === 'semantic-target' && resolution.node.rect) return 4; + if (resolution.reason === 'same-rect-descendant' && resolution.node.rect) return 4; + if ( + resolution.reason === 'hittable-ancestor' && + resolution.node.rect && + !isRootInteractionContainer(resolution.node, nodes[0]) + ) { + return 2; + } + if (node.hittable && node.rect && !isRootInteractionContainer(node, nodes[0])) return 3; + return node.rect ? 1 : 0; +} + +function rectArea(node: SnapshotState['nodes'][number]): number { + return node.rect ? node.rect.width * node.rect.height : Number.POSITIVE_INFINITY; } function resolveInteractiveMatchNode( nodes: SnapshotState['nodes'], node: SnapshotState['nodes'][number], ): SnapshotState['nodes'][number] { - const ancestor = findNearestHittableAncestor(nodes, node); - if (!ancestor) return node; - if (node.rect && isRootInteractionContainer(ancestor, nodes[0])) { - return node; - } - return ancestor; + return resolveActionableTouchNode(nodes, node); } function isRootInteractionContainer( diff --git a/src/daemon/handlers/interaction-touch-targets.ts b/src/daemon/handlers/interaction-touch-targets.ts index 264302ef4..d682531c0 100644 --- a/src/daemon/handlers/interaction-touch-targets.ts +++ b/src/daemon/handlers/interaction-touch-targets.ts @@ -110,8 +110,7 @@ export function parseFillTarget(positionals: string[]): ParsedFillTarget { ), }; } - const text = parsed.text.trim(); - if (!text) { + if (!parsed.text.trim()) { return { ok: false, response: errorResponse('INVALID_ARGS', 'fill requires text after selector'), @@ -120,7 +119,7 @@ export function parseFillTarget(positionals: string[]): ParsedFillTarget { return { ok: true, target: { kind: 'selector', selector: parsed.target.selector }, - text, + text: parsed.text, }; } diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/handlers/interaction-touch.ts index abeab8f6d..82f4ee072 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/handlers/interaction-touch.ts @@ -261,7 +261,7 @@ function readDirectIosSelectorTapTarget(params: { if (!selector) return null; return { ...selector, - ...(flags?.allowNonHittableSelectorTap ? { allowNonHittableTap: true } : {}), + ...(flags?.maestro?.allowNonHittableSelectorTap ? { allowNonHittableTap: true } : {}), }; } @@ -371,6 +371,20 @@ async function dispatchFillViaRuntime( if (invalidRefFlagsResponse) return invalidRefFlagsResponse; await refreshAndroidRefSnapshotIfFreshnessActive(params, session); } + const directSelector = readDirectIosSelectorFillTarget({ + session, + target: parsedTarget.target, + flags: req.flags, + }); + if (directSelector) { + const directResponse = await dispatchDirectIosSelectorFill( + params, + session, + directSelector, + parsedTarget.text, + ); + if (directResponse) return directResponse; + } return await dispatchRuntimeInteraction(params, { run: async (runtime) => @@ -412,6 +426,74 @@ async function dispatchFillViaRuntime( }); } +function readDirectIosSelectorFillTarget(params: { + session: SessionState; + target: InteractionTarget; + flags: CommandFlags | undefined; +}): DirectIosSelectorTarget | null { + const { session, target, flags } = params; + if (target.kind !== 'selector') return null; + const selector = readSimpleIosSelectorTarget({ session, selectorExpression: target.selector }); + if (!selector) return null; + return { + ...selector, + ...(flags?.maestro?.allowNonHittableSelectorTap ? { allowNonHittableTap: true } : {}), + }; +} + +async function dispatchDirectIosSelectorFill( + params: InteractionHandlerParams, + session: SessionState, + selector: DirectIosSelectorTarget, + text: string, +): Promise { + const actionStartedAt = Date.now(); + try { + const data = + (await dispatchCommand(session.device, 'fill', [text], params.req.flags?.out, { + ...params.contextFromFlags(params.req.flags, session.appBundleId, session.trace?.outPath), + directElementSelector: selector, + surface: session.surface, + })) ?? {}; + const actionFinishedAt = Date.now(); + const point = readPointFromDirectSelectorTapResult(data); + const responseData = buildTouchVisualizationResult({ + data, + fallbackX: point.x, + fallbackY: point.y, + referenceFrame: readReferenceFrameFromDirectSelectorTapResult(data), + extra: { + selector: selector.raw, + text, + }, + }); + return finalizeTouchInteraction({ + session, + sessionStore: params.sessionStore, + command: params.req.command, + positionals: params.req.positionals ?? [], + flags: params.req.flags, + result: responseData, + responseData, + actionStartedAt, + actionFinishedAt, + }); + } catch (error) { + if (!isDirectIosSelectorFallbackError(error)) { + return { ok: false, error: normalizeError(error) }; + } + emitDiagnostic({ + level: 'debug', + phase: 'ios_direct_selector_fill_fallback', + data: { + selector: selector.raw, + error: error instanceof Error ? error.message : String(error), + }, + }); + return null; + } +} + async function dispatchRuntimeInteraction< TResult extends PressCommandResult | FillCommandResult | LongPressCommandResult, >( diff --git a/src/daemon/handlers/session-replay-action-runtime.ts b/src/daemon/handlers/session-replay-action-runtime.ts new file mode 100644 index 000000000..6ad7e658e --- /dev/null +++ b/src/daemon/handlers/session-replay-action-runtime.ts @@ -0,0 +1,148 @@ +import fs from 'node:fs'; +import type { CommandFlags } from '../../core/dispatch.ts'; +import { + mergeReplayVarScopeValues, + resolveReplayAction, + type ReplayVarScope, +} from '../../replay/vars.ts'; +import type { DaemonRequest, DaemonResponse, SessionAction } from '../types.ts'; +import { mergeParentFlags } from './handler-utils.ts'; +import { invokeMaestroRuntimeCommand } from './session-replay-maestro-runtime.ts'; + +type ReplayBaseRequest = Omit; + +type ReplayActionInvoker = (params: { + action: SessionAction; + line: number; + step: number; +}) => Promise; + +export async function invokeReplayAction(params: { + req: DaemonRequest; + sessionName: string; + action: SessionAction; + scope: ReplayVarScope; + filePath: string; + line: number; + step: number; + tracePath?: string; + invoke: (req: DaemonRequest) => Promise; +}): Promise { + const { req, sessionName, action, scope, filePath, line, step, tracePath, invoke } = params; + const resolved = resolveReplayAction(action, scope, { file: filePath, line }); + const invokeNestedReplayAction: ReplayActionInvoker = (nested) => + invokeReplayAction({ + req, + sessionName, + action: nested.action, + scope, + filePath, + line: nested.line, + step: nested.step, + tracePath, + invoke, + }); + const startedAt = Date.now(); + appendReplayTraceEvent(tracePath, { + type: 'replay_action_start', + ts: new Date(startedAt).toISOString(), + replayPath: filePath, + line, + step, + command: resolved.command, + positionals: resolved.positionals ?? [], + }); + + const response = await invokeResolvedReplayAction({ + req, + sessionName, + resolved, + scope, + line, + step, + invoke, + invokeReplayAction: invokeNestedReplayAction, + }); + + const finishedAt = Date.now(); + appendReplayTraceEvent(tracePath, { + type: 'replay_action_stop', + ts: new Date(finishedAt).toISOString(), + replayPath: filePath, + line, + step, + command: resolved.command, + ok: response.ok, + durationMs: finishedAt - startedAt, + errorCode: response.ok ? undefined : response.error.code, + }); + return response; +} + +async function invokeResolvedReplayAction(params: { + req: DaemonRequest; + sessionName: string; + resolved: SessionAction; + scope: ReplayVarScope; + line: number; + step: number; + invoke: (req: DaemonRequest) => Promise; + invokeReplayAction: ReplayActionInvoker; +}): Promise { + const { req, sessionName, resolved, scope, line, step, invoke, invokeReplayAction } = params; + const flags = buildReplayActionFlags(req.flags, resolved.flags); + const baseReq: ReplayBaseRequest = { + token: req.token, + session: sessionName, + flags, + runtime: resolved.runtime, + meta: req.meta, + }; + const response = + (await invokeMaestroRuntimeCommand({ + command: resolved.command, + baseReq, + positionals: resolved.positionals ?? [], + batchSteps: resolved.flags?.batchSteps, + scope, + line, + step, + invoke, + invokeReplayAction, + })) ?? + (await invoke({ + ...baseReq, + command: resolved.command, + positionals: resolved.positionals ?? [], + })); + if (response.ok) { + const outputEnv = readReplayOutputEnv(response.data); + if (outputEnv) mergeReplayVarScopeValues(scope, outputEnv); + } + return response; +} + +function readReplayOutputEnv(data: unknown): Record | null { + if (!data || typeof data !== 'object') return null; + const raw = (data as { outputEnv?: unknown }).outputEnv; + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return null; + const entries = Object.entries(raw).filter( + (entry): entry is [string, string] => typeof entry[1] === 'string', + ); + return entries.length > 0 ? Object.fromEntries(entries) : null; +} + +function appendReplayTraceEvent( + tracePath: string | undefined, + event: Record, +): void { + if (!tracePath) return; + fs.appendFileSync(tracePath, `${JSON.stringify(event)}\n`); +} + +export function buildReplayActionFlags( + parentFlags: CommandFlags | undefined, + actionFlags: SessionAction['flags'] | undefined, +): CommandFlags { + return mergeParentFlags(parentFlags, { ...(actionFlags ?? {}) }); +} diff --git a/src/daemon/handlers/session-replay-maestro-runtime.ts b/src/daemon/handlers/session-replay-maestro-runtime.ts index 56cac5e6a..00923591d 100644 --- a/src/daemon/handlers/session-replay-maestro-runtime.ts +++ b/src/daemon/handlers/session-replay-maestro-runtime.ts @@ -1,15 +1,22 @@ import { type CommandFlags } from '../../core/dispatch.ts'; import { MAESTRO_RUNTIME_COMMAND } from '../../compat/maestro/runtime-commands.ts'; -import type { SnapshotState } from '../../utils/snapshot.ts'; +import { executeRunScriptFile } from '../../compat/maestro/run-script.ts'; +import type { Platform } from '../../utils/device.ts'; +import { type Rect, type SnapshotNode, type SnapshotState } from '../../utils/snapshot.ts'; import { sleep } from '../../utils/timeouts.ts'; +import { asAppError } from '../../utils/errors.ts'; +import type { ReplayVarScope } from '../../replay/vars.ts'; import { parseSelectorChain } from '../selectors.ts'; -import { getSnapshotReferenceFrame } from '../touch-reference-frame.ts'; +import { matchesSelector } from '../selectors-match.ts'; +import { getSnapshotReferenceFrame, type TouchReferenceFrame } from '../touch-reference-frame.ts'; import type { DaemonRequest, DaemonResponse, SessionAction } from '../types.ts'; import { errorResponse } from './response.ts'; const MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS = 500; const MAESTRO_TAP_ON_TIMEOUT_MS = 30000; +const MAESTRO_OPTIONAL_TAP_ON_TIMEOUT_MS = 3000; const MAESTRO_TAP_ON_RETRY_MS = 250; +const MAESTRO_ANIMATION_POLL_MS = 250; type ReplayBaseRequest = Omit; @@ -20,6 +27,7 @@ type MaestroReplayInvoker = (params: { }) => Promise; type MaestroRuntimeInvoke = (req: DaemonRequest) => Promise; +type FailedDaemonResponse = Extract; type MaestroScrollUntilVisibleParams = { baseReq: ReplayBaseRequest; @@ -33,36 +41,144 @@ type MaestroTapOnParams = { invoke: MaestroRuntimeInvoke; }; +type MaestroTapOnOptions = { + childOf?: string; + index?: number; +}; + type MaestroRunFlowWhenCondition = | { ok: true; mode: string; predicate: string; selector: string } | { ok: false; response: DaemonResponse }; +type MaestroSnapshotTarget = { + node: SnapshotNode; + rect: Rect; + frame?: TouchReferenceFrame; +}; + export async function invokeMaestroRuntimeCommand(params: { command: string; baseReq: ReplayBaseRequest; positionals: string[]; batchSteps: CommandFlags['batchSteps'] | undefined; + scope: ReplayVarScope; line: number; step: number; invoke: (req: DaemonRequest) => Promise; invokeReplayAction: MaestroReplayInvoker; }): Promise { switch (params.command) { + case MAESTRO_RUNTIME_COMMAND.assertNotVisible: + return await invokeMaestroAssertNotVisible(params); + case MAESTRO_RUNTIME_COMMAND.pressEnter: + return await invokeMaestroPressEnter(params); + case MAESTRO_RUNTIME_COMMAND.waitForAnimationToEnd: + return await invokeMaestroWaitForAnimationToEnd(params); case MAESTRO_RUNTIME_COMMAND.scrollUntilVisible: return await invokeMaestroScrollUntilVisible(params); + case MAESTRO_RUNTIME_COMMAND.swipeOn: + return await invokeMaestroSwipeOn(params); case MAESTRO_RUNTIME_COMMAND.tapOn: return await invokeMaestroTapOn(params); case MAESTRO_RUNTIME_COMMAND.tapPointPercent: return await invokeMaestroTapPointPercent(params); case MAESTRO_RUNTIME_COMMAND.runFlowWhen: return await invokeMaestroRunFlowWhen(params); - case MAESTRO_RUNTIME_COMMAND.pressEnter: - return await invokeMaestroPressEnter(params); + case MAESTRO_RUNTIME_COMMAND.runScript: + return invokeMaestroRunScript(params); default: return undefined; } } +async function invokeMaestroPressEnter(params: { + baseReq: ReplayBaseRequest; + invoke: MaestroRuntimeInvoke; +}): Promise { + const keyboardResponse = await params.invoke({ + ...params.baseReq, + command: 'keyboard', + positionals: ['enter'], + }); + if (keyboardResponse.ok) return keyboardResponse; + + return await params.invoke({ + ...params.baseReq, + command: 'type', + positionals: ['\n'], + }); +} + +async function invokeMaestroAssertNotVisible(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; +}): Promise { + const [selector] = params.positionals; + if (!selector) { + return errorResponse('INVALID_ARGS', 'assertNotVisible requires a selector.'); + } + const response = await params.invoke({ + ...params.baseReq, + command: 'is', + positionals: ['visible', selector], + flags: { ...params.baseReq.flags, noRecord: true }, + }); + if (!response.ok) { + return { ok: true, data: { pass: true, selector, absent: true } }; + } + if (response.data?.pass === false) { + return { ok: true, data: { pass: true, selector } }; + } + return errorResponse('COMMAND_FAILED', `Expected not visible but matched: ${selector}`); +} + +async function invokeMaestroWaitForAnimationToEnd(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; +}): Promise { + const timeoutMs = Number(params.positionals[0] ?? 15000); + if (!Number.isFinite(timeoutMs) || timeoutMs < 0) { + return errorResponse('INVALID_ARGS', 'waitForAnimationToEnd timeout must be a number.'); + } + const startedAt = Date.now(); + let previousSignature: string | undefined; + let lastResponse: DaemonResponse | undefined; + + while (Date.now() - startedAt < timeoutMs) { + const response = await params.invoke({ + ...params.baseReq, + command: 'snapshot', + positionals: [], + flags: { + ...params.baseReq.flags, + noRecord: true, + snapshotRaw: true, + snapshotForceFull: true, + }, + }); + if (!response.ok) { + lastResponse = response; + await sleep(MAESTRO_ANIMATION_POLL_MS); + continue; + } + const snapshot = readSnapshotState(response.data); + if (!snapshot) return response; + const signature = snapshotStabilitySignature(snapshot); + if (previousSignature === signature) { + return { ok: true, data: { stable: true, timeoutMs } }; + } + previousSignature = signature; + lastResponse = response; + await sleep(MAESTRO_ANIMATION_POLL_MS); + } + + return lastResponse?.ok === false + ? lastResponse + : { ok: true, data: { stable: false, timeoutMs } }; +} + async function invokeMaestroScrollUntilVisible( params: MaestroScrollUntilVisibleParams, ): Promise { @@ -76,17 +192,17 @@ async function invokeMaestroScrollUntilVisible( } const fuzzyTextQuery = extractMaestroVisibleTextQuery(selector); const attempts = Math.max(1, Math.ceil(timeoutMs / MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS)); - let lastWaitResponse: DaemonResponse | undefined; + let lastWaitResponse: FailedDaemonResponse | null = null; for (let index = 0; index < attempts; index += 1) { - const probe = await probeMaestroScrollVisibility( + const probeResponse = await probeMaestroScrollVisibility( params, selector, fuzzyTextQuery, scrollProbeMs(timeoutMs, index), ); - if (probe.visible) return probe.response; - lastWaitResponse = probe.response; + if (probeResponse.ok) return probeResponse; + lastWaitResponse = probeResponse; if (index === attempts - 1) break; @@ -106,21 +222,20 @@ async function probeMaestroScrollVisibility( selector: string, fuzzyTextQuery: string | null, probeMs: number, -): Promise<{ visible: boolean; response: DaemonResponse }> { +): Promise { const waitResponse = await params.invoke({ ...params.baseReq, command: 'wait', positionals: [selector, String(probeMs)], }); - if (waitResponse.ok) return { visible: true, response: waitResponse }; - if (!fuzzyTextQuery) return { visible: false, response: waitResponse }; + if (waitResponse.ok || !fuzzyTextQuery) return waitResponse; const fuzzyResponse = await params.invoke({ ...params.baseReq, command: 'find', positionals: [fuzzyTextQuery, 'wait', String(probeMs)], }); - return { visible: fuzzyResponse.ok, response: fuzzyResponse }; + return fuzzyResponse; } function scrollProbeMs(timeoutMs: number, index: number): number { @@ -192,34 +307,54 @@ function readSnapshotState(data: unknown): SnapshotState | undefined { return undefined; } +function snapshotStabilitySignature(snapshot: SnapshotState): string { + return JSON.stringify( + snapshot.nodes.map((node) => ({ + index: node.index, + parentIndex: node.parentIndex, + type: node.type, + identifier: node.identifier, + label: node.label, + value: node.value, + rect: node.rect + ? { + x: Math.round(node.rect.x), + y: Math.round(node.rect.y), + width: Math.round(node.rect.width), + height: Math.round(node.rect.height), + } + : undefined, + })), + ); +} + async function invokeMaestroTapOn(params: MaestroTapOnParams): Promise { - const [selector] = params.positionals; + const [selector, rawOptions] = params.positionals; if (!selector) { return errorResponse('INVALID_ARGS', 'tapOn requires a selector.'); } + const options = readMaestroTapOnOptions(rawOptions); + if (!options.ok) return options.response; const startedAt = Date.now(); const fuzzyTextQuery = extractMaestroVisibleTextQuery(selector); + const timeoutMs = + params.baseReq.flags?.maestro?.optional === true + ? MAESTRO_OPTIONAL_TAP_ON_TIMEOUT_MS + : MAESTRO_TAP_ON_TIMEOUT_MS; let lastResponse: DaemonResponse | undefined; - while (Date.now() - startedAt < MAESTRO_TAP_ON_TIMEOUT_MS) { + while (Date.now() - startedAt < timeoutMs) { + const attempt = await invokeMaestroSnapshotTapOn(params, selector, options.value ?? {}); + if (attempt.ok) return attempt; + lastResponse = attempt; if (fuzzyTextQuery) { - const attempt = await invokeMaestroFuzzyTapOn(params, fuzzyTextQuery); - if (!attempt.retry) return attempt.response; - lastResponse = attempt.response; - await sleep(MAESTRO_TAP_ON_RETRY_MS); - continue; + const fuzzyAttempt = await invokeMaestroFuzzyTapOn(params, fuzzyTextQuery); + if (!fuzzyAttempt.retry) return fuzzyAttempt.response; + lastResponse = fuzzyAttempt.response; } - - const clickResponse = await params.invoke({ - ...params.baseReq, - command: 'click', - positionals: [selector], - }); - if (clickResponse.ok) return clickResponse; - lastResponse = clickResponse; await sleep(MAESTRO_TAP_ON_RETRY_MS); } - if (params.baseReq.flags?.maestroOptional === true) { + if (params.baseReq.flags?.maestro?.optional === true) { return { ok: true, data: { skipped: true, optional: true, selector } }; } return ( @@ -227,6 +362,45 @@ async function invokeMaestroTapOn(params: MaestroTapOnParams): Promise { + const target = await resolveMaestroSnapshotTarget(params, selector, options, 'tapOn'); + if (!target.ok) return target.response; + const point = pointForMaestroTapOnTarget(target.target, selector); + return await params.invoke({ + ...params.baseReq, + command: 'click', + positionals: [String(point.x), String(point.y)], + }); +} + +async function invokeMaestroSwipeOn(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; +}): Promise { + const [selector, direction = 'up', durationMs] = params.positionals; + if (!selector) return errorResponse('INVALID_ARGS', 'swipe.label requires a label selector.'); + const target = await resolveMaestroSnapshotTarget(params, selector, {}, 'swipe.label'); + if (!target.ok) return target.response; + const swipe = swipeCoordinatesFromTarget(target.target, direction); + if (!swipe.ok) return swipe.response; + return await params.invoke({ + ...params.baseReq, + command: 'swipe', + positionals: [ + String(swipe.start.x), + String(swipe.start.y), + String(swipe.end.x), + String(swipe.end.y), + ...(durationMs ? [durationMs] : []), + ], + }); +} + async function invokeMaestroFuzzyTapOn( params: MaestroTapOnParams, query: string, @@ -235,18 +409,350 @@ async function invokeMaestroFuzzyTapOn( ...params.baseReq, command: 'find', positionals: [query, 'click'], + flags: { + ...params.baseReq.flags, + findFirst: true, + }, }); if (findResponse.ok) return { retry: false, response: findResponse }; - if (params.baseReq.flags?.maestroOptional !== true) { - return { retry: true, response: findResponse }; - } + return { retry: true, response: findResponse }; +} - const nativeLabelResponse = await params.invoke({ +async function resolveMaestroSnapshotTarget( + params: { + baseReq: ReplayBaseRequest; + invoke: MaestroRuntimeInvoke; + }, + selector: string, + options: MaestroTapOnOptions, + commandLabel: string, +): Promise<{ ok: true; target: MaestroSnapshotTarget } | { ok: false; response: DaemonResponse }> { + const snapshotResponse = await params.invoke({ ...params.baseReq, - command: 'click', - positionals: [simpleLabelSelector(query)], + command: 'snapshot', + positionals: [], + flags: { + ...params.baseReq.flags, + noRecord: true, + snapshotRaw: true, + snapshotForceFull: true, + }, }); - return { retry: !nativeLabelResponse.ok, response: nativeLabelResponse }; + if (!snapshotResponse.ok) return { ok: false, response: snapshotResponse }; + + const snapshot = readSnapshotState(snapshotResponse.data); + if (!snapshot) { + return { + ok: false, + response: errorResponse( + 'COMMAND_FAILED', + `Unable to read snapshot data for ${commandLabel}.`, + ), + }; + } + + const frame = getSnapshotReferenceFrame(snapshot); + const resolution = resolveMaestroNodeFromSnapshot( + snapshot, + selector, + options, + readMaestroSelectorPlatform(params.baseReq.flags), + frame, + ); + if (!resolution.ok) { + return { + ok: false, + response: errorResponse('ELEMENT_NOT_FOUND', resolution.message), + }; + } + return { + ok: true, + target: { + node: resolution.node, + rect: resolution.rect, + frame, + }, + }; +} + +function resolveMaestroNodeFromSnapshot( + snapshot: SnapshotState, + selector: string, + options: MaestroTapOnOptions, + platform: Platform, + frame: TouchReferenceFrame | undefined, +): { ok: true; node: SnapshotNode; rect: Rect } | { ok: false; message: string } { + let matches = findMaestroSelectorMatches(snapshot, selector, platform); + if (options.childOf) { + const parents = findMaestroSelectorMatches(snapshot, options.childOf, platform); + if (parents.length === 0) { + return { ok: false, message: `Maestro childOf parent did not match: ${options.childOf}` }; + } + matches = matches.filter((node) => + parents.some((parent) => isDescendantOfSnapshotNode(snapshot.nodes, node, parent)), + ); + } + + const target = selectMaestroSnapshotMatch( + snapshot.nodes, + matches, + options.index, + extractMaestroVisibleTextQuery(selector) !== null, + frame, + ); + if (!target) { + const index = options.index ?? 0; + return { + ok: false, + message: `Maestro selector did not match index ${index}: ${selector}`, + }; + } + return { ok: true, node: target.node, rect: target.rect }; +} + +function findMaestroSelectorMatches( + snapshot: SnapshotState, + selectorExpression: string, + platform: Platform, +): SnapshotNode[] { + const chain = parseSelectorChain(selectorExpression); + for (const selector of chain.selectors) { + const matches = snapshot.nodes.filter((node) => matchesSelector(node, selector, platform)); + if (matches.length > 0) return matches; + } + return []; +} + +function resolveNodeRect(nodes: SnapshotState['nodes'], node: SnapshotNode): Rect | null { + if (node.rect && node.rect.width > 0 && node.rect.height > 0) return node.rect; + let current: SnapshotNode | undefined = node; + const byIndex = new Map(nodes.map((candidate) => [candidate.index, candidate])); + while (typeof current.parentIndex === 'number') { + current = byIndex.get(current.parentIndex) ?? nodes[current.parentIndex]; + if (!current) return null; + if (current.rect && current.rect.width > 0 && current.rect.height > 0) return current.rect; + } + return null; +} + +function selectMaestroSnapshotMatch( + nodes: SnapshotState['nodes'], + matches: SnapshotNode[], + index: number | undefined, + preferOnScreen: boolean, + frame: TouchReferenceFrame | undefined, +): { node: SnapshotNode; rect: Rect } | null { + const resolved = matches + .map((node) => { + const rect = resolveNodeRect(nodes, node); + return rect ? { node, rect } : null; + }) + .filter((candidate): candidate is { node: SnapshotNode; rect: Rect } => Boolean(candidate)); + const candidates = + preferOnScreen && index === undefined ? preferOnScreenMatches(resolved, frame) : resolved; + if (index !== undefined) return candidates[index] ?? null; + return candidates.sort(compareMaestroSnapshotMatches)[0] ?? null; +} + +function preferOnScreenMatches( + matches: { node: SnapshotNode; rect: Rect }[], + frame: TouchReferenceFrame | undefined, +): { node: SnapshotNode; rect: Rect }[] { + const onScreen = matches.filter((match) => isRectOnScreen(match.rect, frame)); + return onScreen.length > 0 ? onScreen : matches; +} + +function isRectOnScreen(rect: Rect, frame: TouchReferenceFrame | undefined): boolean { + const maxX = frame?.referenceWidth ?? Number.POSITIVE_INFINITY; + const maxY = frame?.referenceHeight ?? Number.POSITIVE_INFINITY; + return rect.x < maxX && rect.y < maxY && rect.x + rect.width > 0 && rect.y + rect.height > 0; +} + +function compareMaestroSnapshotMatches( + left: { node: SnapshotNode; rect: Rect }, + right: { node: SnapshotNode; rect: Rect }, +): number { + const typeRank = maestroTapTargetTypeRank(left.node) - maestroTapTargetTypeRank(right.node); + if (typeRank !== 0) return typeRank; + + const areaRank = left.rect.width * left.rect.height - right.rect.width * right.rect.height; + if (areaRank !== 0) return areaRank; + + return (right.node.depth ?? 0) - (left.node.depth ?? 0); +} + +function maestroTapTargetTypeRank(node: SnapshotNode): number { + switch (node.type?.toLowerCase()) { + case 'button': + case 'link': + case 'textfield': + case 'textview': + case 'searchfield': + case 'switch': + case 'slider': + return 0; + case 'cell': + return 1; + case 'statictext': + return 2; + default: + return 3; + } +} + +function isDescendantOfSnapshotNode( + nodes: SnapshotState['nodes'], + node: SnapshotNode, + ancestor: SnapshotNode, +): boolean { + let current: SnapshotNode | undefined = node; + const byIndex = new Map(nodes.map((candidate) => [candidate.index, candidate])); + while (typeof current.parentIndex === 'number') { + current = byIndex.get(current.parentIndex) ?? nodes[current.parentIndex]; + if (!current) return false; + if (current === ancestor || current.index === ancestor.index) return true; + } + return false; +} + +function readMaestroTapOnOptions( + rawOptions: string | undefined, +): { ok: true; value: MaestroTapOnOptions | null } | { ok: false; response: DaemonResponse } { + if (!rawOptions) return { ok: true, value: null }; + try { + const value = JSON.parse(rawOptions) as MaestroTapOnOptions; + return { ok: true, value }; + } catch { + return { + ok: false, + response: errorResponse('INVALID_ARGS', 'tapOn runtime options must be valid JSON.'), + }; + } +} + +function readMaestroSelectorPlatform(flags: ReplayBaseRequest['flags']): Platform { + return flags?.platform === 'android' ? 'android' : 'ios'; +} + +function swipeCoordinatesFromTarget( + target: MaestroSnapshotTarget, + direction: string, +): + | { ok: true; start: { x: number; y: number }; end: { x: number; y: number } } + | { ok: false; response: DaemonResponse } { + const center = pointInsideRect(target.rect); + const frame = target.frame; + const horizontalDistance = swipeDistance(frame?.referenceWidth, target.rect.width); + const verticalDistance = swipeDistance(frame?.referenceHeight, target.rect.height); + const minX = 8; + const minY = 8; + const maxX = frame ? frame.referenceWidth - 8 : center.x + horizontalDistance; + const maxY = frame ? frame.referenceHeight - 8 : center.y + verticalDistance; + switch (direction.toLowerCase()) { + case 'up': + return { + ok: true, + start: center, + end: { x: center.x, y: clampCoordinate(center.y - verticalDistance, minY, maxY) }, + }; + case 'down': + return { + ok: true, + start: center, + end: { x: center.x, y: clampCoordinate(center.y + verticalDistance, minY, maxY) }, + }; + case 'left': + return { + ok: true, + start: center, + end: { x: clampCoordinate(center.x - horizontalDistance, minX, maxX), y: center.y }, + }; + case 'right': + return { + ok: true, + start: center, + end: { x: clampCoordinate(center.x + horizontalDistance, minX, maxX), y: center.y }, + }; + default: + return { + ok: false, + response: errorResponse( + 'INVALID_ARGS', + 'swipe.label direction must be up, down, left, or right.', + ), + }; + } +} + +function swipeDistance(frameSize: number | undefined, rectSize: number): number { + const screenRelative = typeof frameSize === 'number' ? frameSize * 0.35 : 0; + return Math.round(Math.min(360, Math.max(120, screenRelative, rectSize * 1.5))); +} + +function clampCoordinate(value: number, min: number, max: number): number { + return Math.round(Math.min(max, Math.max(min, value))); +} + +function pointInsideRect(rect: Rect): { x: number; y: number } { + return { + x: interiorCoordinate(rect.x, rect.width), + y: interiorCoordinate(rect.y, rect.height), + }; +} + +function pointForMaestroTapOnTarget( + target: MaestroSnapshotTarget, + selector: string, +): { x: number; y: number } { + if (!shouldBiasMaestroVisibleTextTap(target.node, selector, target.rect)) { + return pointInsideRect(target.rect); + } + return { + x: interiorCoordinate(target.rect.x, Math.min(target.rect.width, 168)), + y: interiorCoordinate(target.rect.y, Math.min(target.rect.height, 48)), + }; +} + +function shouldBiasMaestroVisibleTextTap( + node: SnapshotNode, + selector: string, + rect: Rect, +): boolean { + if (!extractMaestroVisibleTextQuery(selector)) return false; + if (rect.height < 70 || rect.width < 120) return false; + const type = node.type?.toLowerCase(); + return type === 'cell' || type === 'other' || type === 'scrollview'; +} + +function interiorCoordinate(origin: number, size: number): number { + if (size <= 1) return Math.floor(origin); + const min = Math.ceil(origin); + const max = Math.floor(origin + size - 1); + return clampCoordinate(origin + size / 2, min, max); +} + +function invokeMaestroRunScript(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + scope: ReplayVarScope; +}): DaemonResponse { + const [scriptPath] = params.positionals; + if (!scriptPath) { + return errorResponse('INVALID_ARGS', 'runScript requires a file path.'); + } + try { + const outputEnv = executeRunScriptFile({ + scriptPath, + env: { + ...params.scope.values, + ...(params.baseReq.flags?.maestro?.runScriptEnv ?? {}), + }, + }); + return { ok: true, data: { outputEnv } }; + } catch (error) { + const appError = asAppError(error); + return errorResponse(appError.code, appError.message, appError.details); + } } async function invokeMaestroRunFlowWhen(params: { @@ -324,40 +830,11 @@ async function invokeMaestroRunFlowWhenSteps( function isMaestroWhenConditionMiss(response: DaemonResponse): boolean { if (response.ok) return response.data?.pass === false; - if (response.error.code !== 'COMMAND_FAILED') return false; - return response.error.details?.blockedBy !== 'android_foreground_surface'; -} - -async function invokeMaestroPressEnter(params: { - baseReq: ReplayBaseRequest; - invoke: (req: DaemonRequest) => Promise; -}): Promise { - const response = await params.invoke({ - ...params.baseReq, - command: 'type', - positionals: ['\n'], - }); - if (response.ok) return response; - const message = response.error.message.toLowerCase(); - if (!message.includes('fetch failed')) return response; - - // Maestro compatibility: some iOS apps submit on Enter and immediately reset - // the runner transport. Treat this as recovered only after a fresh snapshot - // proves the runner connection is usable again; it does not assert UI state. - const snapshotResponse = await params.invoke({ - ...params.baseReq, - command: 'snapshot', - positionals: [], - flags: { ...params.baseReq.flags, noRecord: true }, - }); - if (!snapshotResponse.ok) return response; - return { - ok: true, - data: { - recovered: true, - warning: 'Enter key submit reset the iOS runner transport; recovered after snapshot.', - }, - }; + const details = response.error.details; + return ( + details?.command === 'is' && + (details.reason === 'selector_not_found' || details.reason === 'predicate_failed') + ); } function batchStepToSessionAction( @@ -389,16 +866,12 @@ function extractMaestroVisibleTextQuery(selectorExpression: string): string | nu return first; } -function simpleLabelSelector(value: string): string { - return `label=${JSON.stringify(value)}`; -} - function withMaestroScrollTimeoutContext( - response: DaemonResponse | undefined, + response: FailedDaemonResponse | null, selector: string, timeoutMs: number, ): DaemonResponse { - if (!response || response.ok) { + if (!response) { return errorResponse( 'COMMAND_FAILED', `scrollUntilVisible timed out after ${timeoutMs}ms for selector: ${selector}`, diff --git a/src/daemon/handlers/session-replay-runtime.ts b/src/daemon/handlers/session-replay-runtime.ts index b2774457d..9b830fee0 100644 --- a/src/daemon/handlers/session-replay-runtime.ts +++ b/src/daemon/handlers/session-replay-runtime.ts @@ -8,17 +8,14 @@ import { SessionStore } from '../session-store.ts'; import { type ReplayScriptMetadata, writeReplayScript } from '../../replay/script.ts'; import { healReplayAction } from './session-replay-heal.ts'; import { formatScriptActionSummary } from '../../replay/script-utils.ts'; -import { mergeParentFlags } from './handler-utils.ts'; import { errorResponse } from './response.ts'; -import { invokeMaestroRuntimeCommand } from './session-replay-maestro-runtime.ts'; +import { invokeReplayAction } from './session-replay-action-runtime.ts'; import { buildReplayVarScope, collectReplayShellEnv, parseReplayCliEnvEntries, readReplayCliEnvEntries, readReplayShellEnvSource, - resolveReplayAction, - type ReplayVarScope, } from '../../replay/vars.ts'; // fallow-ignore-next-line complexity @@ -158,87 +155,6 @@ export async function runReplayScriptFile(params: { } } -async function invokeReplayAction(params: { - req: DaemonRequest; - sessionName: string; - action: SessionAction; - scope: ReplayVarScope; - filePath: string; - line: number; - step: number; - tracePath?: string; - invoke: (req: DaemonRequest) => Promise; -}): Promise { - const { req, sessionName, action, scope, filePath, line, step, tracePath, invoke } = params; - const resolved = resolveReplayAction(action, scope, { file: filePath, line }); - const startedAt = Date.now(); - appendReplayTraceEvent(tracePath, { - type: 'replay_action_start', - ts: new Date(startedAt).toISOString(), - replayPath: filePath, - line, - step, - command: resolved.command, - positionals: resolved.positionals ?? [], - }); - const flags = buildReplayActionFlags(req.flags, resolved.flags); - const baseReq = { - token: req.token, - session: sessionName, - flags, - runtime: resolved.runtime, - meta: req.meta, - }; - const response = - (await invokeMaestroRuntimeCommand({ - command: resolved.command, - baseReq, - positionals: resolved.positionals ?? [], - batchSteps: resolved.flags?.batchSteps, - line, - step, - invoke, - invokeReplayAction: async (nested) => - await invokeReplayAction({ - req, - sessionName, - action: nested.action, - scope, - filePath, - line: nested.line, - step: nested.step, - tracePath, - invoke, - }), - })) ?? - (await invoke({ - ...baseReq, - command: resolved.command, - positionals: resolved.positionals ?? [], - })); - const finishedAt = Date.now(); - appendReplayTraceEvent(tracePath, { - type: 'replay_action_stop', - ts: new Date(finishedAt).toISOString(), - replayPath: filePath, - line, - step, - command: resolved.command, - ok: response.ok, - durationMs: finishedAt - startedAt, - errorCode: response.ok ? undefined : response.error.code, - }); - return response; -} - -function appendReplayTraceEvent( - tracePath: string | undefined, - event: Record, -): void { - if (!tracePath) return; - fs.appendFileSync(tracePath, `${JSON.stringify(event)}\n`); -} - // fallow-ignore-next-line complexity function buildReplayBuiltinVars(params: { req: DaemonRequest; @@ -340,29 +256,21 @@ function isReplayArtifactPath(candidate: string): boolean { } } -export function buildReplayActionFlags( - parentFlags: CommandFlags | undefined, - actionFlags: SessionAction['flags'] | undefined, -): CommandFlags { - return mergeParentFlags(parentFlags, { ...(actionFlags ?? {}) }); -} - // fallow-ignore-next-line complexity function actionsContainInterpolation(actions: SessionAction[]): boolean { for (const action of actions) { for (const positional of action.positionals ?? []) { if (typeof positional === 'string' && positional.includes('${')) return true; } - if (action.flags) { - for (const value of Object.values(action.flags)) { - if (typeof value === 'string' && value.includes('${')) return true; - } - } - if (action.runtime) { - for (const value of Object.values(action.runtime)) { - if (typeof value === 'string' && value.includes('${')) return true; - } - } + if (containsInterpolation(action.flags)) return true; + if (containsInterpolation(action.runtime)) return true; } return false; } + +function containsInterpolation(value: unknown): boolean { + if (typeof value === 'string') return value.includes('${'); + if (Array.isArray(value)) return value.some(containsInterpolation); + if (value && typeof value === 'object') return Object.values(value).some(containsInterpolation); + return false; +} diff --git a/src/daemon/handlers/session.ts b/src/daemon/handlers/session.ts index 571c09f0b..08c0c05c3 100644 --- a/src/daemon/handlers/session.ts +++ b/src/daemon/handlers/session.ts @@ -203,13 +203,15 @@ export async function handleSessionCommands(params: { if (req.command === PUBLIC_COMMANDS.keyboard) { const session = sessionStore.get(sessionName); const keyboardAction = req.positionals?.[0]?.trim().toLowerCase(); - if (!session && keyboardAction === 'dismiss') { + const needsForegroundIosApp = + keyboardAction === 'dismiss' || keyboardAction === 'enter' || keyboardAction === 'return'; + if (!session && needsForegroundIosApp) { const flags = req.flags ?? {}; const normalizedPlatform = normalizePlatformSelector(flags.platform); if (normalizedPlatform === 'ios') { return errorResponse( 'SESSION_NOT_FOUND', - 'iOS keyboard dismiss requires an active session so the target app stays foregrounded. Run open first.', + 'iOS keyboard action requires an active session so the target app stays foregrounded. Run open first.', ); } } diff --git a/src/platforms/android/input-actions.ts b/src/platforms/android/input-actions.ts index 45c750521..4a43fd173 100644 --- a/src/platforms/android/input-actions.ts +++ b/src/platforms/android/input-actions.ts @@ -47,6 +47,10 @@ export async function homeAndroid(device: DeviceInfo): Promise { await runAndroidAdb(device, ['shell', 'input', 'keyevent', '3']); } +export async function pressAndroidEnter(device: DeviceInfo): Promise { + await runAndroidAdb(device, ['shell', 'input', 'keyevent', 'ENTER']); +} + export async function rotateAndroid( device: DeviceInfo, orientation: DeviceRotation, diff --git a/src/platforms/ios/__tests__/runner-client.test.ts b/src/platforms/ios/__tests__/runner-client.test.ts index cdcfeb02b..00ba17ad6 100644 --- a/src/platforms/ios/__tests__/runner-client.test.ts +++ b/src/platforms/ios/__tests__/runner-client.test.ts @@ -139,6 +139,7 @@ const runnerProtocolCommandFixtures: Record { + return await runIosRunnerCommand( + device, + { + command: 'type', + selectorKey: selector.key, + selectorValue: selector.value, + allowNonHittableSelectorTap: selector.allowNonHittableTap, + text, + delayMs, + textEntryMode: 'replace', + appBundleId: ctx.appBundleId, + }, + runnerOpts, + ); + }, fill: async (x, y, text, delayMs) => { return await runIosRunnerCommand( device, diff --git a/src/platforms/ios/runner-contract.ts b/src/platforms/ios/runner-contract.ts index e87b6ccf9..a7149a8e7 100644 --- a/src/platforms/ios/runner-contract.ts +++ b/src/platforms/ios/runner-contract.ts @@ -34,6 +34,7 @@ export type RunnerCommand = { | 'transformGesture' | 'appSwitcher' | 'keyboardDismiss' + | 'keyboardReturn' | 'alert' | 'pinch' | 'recordStart' diff --git a/src/replay/vars.ts b/src/replay/vars.ts index dc3fb7d54..969bca5d0 100644 --- a/src/replay/vars.ts +++ b/src/replay/vars.ts @@ -2,7 +2,7 @@ import { AppError } from '../utils/errors.ts'; import type { SessionAction } from '../daemon/types.ts'; export type ReplayVarScope = { - readonly values: Readonly>; + readonly values: Record; }; export type ReplayVarSources = { @@ -13,7 +13,7 @@ export type ReplayVarSources = { }; export const REPLAY_VAR_KEY_RE = /^[A-Z_][A-Z0-9_]*$/; -const INTERPOLATION_RE = /(\\\$\{)|\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-((?:[^}\\]|\\.)*))?\}/g; +const INTERPOLATION_RE = /(\\\$\{)|\$\{([A-Za-z_][A-Za-z0-9_.]*)(?::-((?:[^}\\]|\\.)*))?\}/g; const SHELL_PREFIX = 'AD_VAR_'; const RESERVED_NAMESPACE_PREFIX = 'AD_'; @@ -53,6 +53,13 @@ export function buildReplayVarScope(sources: ReplayVarSources): ReplayVarScope { return { values: merged }; } +export function mergeReplayVarScopeValues( + scope: ReplayVarScope, + values: Record, +): void { + Object.assign(scope.values, values); +} + export function collectReplayShellEnv(processEnv: NodeJS.ProcessEnv): Record { const result: Record = {}; for (const [rawKey, value] of Object.entries(processEnv)) { @@ -156,11 +163,20 @@ function resolveStringProps( loc: { file: string; line: number }, ): T | undefined { if (!obj) return obj; - const next: Record = { ...(obj as Record) }; - for (const [key, value] of Object.entries(next)) { - if (typeof value === 'string') { - next[key] = resolveReplayString(value, scope, loc); - } + return resolveStringValue(obj, scope, loc) as T; +} + +function resolveStringValue( + value: unknown, + scope: ReplayVarScope, + loc: { file: string; line: number }, +): unknown { + if (typeof value === 'string') return resolveReplayString(value, scope, loc); + if (Array.isArray(value)) return value.map((entry) => resolveStringValue(entry, scope, loc)); + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value).map(([key, entry]) => [key, resolveStringValue(entry, scope, loc)]), + ); } - return next as T; + return value; } diff --git a/src/utils/__tests__/args.test.ts b/src/utils/__tests__/args.test.ts index baab1af9d..b8a1e6cc2 100644 --- a/src/utils/__tests__/args.test.ts +++ b/src/utils/__tests__/args.test.ts @@ -120,13 +120,14 @@ test('parseArgs recognizes command-specific flag combinations', async () => { }, { label: 'replay maestro flow', - argv: ['replay', './flow.yaml', '--maestro', '--env', 'USER=Ada'], + argv: ['replay', './flow.yaml', '--maestro', '--env', 'USER=Ada', '--timeout', '240000'], strictFlags: true, assertParsed: (parsed) => { assert.equal(parsed.command, 'replay'); assert.deepEqual(parsed.positionals, ['./flow.yaml']); assert.equal(parsed.flags.replayMaestro, true); assert.deepEqual(parsed.flags.replayEnv, ['USER=Ada']); + assert.equal(parsed.flags.timeoutMs, 240000); }, }, ]; @@ -369,6 +370,10 @@ test('parseArgs accepts keyboard subcommands', () => { const dismiss = parseArgs(['keyboard', 'dismiss'], { strictFlags: true }); assert.equal(dismiss.command, 'keyboard'); assert.deepEqual(dismiss.positionals, ['dismiss']); + + const enter = parseArgs(['keyboard', 'enter'], { strictFlags: true }); + assert.equal(enter.command, 'keyboard'); + assert.deepEqual(enter.positionals, ['enter']); }); test('parseArgs accepts scroll pixel distance flag', () => { @@ -918,6 +923,7 @@ test('usageForCommand includes Maestro replay flag', () => { assert.match(help, /doubleTapOn/); assert.match(help, /pasteText/); assert.match(help, /runFlow file\/inline/); + assert.match(help, /ordered trusted runScript/); assert.match(help, /repeat\.times/); assert.match(help, /stopApp/); assert.match(help, /Unsupported syntax fails loudly/); @@ -1416,8 +1422,11 @@ test('clipboard command usage is documented', () => { test('keyboard command usage is documented', () => { const help = usageForCommand('keyboard'); if (help === null) throw new Error('Expected command help text'); - assert.match(help, /keyboard \[status\|get\|dismiss\]/); - assert.match(help, /Inspect Android keyboard visibility\/type or dismiss the device keyboard/); + assert.match(help, /keyboard \[status\|get\|dismiss\|enter\|return\]/); + assert.match( + help, + /Inspect Android keyboard visibility\/type or press\/dismiss the device keyboard/, + ); }); test('rotate command usage is documented', () => { diff --git a/src/utils/command-schema.ts b/src/utils/command-schema.ts index f0bfd1993..d29fa4985 100644 --- a/src/utils/command-schema.ts +++ b/src/utils/command-schema.ts @@ -1261,7 +1261,7 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ type: 'boolean', usageLabel: '--maestro', usageDescription: - 'Replay: treat input as a Maestro YAML compatibility flow. Supported subset: launchApp with Apple-platform launch arguments and iOS simulator clearState, runFlow file/inline with when.platform/visible/notVisible, runScript file/env with parse-time http.post/json/output variables, onFlowStart/onFlowComplete, deterministic repeat.times, tapOn including optional and absolute/percentage point taps, doubleTapOn, longPressOn, inputText, pasteText, openLink, assertVisible, assertNotVisible, extendedWaitUntil, scroll, scrollUntilVisible, absolute/percentage swipe, takeScreenshot, hideKeyboard, pressKey back/enter/home, back, waitForAnimationToEnd, and stopApp. ' + + 'Replay: treat input as a Maestro YAML compatibility flow. Supported subset: launchApp with Apple-platform launch arguments and iOS simulator clearState, runFlow file/inline with when.platform/visible/notVisible, ordered trusted runScript file/env steps with http.post/json/output variables, onFlowStart/onFlowComplete, deterministic repeat.times, tapOn including optional, index, childOf, label, and absolute/percentage point taps, doubleTapOn, longPressOn, inputText, eraseText for focused fields, pasteText, openLink, assertVisible, assertNotVisible, extendedWaitUntil, scroll, scrollUntilVisible, absolute/percentage swipe plus swipe.label, takeScreenshot, hideKeyboard, pressKey back/enter/home, back, waitForAnimationToEnd, and stopApp. ' + 'Unsupported syntax fails loudly with a link to https://github.com/callstackincubator/agent-device/issues/558', }, { @@ -1286,7 +1286,7 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ type: 'int', min: 1, usageLabel: '--timeout ', - usageDescription: 'Test: maximum wall-clock time per script attempt', + usageDescription: 'Replay/Test: maximum wall-clock time per script attempt', }, { key: 'retries', @@ -1597,9 +1597,10 @@ const COMMAND_SCHEMAS: Record = { allowedFlags: [], }, keyboard: { - usageOverride: 'keyboard [status|get|dismiss]', - helpDescription: 'Inspect Android keyboard visibility/type or dismiss the device keyboard', - summary: 'Inspect or dismiss the device keyboard', + usageOverride: 'keyboard [status|get|dismiss|enter|return]', + helpDescription: + 'Inspect Android keyboard visibility/type or press/dismiss the device keyboard', + summary: 'Inspect, press, or dismiss the device keyboard', positionalArgs: ['action?'], allowedFlags: [], }, @@ -1674,7 +1675,7 @@ const COMMAND_SCHEMAS: Record = { replay: { helpDescription: 'Replay a recorded session', positionalArgs: ['path'], - allowedFlags: ['replayUpdate', 'replayMaestro', 'replayEnv'], + allowedFlags: ['replayUpdate', 'replayMaestro', 'replayEnv', 'timeoutMs'], skipCapabilityCheck: true, }, test: { diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index 7846bf503..a42384dcd 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -61,11 +61,11 @@ Maestro compatibility translates supported YAML commands into Agent Device repla - Supported and unsupported capabilities: https://github.com/callstackincubator/agent-device/issues/558 - New focused compatibility request: https://github.com/callstackincubator/agent-device/issues/new -Currently supported areas include app launch with Apple-platform launch arguments and iOS simulator `clearState`, file and inline `runFlow` with `when.platform`, `when.visible`, and `when.notVisible`, `onFlowStart` / `onFlowComplete`, deterministic `repeat.times`, `tapOn` including `optional` and absolute/percentage point taps, `doubleTapOn`, `longPressOn`, `inputText`, `pasteText`, `openLink`, visibility assertions, `extendedWaitUntil`, `scroll`, `scrollUntilVisible`, absolute/percentage `swipe`, screenshots, keyboard dismiss, basic `pressKey`, `back`, animation waits, and `stopApp`. `runScript` is supported only as a Maestro compatibility feature for file/env scripts that use `http.post`, `json`, and `output` variables; it executes during flow parsing, can make network requests, and is not a native `.ad` command or security sandbox. +Currently supported areas include app launch with Apple-platform launch arguments and iOS simulator `clearState`, file and inline `runFlow` with `when.platform`, `when.visible`, and `when.notVisible`, `onFlowStart` / `onFlowComplete`, deterministic `repeat.times`, `tapOn` including `optional`, `index`, `childOf`, `label`, and absolute/percentage point taps, `doubleTapOn`, `longPressOn`, `inputText`, focused-field `eraseText`, `pasteText`, `openLink`, visibility assertions, `extendedWaitUntil`, `scroll`, `scrollUntilVisible`, absolute/percentage `swipe`, `swipe.label`, screenshots, keyboard dismiss, basic `pressKey`, `back`, animation waits, and `stopApp`. `runScript` is supported only as an ordered Maestro compatibility step for trusted file/env scripts that use `http.post`, `json`, and `output` variables; it can make network requests, and is not a native `.ad` command or security sandbox. Script execution uses Node `vm` only for compatibility isolation, not for security; the script timeout bounds synchronous execution, while `http.post` requests are bounded by the helper process timeout. Output keys cannot contain `.` because exported variables are addressed as `output.`. Maestro `env` values use the same replay precedence as `.ad` files: flow `env` is the default, shell `AD_VAR_*` values override it, and CLI `-e KEY=VALUE` wins over both. -Unsupported Maestro features such as `repeat.while`, `runFlow.when.true`, full expression predicates, `evalScript`, text clearing, selector relations such as `index` / `childOf`, device utility commands, Android app launch arguments, and Android app state reset are tracked separately because they require neutral Agent Device runtime or device capabilities before they can be mapped safely. +Unsupported Maestro features such as `repeat.while`, `runFlow.when.true`, full expression predicates, `evalScript`, device utility commands, Android app launch arguments, and Android app state reset are tracked separately because they require neutral Agent Device runtime or device capabilities before they can be mapped safely. ## Run a lightweight `.ad` suite From d7077aea4467c3d4fff6f96712aa0c3152684359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 26 May 2026 09:55:45 +0200 Subject: [PATCH 5/6] refactor: tighten Maestro replay compatibility boundary --- .../RunnerTests+CommandExecution.swift | 4 +- .../RunnerTests+Models.swift | 2 +- .../maestro/__tests__/replay-flow.test.ts | 12 +-- src/compat/maestro/device-actions.ts | 2 +- src/compat/maestro/interactions.ts | 2 +- src/compat/maestro/replay-flow.ts | 2 +- src/core/dispatch-context.ts | 6 +- src/core/dispatch.ts | 4 +- src/core/interactor-types.ts | 2 +- src/daemon/__tests__/context.test.ts | 4 +- src/daemon/context.ts | 2 +- src/daemon/direct-ios-selector.ts | 2 +- .../handlers/__tests__/interaction.test.ts | 6 +- src/daemon/handlers/interaction-touch.ts | 8 +- .../session-replay-maestro-runtime.ts | 81 ++++++++++++++----- src/platforms/ios/apps.ts | 2 +- src/platforms/ios/interactions.ts | 4 +- src/platforms/ios/runner-contract.ts | 2 +- 18 files changed, 94 insertions(+), 53 deletions(-) diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 5556202a3..0965ab9ce 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -256,7 +256,7 @@ extension RunnerTests { app: activeApp, selectorKey: selectorKey, selectorValue: selectorValue, - allowNonHittableFallback: command.allowNonHittableSelectorTap == true + allowNonHittableFallback: command.allowNonHittableCoordinateFallback == true ) if match.isAmbiguous { return Response(ok: false, error: ErrorPayload(code: "AMBIGUOUS_MATCH", message: "selector matched multiple elements")) @@ -877,7 +877,7 @@ extension RunnerTests { app: activeApp, selectorKey: selectorKey, selectorValue: selectorValue, - allowNonHittableFallback: command.allowNonHittableSelectorTap == true + allowNonHittableFallback: command.allowNonHittableCoordinateFallback == true ) if match.isAmbiguous { return Response(ok: false, error: ErrorPayload(code: "AMBIGUOUS_MATCH", message: "selector matched multiple elements")) diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index 0020cc81f..5e60b4906 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -40,7 +40,7 @@ struct Command: Codable { let text: String? let selectorKey: String? let selectorValue: String? - let allowNonHittableSelectorTap: Bool? + let allowNonHittableCoordinateFallback: Bool? let delayMs: Int? let textEntryMode: String? let clearFirst: Bool? diff --git a/src/compat/maestro/__tests__/replay-flow.test.ts b/src/compat/maestro/__tests__/replay-flow.test.ts index 27db41a1b..30153fb6b 100644 --- a/src/compat/maestro/__tests__/replay-flow.test.ts +++ b/src/compat/maestro/__tests__/replay-flow.test.ts @@ -80,8 +80,8 @@ env: assert.equal(parsed.actions[3]?.flags.doubleTap, true); assert.equal(parsed.actions[3]?.flags.intervalMs, 150); assert.equal(parsed.actions[4]?.flags.holdMs, 3000); - assert.equal(parsed.actions[1]?.flags.maestro?.allowNonHittableSelectorTap, true); - assert.equal(parsed.actions[6]?.flags?.maestro?.allowNonHittableSelectorTap, undefined); + assert.equal(parsed.actions[1]?.flags.maestro?.allowNonHittableCoordinateFallback, true); + assert.equal(parsed.actions[6]?.flags?.maestro?.allowNonHittableCoordinateFallback, undefined); }); test('parseMaestroReplayFlow maps iOS openLink through the app id when available', () => { @@ -200,7 +200,7 @@ test('parseMaestroReplayFlow marks tapOn before inputText for snapshot tap focus ['type', ['Muted Users']], ], ); - assert.equal(parsed.actions[0]?.flags?.maestro?.allowNonHittableSelectorTap, undefined); + assert.equal(parsed.actions[0]?.flags?.maestro?.allowNonHittableCoordinateFallback, undefined); }); test('parseMaestroReplayFlow coalesces tapOn inputText while preserving pressKey Enter submit', () => { @@ -221,7 +221,7 @@ test('parseMaestroReplayFlow coalesces tapOn inputText while preserving pressKey ], ); assert.deepEqual(parsed.actionLines, [3, 3, 6]); - assert.equal(parsed.actions[1]?.flags?.maestro?.allowNonHittableSelectorTap, true); + assert.equal(parsed.actions[1]?.flags?.maestro?.allowNonHittableCoordinateFallback, true); }); test('parseMaestroReplayFlow rejects relative runScript paths without source path', () => { @@ -430,7 +430,7 @@ test('parseMaestroReplayFlow keeps visible-gated runFlow commands for runtime ev { command: '__maestroTapOn', positionals: ['label="Continue" || text="Continue" || id="Continue"'], - flags: { maestro: { allowNonHittableSelectorTap: true } }, + flags: { maestro: { allowNonHittableCoordinateFallback: true } }, }, ]); }); @@ -454,7 +454,7 @@ test('parseMaestroReplayFlow accepts launchApp reset options', () => { 'open', ['com.callstack.agentdevicelab'], { - maestro: { clearState: true }, + clearAppState: true, launchArgs: ['-EXDevMenuIsOnboardingFinished', 'true', '-Example', 'ignored'], }, ], diff --git a/src/compat/maestro/device-actions.ts b/src/compat/maestro/device-actions.ts index b873acab2..433629b0e 100644 --- a/src/compat/maestro/device-actions.ts +++ b/src/compat/maestro/device-actions.ts @@ -42,7 +42,7 @@ export function convertLaunchApp( const shouldRelaunch = !shouldClearState && (value.stopApp === true || launchArgs.length > 0); return action('open', [appId], { ...(shouldRelaunch ? { relaunch: true } : {}), - ...(shouldClearState ? { maestro: { clearState: true } } : {}), + ...(shouldClearState ? { clearAppState: true } : {}), ...(launchArgs.length > 0 ? { launchArgs } : {}), }); } diff --git a/src/compat/maestro/interactions.ts b/src/compat/maestro/interactions.ts index eeb9f9695..6d18ff55f 100644 --- a/src/compat/maestro/interactions.ts +++ b/src/compat/maestro/interactions.ts @@ -357,7 +357,7 @@ function maestroTapOnFlags(value: unknown): SessionAction['flags'] { ...flags, maestro: { ...(flags.maestro ?? {}), - allowNonHittableSelectorTap: true, + allowNonHittableCoordinateFallback: true, }, }; } diff --git a/src/compat/maestro/replay-flow.ts b/src/compat/maestro/replay-flow.ts index c1b0f669c..a6dd4fa86 100644 --- a/src/compat/maestro/replay-flow.ts +++ b/src/compat/maestro/replay-flow.ts @@ -126,7 +126,7 @@ function optimizeInputTextActions( function clearMaestroNonHittableTap(action: SessionAction): SessionAction { const maestro = { ...(action.flags?.maestro ?? {}) }; - delete maestro.allowNonHittableSelectorTap; + delete maestro.allowNonHittableCoordinateFallback; return { ...action, flags: { diff --git a/src/core/dispatch-context.ts b/src/core/dispatch-context.ts index bb8952881..27b5b2f59 100644 --- a/src/core/dispatch-context.ts +++ b/src/core/dispatch-context.ts @@ -11,14 +11,14 @@ export type BatchStep = { }; export type MaestroRuntimeFlags = { - allowNonHittableSelectorTap?: boolean; - clearState?: boolean; + allowNonHittableCoordinateFallback?: boolean; optional?: boolean; runScriptEnv?: Record; }; export type CommandFlags = Omit & { batchSteps?: BatchStep[]; + clearAppState?: boolean; launchArgs?: string[]; maestro?: MaestroRuntimeFlags; replayBackend?: string; @@ -55,6 +55,6 @@ export type DispatchContext = ScreenshotDispatchFlags & { key: 'id' | 'label' | 'text' | 'value'; value: string; raw: string; - allowNonHittableTap?: boolean; + allowNonHittableCoordinateFallback?: boolean; }; }; diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index 61a57b554..507086c7b 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -236,13 +236,13 @@ async function handleOpenCommand( if (isDeepLinkTarget(app)) { throw new AppError( 'INVALID_ARGS', - 'Maestro launchApp.clearState requires an app target, not a deep link.', + 'Clearing app state requires an app target, not a deep link.', ); } if (device.platform !== 'ios' || device.kind !== 'simulator') { throw new AppError( 'UNSUPPORTED_OPERATION', - 'Maestro launchApp.clearState is currently supported only on iOS simulators.', + 'Clearing app state is currently supported only on iOS simulators.', ); } await clearIosSimulatorAppState(device, app); diff --git a/src/core/interactor-types.ts b/src/core/interactor-types.ts index 6b5abfa36..2ddf377a8 100644 --- a/src/core/interactor-types.ts +++ b/src/core/interactor-types.ts @@ -29,7 +29,7 @@ export type ScreenshotOptions = { export type ElementSelectorTapOptions = { key: 'id' | 'label' | 'text' | 'value'; value: string; - allowNonHittableTap?: boolean; + allowNonHittableCoordinateFallback?: boolean; }; export type SnapshotOptions = BaseSnapshotOptions & { diff --git a/src/daemon/__tests__/context.test.ts b/src/daemon/__tests__/context.test.ts index 35559226d..92c1b6339 100644 --- a/src/daemon/__tests__/context.test.ts +++ b/src/daemon/__tests__/context.test.ts @@ -14,8 +14,8 @@ test('contextFromFlags forwards scroll pixels from CLI flags', () => { assert.equal(context.pixels, 240); }); -test('contextFromFlags maps Maestro clearState to generic app-state clearing', () => { - const flags: CommandFlags = { maestro: { clearState: true } }; +test('contextFromFlags forwards generic app-state clearing', () => { + const flags: CommandFlags = { clearAppState: true }; const context = contextFromFlags('/tmp/agent-device.log', flags); assert.equal(context.clearAppState, true); }); diff --git a/src/daemon/context.ts b/src/daemon/context.ts index 8234a521b..7947241d0 100644 --- a/src/daemon/context.ts +++ b/src/daemon/context.ts @@ -25,7 +25,7 @@ export function contextFromFlags( activity: flags?.activity, launchConsole: flags?.launchConsole, launchArgs: flags?.launchArgs, - clearAppState: flags?.maestro?.clearState, + clearAppState: flags?.clearAppState, verbose: flags?.verbose, logPath, traceLogPath, diff --git a/src/daemon/direct-ios-selector.ts b/src/daemon/direct-ios-selector.ts index aa0e82058..60dfe075f 100644 --- a/src/daemon/direct-ios-selector.ts +++ b/src/daemon/direct-ios-selector.ts @@ -6,7 +6,7 @@ export type DirectIosSelectorTarget = { key: 'id' | 'label' | 'text' | 'value'; value: string; raw: string; - allowNonHittableTap?: boolean; + allowNonHittableCoordinateFallback?: boolean; }; export function readSimpleIosSelectorTarget(params: { diff --git a/src/daemon/handlers/__tests__/interaction.test.ts b/src/daemon/handlers/__tests__/interaction.test.ts index 2801ca469..0fdbf8dea 100644 --- a/src/daemon/handlers/__tests__/interaction.test.ts +++ b/src/daemon/handlers/__tests__/interaction.test.ts @@ -453,7 +453,7 @@ test('fill simple iOS id selector uses direct runner selector fill without snaps } }); -test('click simple iOS selector forwards Maestro non-hittable tap backdoor', async () => { +test('click simple iOS selector forwards Maestro non-hittable coordinate fallback', async () => { const sessionStore = makeSessionStore(); const sessionName = 'ios-maestro-selector-fallback'; sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); @@ -472,7 +472,7 @@ test('click simple iOS selector forwards Maestro non-hittable tap backdoor', asy session: sessionName, command: 'click', positionals: ['id="e2eSignInAlice"'], - flags: { maestro: { allowNonHittableSelectorTap: true } }, + flags: { maestro: { allowNonHittableCoordinateFallback: true } }, }, sessionName, sessionStore, @@ -486,7 +486,7 @@ test('click simple iOS selector forwards Maestro non-hittable tap backdoor', asy key: 'id', value: 'e2eSignInAlice', raw: 'id="e2eSignInAlice"', - allowNonHittableTap: true, + allowNonHittableCoordinateFallback: true, }); }); diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/handlers/interaction-touch.ts index 82f4ee072..3aab53354 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/handlers/interaction-touch.ts @@ -261,7 +261,9 @@ function readDirectIosSelectorTapTarget(params: { if (!selector) return null; return { ...selector, - ...(flags?.maestro?.allowNonHittableSelectorTap ? { allowNonHittableTap: true } : {}), + ...(flags?.maestro?.allowNonHittableCoordinateFallback + ? { allowNonHittableCoordinateFallback: true } + : {}), }; } @@ -437,7 +439,9 @@ function readDirectIosSelectorFillTarget(params: { if (!selector) return null; return { ...selector, - ...(flags?.maestro?.allowNonHittableSelectorTap ? { allowNonHittableTap: true } : {}), + ...(flags?.maestro?.allowNonHittableCoordinateFallback + ? { allowNonHittableCoordinateFallback: true } + : {}), }; } diff --git a/src/daemon/handlers/session-replay-maestro-runtime.ts b/src/daemon/handlers/session-replay-maestro-runtime.ts index 00923591d..6d24c063e 100644 --- a/src/daemon/handlers/session-replay-maestro-runtime.ts +++ b/src/daemon/handlers/session-replay-maestro-runtime.ts @@ -12,11 +12,27 @@ import { getSnapshotReferenceFrame, type TouchReferenceFrame } from '../touch-re import type { DaemonRequest, DaemonResponse, SessionAction } from '../types.ts'; import { errorResponse } from './response.ts'; -const MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS = 500; -const MAESTRO_TAP_ON_TIMEOUT_MS = 30000; -const MAESTRO_OPTIONAL_TAP_ON_TIMEOUT_MS = 3000; -const MAESTRO_TAP_ON_RETRY_MS = 250; -const MAESTRO_ANIMATION_POLL_MS = 250; +// Keep Maestro timing and target-selection heuristics behind one policy so +// generic Agent Device command behavior does not inherit compatibility rules. +const MAESTRO_REPLAY_POLICY = { + animationPollMs: 250, + scrollUntilVisibleProbeMs: 500, + tapOnRetryMs: 250, + tapOnTimeoutMs: 30000, + optionalTapOnTimeoutMs: 3000, + swipe: { + screenRatio: 0.35, + minDistancePx: 120, + maxDistancePx: 360, + marginPx: 8, + }, + largeTextContainerBias: { + minWidth: 120, + minHeight: 70, + width: 168, + height: 48, + }, +} as const; type ReplayBaseRequest = Omit; @@ -160,7 +176,7 @@ async function invokeMaestroWaitForAnimationToEnd(params: { }); if (!response.ok) { lastResponse = response; - await sleep(MAESTRO_ANIMATION_POLL_MS); + await sleep(MAESTRO_REPLAY_POLICY.animationPollMs); continue; } const snapshot = readSnapshotState(response.data); @@ -171,7 +187,7 @@ async function invokeMaestroWaitForAnimationToEnd(params: { } previousSignature = signature; lastResponse = response; - await sleep(MAESTRO_ANIMATION_POLL_MS); + await sleep(MAESTRO_REPLAY_POLICY.animationPollMs); } return lastResponse?.ok === false @@ -191,7 +207,10 @@ async function invokeMaestroScrollUntilVisible( return errorResponse('INVALID_ARGS', 'scrollUntilVisible timeout must be a positive number.'); } const fuzzyTextQuery = extractMaestroVisibleTextQuery(selector); - const attempts = Math.max(1, Math.ceil(timeoutMs / MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS)); + const attempts = Math.max( + 1, + Math.ceil(timeoutMs / MAESTRO_REPLAY_POLICY.scrollUntilVisibleProbeMs), + ); let lastWaitResponse: FailedDaemonResponse | null = null; for (let index = 0; index < attempts; index += 1) { @@ -240,8 +259,8 @@ async function probeMaestroScrollVisibility( function scrollProbeMs(timeoutMs: number, index: number): number { return Math.min( - MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS, - Math.max(1, timeoutMs - index * MAESTRO_SCROLL_UNTIL_VISIBLE_PROBE_MS), + MAESTRO_REPLAY_POLICY.scrollUntilVisibleProbeMs, + Math.max(1, timeoutMs - index * MAESTRO_REPLAY_POLICY.scrollUntilVisibleProbeMs), ); } @@ -339,8 +358,8 @@ async function invokeMaestroTapOn(params: MaestroTapOnParams): Promise Date: Tue, 26 May 2026 10:09:16 +0200 Subject: [PATCH 6/6] refactor: reduce Maestro replay quality debt --- src/commands/selector-read.ts | 31 +- src/commands/system.ts | 77 ++-- src/compat/maestro/interactions.ts | 63 +++- src/compat/maestro/replay-flow.ts | 71 ++-- src/core/dispatch.ts | 353 ++++++++++-------- src/daemon/handlers/interaction-touch.ts | 105 +++--- .../session-replay-maestro-runtime.ts | 178 +++++---- 7 files changed, 510 insertions(+), 368 deletions(-) diff --git a/src/commands/selector-read.ts b/src/commands/selector-read.ts index 526ec7d39..0940189d6 100644 --- a/src/commands/selector-read.ts +++ b/src/commands/selector-read.ts @@ -166,13 +166,7 @@ export const findCommand: RuntimeCommand { + const capture = await captureSelectorSnapshot(runtime, options, { + updateSession: true, + scope: shouldScopeFind(locator) ? options.query : undefined, + }); + const match = findBestMatchesByLocator(capture.snapshot.nodes, locator, options.query, { + requireRect: false, + }).matches[0]; + return { capture, match }; +} + async function waitForSelector( runtime: AgentDeviceRuntime, options: WaitCommandOptions, diff --git a/src/commands/system.ts b/src/commands/system.ts index 20d234b90..a8cbc5116 100644 --- a/src/commands/system.ts +++ b/src/commands/system.ts @@ -207,13 +207,7 @@ export const keyboardCommand: RuntimeCommand< throw new AppError('UNSUPPORTED_OPERATION', 'system.keyboard is not supported by this backend'); } const action = options.action ?? 'status'; - if ( - action !== 'status' && - action !== 'get' && - action !== 'dismiss' && - action !== 'enter' && - action !== 'return' - ) { + if (!isKeyboardAction(action)) { throw new AppError( 'INVALID_ARGS', 'system.keyboard action must be status, get, dismiss, enter, or return', @@ -221,11 +215,12 @@ export const keyboardCommand: RuntimeCommand< } const state = await runtime.backend.setKeyboard(toBackendContext(runtime, options), { action }); const formattedBackendResult = toBackendResult(state); + const keyboardState = isKeyboardResult(state) ? state : {}; if (action === 'enter' || action === 'return') { return { kind: 'keyboardEnterPressed', action: 'enter', - state: isKeyboardResult(state) ? state : {}, + state: keyboardState, ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), ...successText('Keyboard enter pressed'), }; @@ -235,7 +230,7 @@ export const keyboardCommand: RuntimeCommand< return { kind: 'keyboardDismissed', action, - state: isKeyboardResult(state) ? state : {}, + state: keyboardState, ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), ...successText(dismissed === false ? 'Keyboard already hidden' : 'Keyboard dismissed'), }; @@ -243,7 +238,7 @@ export const keyboardCommand: RuntimeCommand< return { kind: 'keyboardState', action, - state: isKeyboardResult(state) ? state : {}, + state: keyboardState, ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), }; }; @@ -373,25 +368,41 @@ function normalizeAlertResult( action: BackendAlertAction, result: BackendAlertResult, ): SystemAlertCommandResult { - if (action === 'get') { - if (result.kind !== 'alertStatus') { - throw new AppError('COMMAND_FAILED', 'system.alert get returned an invalid backend result'); - } - return { kind: 'alertStatus', action, alert: result.alert }; + switch (action) { + case 'get': + return normalizeAlertStatusResult(result); + case 'wait': + return normalizeAlertWaitResult(result); + default: + return normalizeAlertHandledResult(action, result); } - if (action === 'wait') { - if (result.kind !== 'alertWait') { - throw new AppError('COMMAND_FAILED', 'system.alert wait returned an invalid backend result'); - } - return { - kind: 'alertWait', - action, - alert: result.alert, - ...(result.waitedMs !== undefined ? { waitedMs: result.waitedMs } : {}), - ...(result.timedOut !== undefined ? { timedOut: result.timedOut } : {}), - ...successText(result.alert ? 'Alert visible' : 'Alert wait timed out'), - }; +} + +function normalizeAlertStatusResult(result: BackendAlertResult): SystemAlertCommandResult { + if (result.kind !== 'alertStatus') { + throw new AppError('COMMAND_FAILED', 'system.alert get returned an invalid backend result'); + } + return { kind: 'alertStatus', action: 'get', alert: result.alert }; +} + +function normalizeAlertWaitResult(result: BackendAlertResult): SystemAlertCommandResult { + if (result.kind !== 'alertWait') { + throw new AppError('COMMAND_FAILED', 'system.alert wait returned an invalid backend result'); } + return { + kind: 'alertWait', + action: 'wait', + alert: result.alert, + ...(result.waitedMs !== undefined ? { waitedMs: result.waitedMs } : {}), + ...(result.timedOut !== undefined ? { timedOut: result.timedOut } : {}), + ...successText(result.alert ? 'Alert visible' : 'Alert wait timed out'), + }; +} + +function normalizeAlertHandledResult( + action: Extract, + result: BackendAlertResult, +): SystemAlertCommandResult { if (result.kind !== 'alertHandled') { throw new AppError( 'COMMAND_FAILED', @@ -408,6 +419,18 @@ function normalizeAlertResult( }; } +function isKeyboardAction( + action: string, +): action is 'status' | 'get' | 'dismiss' | 'enter' | 'return' { + return ( + action === 'status' || + action === 'get' || + action === 'dismiss' || + action === 'enter' || + action === 'return' + ); +} + function isKeyboardResult(value: unknown): value is BackendKeyboardResult { return Boolean(value && typeof value === 'object'); } diff --git a/src/compat/maestro/interactions.ts b/src/compat/maestro/interactions.ts index 6d18ff55f..897ec824c 100644 --- a/src/compat/maestro/interactions.ts +++ b/src/compat/maestro/interactions.ts @@ -189,27 +189,56 @@ export function convertSwipe(value: unknown, context: MaestroParseContext): Sess assertOnlyKeys(value, 'swipe', ['start', 'end', 'direction', 'duration', 'from', 'label']); const from = value.from ?? (typeof value.label === 'string' ? value.label : undefined); if (from !== undefined) { - const direction = readSwipeDirection( - typeof value.direction === 'string' ? value.direction : 'up', - ); - return action(MAESTRO_RUNTIME_COMMAND.swipeOn, [ - maestroSelector(from, 'swipe.from', [], context), - direction, - ...swipeDurationPositionals(value), - ]); + return convertTargetedSwipe(value, from, context); } if (typeof value.direction === 'string') { return action('scroll', readScrollPositionalsFromDirectionSwipe(value.direction)); } - if (typeof value.start !== 'string' || typeof value.end !== 'string') { - throw unsupportedMaestroSyntax('Only Maestro swipe start/end coordinates are supported.'); - } - const start = parseMaestroPoint(value.start); - const end = parseMaestroPoint(value.end); - const durationMs = - typeof value.duration === 'number' && Number.isFinite(value.duration) - ? String(Math.max(16, Math.floor(value.duration))) - : undefined; + return convertCoordinateSwipe(value); +} + +function convertTargetedSwipe( + value: Record, + from: unknown, + context: MaestroParseContext, +): SessionAction { + const direction = readSwipeDirection( + typeof value.direction === 'string' ? value.direction : 'up', + ); + return action(MAESTRO_RUNTIME_COMMAND.swipeOn, [ + maestroSelector(from, 'swipe.from', [], context), + direction, + ...swipeDurationPositionals(value), + ]); +} + +function convertCoordinateSwipe(value: Record): SessionAction { + const { start, end } = readCoordinateSwipePoints(value); + const durationMs = readSwipeDurationMs(value.duration); + return convertCoordinateSwipePoints(start, end, durationMs); +} + +function readCoordinateSwipePoints(value: Record): { + start: ReturnType; + end: ReturnType; +} { + if (typeof value.start === 'string' && typeof value.end === 'string') { + return { start: parseMaestroPoint(value.start), end: parseMaestroPoint(value.end) }; + } + throw unsupportedMaestroSyntax('Only Maestro swipe start/end coordinates are supported.'); +} + +function readSwipeDurationMs(duration: unknown): string | undefined { + return typeof duration === 'number' && Number.isFinite(duration) + ? String(Math.max(16, Math.floor(duration))) + : undefined; +} + +function convertCoordinateSwipePoints( + start: ReturnType, + end: ReturnType, + durationMs: string | undefined, +): SessionAction { if (start.kind === 'absolute' && end.kind === 'absolute') { return action('swipe', [ String(start.x), diff --git a/src/compat/maestro/replay-flow.ts b/src/compat/maestro/replay-flow.ts index a6dd4fa86..eab80c650 100644 --- a/src/compat/maestro/replay-flow.ts +++ b/src/compat/maestro/replay-flow.ts @@ -82,41 +82,16 @@ function optimizeInputTextActions( actions: SessionAction[], actionLines: number[], ): { actions: SessionAction[]; actionLines: number[] } { - const maestroTapTimeoutMs = '30000'; const mergedActions: SessionAction[] = []; const mergedLines: number[] = []; for (let index = 0; index < actions.length; index += 1) { const action = actions[index]; - const nextAction = actions[index + 1]; - const typedAfterTap = readPlainTypeText(nextAction); - if (typedAfterTap !== null) { - const tapSelector = readPlainMaestroTapSelector(action); - const pressEnterAfterType = - actions[index + 2]?.command === MAESTRO_RUNTIME_COMMAND.pressEnter; - if (tapSelector !== null && pressEnterAfterType) { - mergedActions.push({ - ...action, - command: 'wait', - positionals: [tapSelector, maestroTapTimeoutMs], - }); - mergedLines.push(actionLines[index] ?? 1); - mergedActions.push({ - ...nextAction, - command: 'fill', - positionals: [tapSelector, typedAfterTap], - flags: action.flags, - }); - mergedLines.push(actionLines[index] ?? 1); - mergedActions.push(actions[index + 2] as SessionAction); - mergedLines.push(actionLines[index + 2] ?? actionLines[index] ?? 1); - index += 2; - continue; - } - if (tapSelector !== null) { - mergedActions.push(clearMaestroNonHittableTap(action)); - mergedLines.push(actionLines[index] ?? 1); - continue; - } + const optimized = optimizeTypedAfterTap(actions, actionLines, index); + if (optimized) { + mergedActions.push(...optimized.actions); + mergedLines.push(...optimized.actionLines); + index += optimized.consumed - 1; + continue; } mergedActions.push(action); mergedLines.push(actionLines[index] ?? 1); @@ -124,6 +99,40 @@ function optimizeInputTextActions( return { actions: mergedActions, actionLines: mergedLines }; } +function optimizeTypedAfterTap( + actions: SessionAction[], + actionLines: number[], + index: number, +): { actions: SessionAction[]; actionLines: number[]; consumed: number } | null { + const action = actions[index]; + const nextAction = actions[index + 1]; + const typedAfterTap = readPlainTypeText(nextAction); + const tapSelector = readPlainMaestroTapSelector(action); + if (typedAfterTap === null || tapSelector === null) return null; + const line = actionLines[index] ?? 1; + if (actions[index + 2]?.command !== MAESTRO_RUNTIME_COMMAND.pressEnter) { + return { actions: [clearMaestroNonHittableTap(action)], actionLines: [line], consumed: 1 }; + } + return { + actions: [ + { + ...action, + command: 'wait', + positionals: [tapSelector, '30000'], + }, + { + ...nextAction, + command: 'fill', + positionals: [tapSelector, typedAfterTap], + flags: action.flags, + }, + actions[index + 2] as SessionAction, + ], + actionLines: [line, line, actionLines[index + 2] ?? line], + consumed: 3, + }; +} + function clearMaestroNonHittableTap(action: SessionAction): SessionAction { const maestro = { ...(action.flags?.maestro ?? {}) }; delete maestro.allowNonHittableCoordinateFallback; diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index 507086c7b..05888e718 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -44,7 +44,105 @@ import { parseDeviceRotation } from './device-rotation.ts'; export { resolveTargetDevice } from './dispatch-resolve.ts'; export type { BatchStep, CommandFlags, DispatchContext } from './dispatch-context.ts'; -// fallow-ignore-next-line complexity +type DispatchCommandHandlerParams = { + device: DeviceInfo; + interactor: Interactor; + positionals: string[]; + outPath?: string; + context?: DispatchContext; + runnerCtx: RunnerContext; +}; + +type DispatchCommandHandler = ( + params: DispatchCommandHandlerParams, +) => Promise | void> | Record | void; + +const DISPATCH_COMMAND_HANDLERS: Record = { + open: ({ device, interactor, positionals, context }) => + handleOpenCommand(device, interactor, positionals, context), + close: async ({ interactor, positionals }) => { + const app = positionals[0]; + if (!app) { + return { closed: 'session', ...successText('Closed session') }; + } + await interactor.close(app); + return { app, ...successText(`Closed: ${app}`) }; + }, + press: ({ device, interactor, positionals, context }) => + handlePressCommand(device, interactor, positionals, context), + swipe: ({ device, interactor, positionals, context }) => + handleSwipeCommand(device, interactor, positionals, context), + pan: ({ interactor, positionals }) => handlePanCommand(interactor, positionals), + fling: ({ interactor, positionals }) => handleFlingCommand(interactor, positionals), + longpress: ({ interactor, positionals }) => handleLongPressCommand(interactor, positionals), + focus: ({ interactor, positionals }) => handleFocusCommand(interactor, positionals), + type: ({ interactor, positionals, context }) => + handleTypeCommand(interactor, positionals, context), + fill: ({ interactor, positionals, context }) => + handleFillCommand(interactor, positionals, context), + scroll: ({ interactor, positionals, context }) => + handleScrollCommand(interactor, positionals, context), + pinch: ({ device, interactor, positionals, context }) => + handlePinchCommand(device, interactor, positionals, context), + 'rotate-gesture': ({ device, interactor, positionals }) => + handleRotateGestureCommand(device, interactor, positionals), + 'transform-gesture': ({ device, interactor, positionals }) => + handleTransformGestureCommand(device, interactor, positionals), + 'trigger-app-event': async ({ device, interactor, positionals, context }) => { + const { eventName, payload } = parseTriggerAppEventArgs(positionals); + const eventUrl = resolveAppEventUrl(device.platform, eventName, payload); + await interactor.open(eventUrl, { appBundleId: context?.appBundleId }); + return { + event: eventName, + eventUrl, + transport: 'deep-link', + ...successText(`Triggered app event: ${eventName}`), + }; + }, + screenshot: async ({ interactor, positionals, outPath, context }) => { + const positionalPath = positionals[0]; + const screenshotPath = positionalPath ?? outPath ?? `./screenshot-${Date.now()}.png`; + await fs.mkdir(pathModule.dirname(screenshotPath), { recursive: true }); + const screenshotOptions = screenshotOptionsFromFlags(context); + await interactor.screenshot(screenshotPath, { + appBundleId: context?.appBundleId, + fullscreen: screenshotOptions.fullscreen, + stabilize: screenshotOptions.stabilize, + surface: context?.surface, + }); + return { path: screenshotPath, ...successText(`Saved screenshot: ${screenshotPath}`) }; + }, + back: async ({ interactor, context }) => { + await interactor.back(context?.backMode); + return { action: 'back', mode: context?.backMode ?? 'in-app', ...successText('Back') }; + }, + home: async ({ interactor }) => { + await interactor.home(); + return { action: 'home', ...successText('Home') }; + }, + rotate: async ({ interactor, positionals }) => { + const orientation = parseDeviceRotation(positionals[0]); + await interactor.rotate(orientation); + return { + action: 'rotate', + orientation, + ...successText(`Rotated to ${orientation}`), + }; + }, + 'app-switcher': async ({ interactor }) => { + await interactor.appSwitcher(); + return { action: 'app-switcher', ...successText('Opened app switcher') }; + }, + clipboard: ({ interactor, positionals }) => handleClipboardCommand(interactor, positionals), + keyboard: ({ device, positionals, context, runnerCtx }) => + handleKeyboardCommand(device, positionals, context, runnerCtx), + settings: ({ device, interactor, positionals, context }) => + handleSettingsCommand(device, interactor, positionals, context), + push: ({ device, positionals, context }) => handlePushCommand(device, positionals, context), + snapshot: ({ interactor, context }) => handleSnapshotCommand(interactor, context), + read: ({ device, positionals, context }) => handleReadCommand(device, positionals, context), +}; + export async function dispatchCommand( device: DeviceInfo, command: string, @@ -72,98 +170,9 @@ export async function dispatchCommand( return await withDiagnosticTimer( 'platform_command', async () => { - switch (command) { - case 'open': - return handleOpenCommand(device, interactor, positionals, context); - case 'close': { - const app = positionals[0]; - if (!app) { - return { closed: 'session', ...successText('Closed session') }; - } - await interactor.close(app); - return { app, ...successText(`Closed: ${app}`) }; - } - case 'press': - return handlePressCommand(device, interactor, positionals, context); - case 'swipe': - return handleSwipeCommand(device, interactor, positionals, context); - case 'pan': - return handlePanCommand(interactor, positionals); - case 'fling': - return handleFlingCommand(interactor, positionals); - case 'longpress': - return handleLongPressCommand(interactor, positionals); - case 'focus': - return handleFocusCommand(interactor, positionals); - case 'type': - return handleTypeCommand(interactor, positionals, context); - case 'fill': - return handleFillCommand(interactor, positionals, context); - case 'scroll': - return handleScrollCommand(interactor, positionals, context); - case 'pinch': - return handlePinchCommand(device, interactor, positionals, context); - case 'rotate-gesture': - return handleRotateGestureCommand(device, interactor, positionals); - case 'transform-gesture': - return handleTransformGestureCommand(device, interactor, positionals); - case 'trigger-app-event': { - const { eventName, payload } = parseTriggerAppEventArgs(positionals); - const eventUrl = resolveAppEventUrl(device.platform, eventName, payload); - await interactor.open(eventUrl, { appBundleId: context?.appBundleId }); - return { - event: eventName, - eventUrl, - transport: 'deep-link', - ...successText(`Triggered app event: ${eventName}`), - }; - } - case 'screenshot': { - const positionalPath = positionals[0]; - const screenshotPath = positionalPath ?? outPath ?? `./screenshot-${Date.now()}.png`; - await fs.mkdir(pathModule.dirname(screenshotPath), { recursive: true }); - const screenshotOptions = screenshotOptionsFromFlags(context); - await interactor.screenshot(screenshotPath, { - appBundleId: context?.appBundleId, - fullscreen: screenshotOptions.fullscreen, - stabilize: screenshotOptions.stabilize, - surface: context?.surface, - }); - return { path: screenshotPath, ...successText(`Saved screenshot: ${screenshotPath}`) }; - } - case 'back': - await interactor.back(context?.backMode); - return { action: 'back', mode: context?.backMode ?? 'in-app', ...successText('Back') }; - case 'home': - await interactor.home(); - return { action: 'home', ...successText('Home') }; - case 'rotate': { - const orientation = parseDeviceRotation(positionals[0]); - await interactor.rotate(orientation); - return { - action: 'rotate', - orientation, - ...successText(`Rotated to ${orientation}`), - }; - } - case 'app-switcher': - await interactor.appSwitcher(); - return { action: 'app-switcher', ...successText('Opened app switcher') }; - case 'clipboard': - return handleClipboardCommand(interactor, positionals); - case 'keyboard': - return handleKeyboardCommand(device, positionals, context, runnerCtx); - case 'settings': - return handleSettingsCommand(device, interactor, positionals, context); - case 'push': - return handlePushCommand(device, positionals, context); - case 'snapshot': - return await handleSnapshotCommand(interactor, context); - case 'read': - return handleReadCommand(device, positionals, context); - default: - throw new AppError('INVALID_ARGS', `Unknown command: ${command}`); - } + const handler = DISPATCH_COMMAND_HANDLERS[command]; + if (!handler) throw new AppError('INVALID_ARGS', `Unknown command: ${command}`); + return await handler({ device, interactor, positionals, outPath, context, runnerCtx }); }, { command, @@ -290,13 +299,7 @@ async function handleKeyboardCommand( runnerCtx: RunnerContext, ): Promise> { const action = (positionals[0] ?? 'status').toLowerCase(); - if ( - action !== 'status' && - action !== 'get' && - action !== 'dismiss' && - action !== 'enter' && - action !== 'return' - ) { + if (!isKeyboardAction(action)) { throw new AppError( 'INVALID_ARGS', 'keyboard requires a subcommand: status, get, dismiss, enter, or return', @@ -306,80 +309,108 @@ async function handleKeyboardCommand( throw new AppError('INVALID_ARGS', 'keyboard accepts at most one subcommand argument'); } if (device.platform === 'android') { - if (action === 'enter' || action === 'return') { - await pressAndroidEnter(device); - return { - platform: 'android', - action: 'enter', - ...successText('Keyboard enter pressed'), - }; - } - if (action === 'dismiss') { - const result = await dismissAndroidKeyboard(device); - return { - platform: 'android', - action: 'dismiss', - attempts: result.attempts, - wasVisible: result.wasVisible, - dismissed: result.dismissed, - visible: result.visible, - inputType: result.inputType, - type: result.type, - inputMethodPackage: result.inputMethodPackage, - focusedPackage: result.focusedPackage, - focusedResourceId: result.focusedResourceId, - inputOwner: result.inputOwner, - }; - } - const state = await getAndroidKeyboardState(device); + return await handleAndroidKeyboardCommand(device, action); + } + if (device.platform === 'ios') { + return await handleIosKeyboardCommand(device, action, context, runnerCtx); + } + throw new AppError('UNSUPPORTED_OPERATION', 'keyboard is supported only on Android and iOS'); +} + +function isKeyboardAction( + action: string, +): action is 'status' | 'get' | 'dismiss' | 'enter' | 'return' { + return ( + action === 'status' || + action === 'get' || + action === 'dismiss' || + action === 'enter' || + action === 'return' + ); +} + +async function handleAndroidKeyboardCommand( + device: DeviceInfo, + action: 'status' | 'get' | 'dismiss' | 'enter' | 'return', +): Promise> { + if (action === 'enter' || action === 'return') { + await pressAndroidEnter(device); return { platform: 'android', - action: 'status', - visible: state.visible, - inputType: state.inputType, - type: state.type, - inputMethodPackage: state.inputMethodPackage, - focusedPackage: state.focusedPackage, - focusedResourceId: state.focusedResourceId, - inputOwner: state.inputOwner, + action: 'enter', + ...successText('Keyboard enter pressed'), }; } - if (device.platform === 'ios') { - if (action !== 'dismiss' && action !== 'enter' && action !== 'return') { - throw new AppError( - 'UNSUPPORTED_OPERATION', - 'keyboard status/get is currently supported only on Android; use keyboard dismiss or enter on iOS', - ); - } - if (action === 'enter' || action === 'return') { - const result = await runIosRunnerCommand( - device, - { command: 'keyboardReturn', appBundleId: context?.appBundleId }, - runnerCtx, - ); - return { - platform: 'ios', - action: 'enter', - visible: result.visible, - wasVisible: result.wasVisible, - ...successText('Keyboard enter pressed'), - }; - } + if (action === 'dismiss') { + const result = await dismissAndroidKeyboard(device); + return { + platform: 'android', + action: 'dismiss', + attempts: result.attempts, + wasVisible: result.wasVisible, + dismissed: result.dismissed, + visible: result.visible, + inputType: result.inputType, + type: result.type, + inputMethodPackage: result.inputMethodPackage, + focusedPackage: result.focusedPackage, + focusedResourceId: result.focusedResourceId, + inputOwner: result.inputOwner, + }; + } + const state = await getAndroidKeyboardState(device); + return { + platform: 'android', + action: 'status', + visible: state.visible, + inputType: state.inputType, + type: state.type, + inputMethodPackage: state.inputMethodPackage, + focusedPackage: state.focusedPackage, + focusedResourceId: state.focusedResourceId, + inputOwner: state.inputOwner, + }; +} + +async function handleIosKeyboardCommand( + device: DeviceInfo, + action: 'status' | 'get' | 'dismiss' | 'enter' | 'return', + context: DispatchContext | undefined, + runnerCtx: RunnerContext, +): Promise> { + if (action !== 'dismiss' && action !== 'enter' && action !== 'return') { + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'keyboard status/get is currently supported only on Android; use keyboard dismiss or enter on iOS', + ); + } + if (action === 'enter' || action === 'return') { const result = await runIosRunnerCommand( device, - { command: 'keyboardDismiss', appBundleId: context?.appBundleId }, + { command: 'keyboardReturn', appBundleId: context?.appBundleId }, runnerCtx, ); return { platform: 'ios', - action: 'dismiss', - wasVisible: result.wasVisible, - dismissed: result.dismissed, + action: 'enter', visible: result.visible, - ...successText(result.dismissed ? 'Keyboard dismissed' : 'Keyboard already hidden'), + wasVisible: result.wasVisible, + ...successText('Keyboard enter pressed'), }; } - throw new AppError('UNSUPPORTED_OPERATION', 'keyboard is supported only on Android and iOS'); + const result = await runIosRunnerCommand( + device, + { command: 'keyboardDismiss', appBundleId: context?.appBundleId }, + runnerCtx, + ); + return { + platform: 'ios', + action: 'dismiss', + wasVisible: result.wasVisible, + dismissed: result.dismissed, + visible: result.visible, + ...successText(result.dismissed ? 'Keyboard dismissed' : 'Keyboard already hidden'), + }; } async function handleSettingsCommand( diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/handlers/interaction-touch.ts index 3aab53354..064b95d75 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/handlers/interaction-touch.ts @@ -283,11 +283,44 @@ async function dispatchDirectIosSelectorTap( session: SessionState, selector: DirectIosSelectorTarget, ): Promise { + return await dispatchDirectIosSelectorInteraction({ + params, + session, + selector, + command: 'press', + positionals: [], + extra: { selector: selector.raw }, + fallbackPhase: 'ios_direct_selector_tap_fallback', + }); +} + +async function dispatchDirectIosSelectorInteraction(params: { + params: InteractionHandlerParams; + session: SessionState; + selector: DirectIosSelectorTarget; + command: 'press' | 'fill'; + positionals: string[]; + extra: Record; + fallbackPhase: string; +}): Promise { + const { + params: handlerParams, + session, + selector, + command, + positionals, + extra, + fallbackPhase, + } = params; const actionStartedAt = Date.now(); try { const data = - (await dispatchCommand(session.device, 'press', [], params.req.flags?.out, { - ...params.contextFromFlags(params.req.flags, session.appBundleId, session.trace?.outPath), + (await dispatchCommand(session.device, command, positionals, handlerParams.req.flags?.out, { + ...handlerParams.contextFromFlags( + handlerParams.req.flags, + session.appBundleId, + session.trace?.outPath, + ), directElementSelector: selector, surface: session.surface, })) ?? {}; @@ -298,16 +331,14 @@ async function dispatchDirectIosSelectorTap( fallbackX: point.x, fallbackY: point.y, referenceFrame: readReferenceFrameFromDirectSelectorTapResult(data), - extra: { - selector: selector.raw, - }, + extra, }); return finalizeTouchInteraction({ session, - sessionStore: params.sessionStore, - command: params.req.command, - positionals: params.req.positionals ?? [], - flags: params.req.flags, + sessionStore: handlerParams.sessionStore, + command: handlerParams.req.command, + positionals: handlerParams.req.positionals ?? [], + flags: handlerParams.req.flags, result: responseData, responseData, actionStartedAt, @@ -319,7 +350,7 @@ async function dispatchDirectIosSelectorTap( } emitDiagnostic({ level: 'debug', - phase: 'ios_direct_selector_tap_fallback', + phase: fallbackPhase, data: { selector: selector.raw, error: error instanceof Error ? error.message : String(error), @@ -451,51 +482,15 @@ async function dispatchDirectIosSelectorFill( selector: DirectIosSelectorTarget, text: string, ): Promise { - const actionStartedAt = Date.now(); - try { - const data = - (await dispatchCommand(session.device, 'fill', [text], params.req.flags?.out, { - ...params.contextFromFlags(params.req.flags, session.appBundleId, session.trace?.outPath), - directElementSelector: selector, - surface: session.surface, - })) ?? {}; - const actionFinishedAt = Date.now(); - const point = readPointFromDirectSelectorTapResult(data); - const responseData = buildTouchVisualizationResult({ - data, - fallbackX: point.x, - fallbackY: point.y, - referenceFrame: readReferenceFrameFromDirectSelectorTapResult(data), - extra: { - selector: selector.raw, - text, - }, - }); - return finalizeTouchInteraction({ - session, - sessionStore: params.sessionStore, - command: params.req.command, - positionals: params.req.positionals ?? [], - flags: params.req.flags, - result: responseData, - responseData, - actionStartedAt, - actionFinishedAt, - }); - } catch (error) { - if (!isDirectIosSelectorFallbackError(error)) { - return { ok: false, error: normalizeError(error) }; - } - emitDiagnostic({ - level: 'debug', - phase: 'ios_direct_selector_fill_fallback', - data: { - selector: selector.raw, - error: error instanceof Error ? error.message : String(error), - }, - }); - return null; - } + return await dispatchDirectIosSelectorInteraction({ + params, + session, + selector, + command: 'fill', + positionals: [text], + extra: { selector: selector.raw, text }, + fallbackPhase: 'ios_direct_selector_fill_fallback', + }); } async function dispatchRuntimeInteraction< diff --git a/src/daemon/handlers/session-replay-maestro-runtime.ts b/src/daemon/handlers/session-replay-maestro-runtime.ts index 6d24c063e..8ec68ae15 100644 --- a/src/daemon/handlers/session-replay-maestro-runtime.ts +++ b/src/daemon/handlers/session-replay-maestro-runtime.ts @@ -34,6 +34,18 @@ const MAESTRO_REPLAY_POLICY = { }, } as const; +const MAESTRO_TAP_TARGET_TYPE_RANK = new Map([ + ['button', 0], + ['link', 0], + ['textfield', 0], + ['textview', 0], + ['searchfield', 0], + ['switch', 0], + ['slider', 0], + ['cell', 1], + ['statictext', 2], +]); + type ReplayBaseRequest = Omit; type MaestroReplayInvoker = (params: { @@ -163,29 +175,10 @@ async function invokeMaestroWaitForAnimationToEnd(params: { let lastResponse: DaemonResponse | undefined; while (Date.now() - startedAt < timeoutMs) { - const response = await params.invoke({ - ...params.baseReq, - command: 'snapshot', - positionals: [], - flags: { - ...params.baseReq.flags, - noRecord: true, - snapshotRaw: true, - snapshotForceFull: true, - }, - }); - if (!response.ok) { - lastResponse = response; - await sleep(MAESTRO_REPLAY_POLICY.animationPollMs); - continue; - } - const snapshot = readSnapshotState(response.data); - if (!snapshot) return response; - const signature = snapshotStabilitySignature(snapshot); - if (previousSignature === signature) { - return { ok: true, data: { stable: true, timeoutMs } }; - } - previousSignature = signature; + const response = await captureMaestroRawSnapshot(params); + const poll = readAnimationPollResult(response, previousSignature, timeoutMs); + if (poll.done) return poll.response; + previousSignature = poll.signature ?? previousSignature; lastResponse = response; await sleep(MAESTRO_REPLAY_POLICY.animationPollMs); } @@ -195,6 +188,43 @@ async function invokeMaestroWaitForAnimationToEnd(params: { : { ok: true, data: { stable: false, timeoutMs } }; } +function readAnimationPollResult( + response: DaemonResponse, + previousSignature: string | undefined, + timeoutMs: number, +): { done: true; response: DaemonResponse } | { done: false; signature?: string } { + const signature = readSnapshotStabilitySignature(response); + if (!response.ok) return { done: false }; + if (!signature) return { done: true, response }; + if (previousSignature === signature) { + return { done: true, response: { ok: true, data: { stable: true, timeoutMs } } }; + } + return { done: false, signature }; +} + +async function captureMaestroRawSnapshot(params: { + baseReq: ReplayBaseRequest; + invoke: MaestroRuntimeInvoke; +}): Promise { + return await params.invoke({ + ...params.baseReq, + command: 'snapshot', + positionals: [], + flags: { + ...params.baseReq.flags, + noRecord: true, + snapshotRaw: true, + snapshotForceFull: true, + }, + }); +} + +function readSnapshotStabilitySignature(response: DaemonResponse): string | null { + if (!response.ok) return null; + const snapshot = readSnapshotState(response.data); + return snapshot ? snapshotStabilitySignature(snapshot) : null; +} + async function invokeMaestroScrollUntilVisible( params: MaestroScrollUntilVisibleParams, ): Promise { @@ -356,23 +386,34 @@ async function invokeMaestroTapOn(params: MaestroTapOnParams): Promise { + const attempt = await invokeMaestroSnapshotTapOn(params, selector, options); + if (attempt.ok) return { retry: false, response: attempt }; + if (!fuzzyTextQuery) return { retry: true, response: attempt }; + return await invokeMaestroFuzzyTapOn(params, fuzzyTextQuery); +} + async function invokeMaestroSnapshotTapOn( params: MaestroTapOnParams, selector: string, @@ -423,7 +478,9 @@ async function invokeMaestroSwipeOn(params: { async function invokeMaestroFuzzyTapOn( params: MaestroTapOnParams, query: string, -): Promise<{ retry: boolean; response: DaemonResponse }> { +): Promise< + { retry: false; response: DaemonResponse } | { retry: true; response: FailedDaemonResponse } +> { const findResponse = await params.invoke({ ...params.baseReq, command: 'find', @@ -544,14 +601,11 @@ function findMaestroSelectorMatches( function resolveNodeRect(nodes: SnapshotState['nodes'], node: SnapshotNode): Rect | null { if (node.rect && node.rect.width > 0 && node.rect.height > 0) return node.rect; - let current: SnapshotNode | undefined = node; - const byIndex = new Map(nodes.map((candidate) => [candidate.index, candidate])); - while (typeof current.parentIndex === 'number') { - current = byIndex.get(current.parentIndex) ?? nodes[current.parentIndex]; - if (!current) return null; - if (current.rect && current.rect.width > 0 && current.rect.height > 0) return current.rect; - } - return null; + return ( + findSnapshotAncestor(nodes, node, (ancestor) => + ancestor.rect && ancestor.rect.width > 0 && ancestor.rect.height > 0 ? ancestor : null, + )?.rect ?? null + ); } function selectMaestroSnapshotMatch( @@ -601,22 +655,7 @@ function compareMaestroSnapshotMatches( } function maestroTapTargetTypeRank(node: SnapshotNode): number { - switch (node.type?.toLowerCase()) { - case 'button': - case 'link': - case 'textfield': - case 'textview': - case 'searchfield': - case 'switch': - case 'slider': - return 0; - case 'cell': - return 1; - case 'statictext': - return 2; - default: - return 3; - } + return MAESTRO_TAP_TARGET_TYPE_RANK.get(node.type?.toLowerCase() ?? '') ?? 3; } function isDescendantOfSnapshotNode( @@ -624,14 +663,27 @@ function isDescendantOfSnapshotNode( node: SnapshotNode, ancestor: SnapshotNode, ): boolean { + return Boolean( + findSnapshotAncestor(nodes, node, (candidate) => + candidate === ancestor || candidate.index === ancestor.index ? candidate : null, + ), + ); +} + +function findSnapshotAncestor( + nodes: SnapshotState['nodes'], + node: SnapshotNode, + resolve: (ancestor: SnapshotNode) => T | null, +): T | null { let current: SnapshotNode | undefined = node; const byIndex = new Map(nodes.map((candidate) => [candidate.index, candidate])); while (typeof current.parentIndex === 'number') { current = byIndex.get(current.parentIndex) ?? nodes[current.parentIndex]; - if (!current) return false; - if (current === ancestor || current.index === ancestor.index) return true; + if (!current) return null; + const result = resolve(current); + if (result) return result; } - return false; + return null; } function readMaestroTapOnOptions(