From fb39a7039aaccc174e6c6f88364d27f2fa2d41c0 Mon Sep 17 00:00:00 2001 From: timo <44401485+Timo972@users.noreply.github.com> Date: Wed, 10 Jun 2026 19:42:10 +0200 Subject: [PATCH 01/11] feat(screencapture): ReplayKit broadcast extension as high-fps frame source Add a broadcast upload extension (WebDriverAgentBroadcast.appex, embedded into the generated Runner.app) that receives the system's screen frames via ReplayKit, encodes them per capture session (one hardware H264/H265 encoder each, VTPixelTransfer letterbox scaling, 420v end to end) and ships the elementary stream to WDA over loopback TCP :9300. Capture sessions switch to the broadcast source at the extension's first IDR and revert to the XCTest screenshot loop (forced keyframe, no client reconnect) when the broadcast stops, raising achievable stream rates from ~10-20fps to 30-60fps on real devices. - New endpoints: POST /mobilerun/screencapture/broadcast/start (drives RPSystemBroadcastPickerView + system sheet via UI automation), GET .../broadcast (status), POST .../broadcast/stop. Plain /screencapture/start is unchanged; sessions attach automatically and report their active source as "replaykit" or "screenshot". - Shared wire protocol (FBBroadcastProtocol) compiled into both the lib and the appex; the appex cannot link WebDriverAgentLib (XCTest private API is forbidden in extensions) and reuses FBVideoEncoder + GCDAsyncSocket as shared sources instead. - Embedding happens via a scheme build post-action (Scripts/embed-broadcast-extension.sh) because nothing built into the .xctest reaches the auto-generated Runner.app; the script also rewrites the appex bundle id to .broadcast and re-signs inner-first. - Simulator/tvOS: broadcast endpoints return unsupportedOperation; the screenshot pipeline is untouched. Audio capture is a follow-up. Co-Authored-By: Claude Fable 5 --- Scripts/embed-broadcast-extension.sh | 81 ++++ WebDriverAgent.xcodeproj/project.pbxproj | 182 +++++++++ .../WebDriverAgentRunner-nodebug.xcscheme | 18 + .../xcschemes/WebDriverAgentRunner.xcscheme | 16 + .../FBBroadcastSampleHandler.h | 22 ++ .../FBBroadcastSampleHandler.m | 138 +++++++ .../FBExtBroadcastClient.h | 54 +++ .../FBExtBroadcastClient.m | 302 +++++++++++++++ WebDriverAgentBroadcast/FBExtLogging.h | 27 ++ .../FBExtSessionPipeline.h | 65 ++++ .../FBExtSessionPipeline.m | 272 ++++++++++++++ WebDriverAgentBroadcast/Info.plist | 35 ++ .../Commands/FBScreenCaptureCommands.m | 64 ++++ WebDriverAgentLib/Routing/FBWebServer.m | 4 + .../Utilities/FBBroadcastControlServer.h | 84 +++++ .../Utilities/FBBroadcastControlServer.m | 274 ++++++++++++++ .../Utilities/FBBroadcastManager.h | 86 +++++ .../Utilities/FBBroadcastManager.m | 350 ++++++++++++++++++ .../Utilities/FBBroadcastPickerHost.h | 35 ++ .../Utilities/FBBroadcastPickerHost.m | 129 +++++++ .../Utilities/FBBroadcastProtocol.h | 140 +++++++ .../Utilities/FBBroadcastProtocol.m | 147 ++++++++ WebDriverAgentLib/Utilities/FBConfiguration.h | 14 + WebDriverAgentLib/Utilities/FBConfiguration.m | 20 + .../Utilities/FBVideoStreamManager.h | 3 + .../Utilities/FBVideoStreamManager.m | 20 +- .../Utilities/FBVideoStreamSession.h | 38 ++ .../Utilities/FBVideoStreamSession.m | 102 ++++- docs/broadcast-extension.md | 113 ++++++ package.json | 2 + 30 files changed, 2833 insertions(+), 4 deletions(-) create mode 100755 Scripts/embed-broadcast-extension.sh create mode 100644 WebDriverAgentBroadcast/FBBroadcastSampleHandler.h create mode 100644 WebDriverAgentBroadcast/FBBroadcastSampleHandler.m create mode 100644 WebDriverAgentBroadcast/FBExtBroadcastClient.h create mode 100644 WebDriverAgentBroadcast/FBExtBroadcastClient.m create mode 100644 WebDriverAgentBroadcast/FBExtLogging.h create mode 100644 WebDriverAgentBroadcast/FBExtSessionPipeline.h create mode 100644 WebDriverAgentBroadcast/FBExtSessionPipeline.m create mode 100644 WebDriverAgentBroadcast/Info.plist create mode 100644 WebDriverAgentLib/Utilities/FBBroadcastControlServer.h create mode 100644 WebDriverAgentLib/Utilities/FBBroadcastControlServer.m create mode 100644 WebDriverAgentLib/Utilities/FBBroadcastManager.h create mode 100644 WebDriverAgentLib/Utilities/FBBroadcastManager.m create mode 100644 WebDriverAgentLib/Utilities/FBBroadcastPickerHost.h create mode 100644 WebDriverAgentLib/Utilities/FBBroadcastPickerHost.m create mode 100644 WebDriverAgentLib/Utilities/FBBroadcastProtocol.h create mode 100644 WebDriverAgentLib/Utilities/FBBroadcastProtocol.m create mode 100644 docs/broadcast-extension.md diff --git a/Scripts/embed-broadcast-extension.sh b/Scripts/embed-broadcast-extension.sh new file mode 100755 index 000000000..e2845eefe --- /dev/null +++ b/Scripts/embed-broadcast-extension.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# Embed the WebDriverAgentBroadcast ReplayKit upload extension into the +# wrapping XCTRunner host app. +# +# Apple's USES_XCTRUNNER auto-generates a Runner.app around UI-testing +# .xctest bundles after all target build phases have run, so an appex +# cannot reach Runner.app/PlugIns through a regular embed build phase. +# The extension target is built into BUILT_PRODUCTS_DIR via a target +# dependency of WebDriverAgentRunner; this scheme post-action copies it +# into Runner.app/PlugIns, fixes its bundle id to match the host app +# (extensions must be prefixed by the host's CFBundleIdentifier, which +# Xcode suffixes with '.xctrunner') and re-signs inner-first. +# +# Limitations: +# - Touches XCTRunner internals; may need updates across Xcode versions. +# - iOS only; the extension is not built for tvOS. +# - Cloud device farms that re-sign WDA must re-sign the nested appex +# with the same team first (or use codesign --deep); see +# docs/broadcast-extension.md. + +set -euo pipefail + +RUNNER_APP="${BUILT_PRODUCTS_DIR}/${PRODUCT_NAME}-Runner.app" +APPEX_NAME="WebDriverAgentBroadcast.appex" +APPEX_SRC="${BUILT_PRODUCTS_DIR}/${APPEX_NAME}" + +if [ ! -d "$RUNNER_APP" ]; then + echo "warning: ${PRODUCT_NAME}-Runner.app not found at $RUNNER_APP; skipping broadcast extension embed" + exit 0 +fi + +if [ ! -d "$APPEX_SRC" ]; then + echo "warning: $APPEX_NAME not found at $APPEX_SRC; skipping broadcast extension embed" + exit 0 +fi + +APPEX_DST="$RUNNER_APP/PlugIns/$APPEX_NAME" +rm -rf "$APPEX_DST" +mkdir -p "$RUNNER_APP/PlugIns" +cp -R "$APPEX_SRC" "$APPEX_DST" + +# Extensions must carry a bundle id prefixed by the host app's. The host id is only final at +# this point (Xcode appends '.xctrunner'; downstream tooling may override the prefix), so +# always derive the appex id from the embedded Runner.app instead of trusting build settings. +HOST_ID=$(/usr/libexec/PlistBuddy -c "Print :CFBundleIdentifier" "$RUNNER_APP/Info.plist") +WANT_ID="${HOST_ID}.broadcast" +CURRENT_ID=$(/usr/libexec/PlistBuddy -c "Print :CFBundleIdentifier" "$APPEX_DST/Info.plist") +if [ "$CURRENT_ID" != "$WANT_ID" ]; then + /usr/libexec/PlistBuddy -c "Set :CFBundleIdentifier $WANT_ID" "$APPEX_DST/Info.plist" +fi + +# Re-codesign since we modified the bundle after Xcode signed it: the appex first (its bundle +# id may just have changed, so do NOT preserve the identifier), then the app so its seal covers +# the new nested code. In a scheme post-action context Xcode's CODE_SIGN_* env vars are not +# exposed, so discover the existing signing identity from the already-signed bundle. +if [ -d "$RUNNER_APP/_CodeSignature" ]; then + # Capture the signature info once. Piping codesign straight into + # `awk ... exit` makes awk close the pipe early, killing codesign with + # SIGPIPE -- which `set -o pipefail` turns into a fatal error. That trips + # only when an Authority line exists, i.e. on every real-device build. + SIGN_INFO=$(codesign -dvv "$RUNNER_APP" 2>&1 || true) + EXISTING_IDENT="${EXPANDED_CODE_SIGN_IDENTITY:-}" + if [ -z "$EXISTING_IDENT" ]; then + EXISTING_IDENT=$(awk -F'=' '/^Authority/ {print $2; exit}' <<< "$SIGN_INFO") + fi + # Simulator builds are ad-hoc signed: there is no Authority line, but the + # bundle can still be re-signed ad-hoc with an identity of "-". + if [ -z "$EXISTING_IDENT" ] && grep -q '^Signature=adhoc' <<< "$SIGN_INFO"; then + EXISTING_IDENT="-" + fi + if [ -n "$EXISTING_IDENT" ]; then + codesign --force --sign "$EXISTING_IDENT" \ + --preserve-metadata=entitlements "$APPEX_DST" + codesign --force --sign "$EXISTING_IDENT" \ + --preserve-metadata=identifier,entitlements "$RUNNER_APP" + else + echo "warning: bundle is signed but no identity discovered; signature will be invalid" + fi +fi + +echo "embedded $APPEX_NAME into $RUNNER_APP (bundle id $WANT_ID)" diff --git a/WebDriverAgent.xcodeproj/project.pbxproj b/WebDriverAgent.xcodeproj/project.pbxproj index f0388b4db..76234df98 100644 --- a/WebDriverAgent.xcodeproj/project.pbxproj +++ b/WebDriverAgent.xcodeproj/project.pbxproj @@ -39,6 +39,28 @@ FBCAFE00000000000000001A /* FBPixelBufferConverterTests.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000000019 /* FBPixelBufferConverterTests.m */; }; FBCAFE00000000000000001C /* FBVideoEncoderTests.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE00000000000000001B /* FBVideoEncoderTests.m */; }; FBCAFE000000000000004002 /* FBVideoStreamSessionTests.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000004001 /* FBVideoStreamSessionTests.m */; }; + FBCAFE000000000000005103 /* FBBroadcastSampleHandler.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005102 /* FBBroadcastSampleHandler.m */; }; + FBCAFE000000000000005106 /* FBExtBroadcastClient.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005105 /* FBExtBroadcastClient.m */; }; + FBCAFE000000000000005109 /* FBExtSessionPipeline.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005108 /* FBExtSessionPipeline.m */; }; + FBCAFE000000000000005201 /* FBVideoEncoder.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000000010 /* FBVideoEncoder.m */; }; + FBCAFE000000000000005202 /* GCDAsyncSocket.m in Sources */ = {isa = PBXBuildFile; fileRef = 718226C82587443600661B83 /* GCDAsyncSocket.m */; }; + FBCAFE000000000000005203 /* FBBroadcastProtocol.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005301 /* FBBroadcastProtocol.m */; }; + FBCAFE000000000000005302 /* FBBroadcastProtocol.h in Headers */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005300 /* FBBroadcastProtocol.h */; }; + FBCAFE000000000000005303 /* FBBroadcastProtocol.h in Headers */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005300 /* FBBroadcastProtocol.h */; }; + FBCAFE000000000000005304 /* FBBroadcastProtocol.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005301 /* FBBroadcastProtocol.m */; }; + FBCAFE000000000000005305 /* FBBroadcastProtocol.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005301 /* FBBroadcastProtocol.m */; }; + FBCAFE000000000000005312 /* FBBroadcastControlServer.h in Headers */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005310 /* FBBroadcastControlServer.h */; }; + FBCAFE000000000000005313 /* FBBroadcastControlServer.h in Headers */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005310 /* FBBroadcastControlServer.h */; }; + FBCAFE000000000000005314 /* FBBroadcastControlServer.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005311 /* FBBroadcastControlServer.m */; }; + FBCAFE000000000000005315 /* FBBroadcastControlServer.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005311 /* FBBroadcastControlServer.m */; }; + FBCAFE000000000000005322 /* FBBroadcastManager.h in Headers */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005320 /* FBBroadcastManager.h */; }; + FBCAFE000000000000005323 /* FBBroadcastManager.h in Headers */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005320 /* FBBroadcastManager.h */; }; + FBCAFE000000000000005324 /* FBBroadcastManager.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005321 /* FBBroadcastManager.m */; }; + FBCAFE000000000000005325 /* FBBroadcastManager.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005321 /* FBBroadcastManager.m */; }; + FBCAFE000000000000005332 /* FBBroadcastPickerHost.h in Headers */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005330 /* FBBroadcastPickerHost.h */; }; + FBCAFE000000000000005333 /* FBBroadcastPickerHost.h in Headers */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005330 /* FBBroadcastPickerHost.h */; }; + FBCAFE000000000000005334 /* FBBroadcastPickerHost.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005331 /* FBBroadcastPickerHost.m */; }; + FBCAFE000000000000005335 /* FBBroadcastPickerHost.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005331 /* FBBroadcastPickerHost.m */; }; 0E0413382DF1E15100AF007C /* XCUIElement+FBMinMax.m in Sources */ = {isa = PBXBuildFile; fileRef = 0E0413372DF1E15100AF007C /* XCUIElement+FBMinMax.m */; }; 0E0413392DF1E15100AF007C /* XCUIElement+FBMinMax.m in Sources */ = {isa = PBXBuildFile; fileRef = 0E0413372DF1E15100AF007C /* XCUIElement+FBMinMax.m */; }; 0E04133B2DF1E15900AF007C /* XCUIElement+FBMinMax.h in Headers */ = {isa = PBXBuildFile; fileRef = 0E04133A2DF1E15900AF007C /* XCUIElement+FBMinMax.h */; }; @@ -944,6 +966,13 @@ remoteGlobalIDString = EE158A981CBD452B00A3E3F0; remoteInfo = WebDriverAgentLib; }; + FBCAFE000000000000005009 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 91F9DAE11B99DBC2001349B2 /* Project object */; + proxyType = 1; + remoteGlobalIDString = FBCAFE000000000000005001; + remoteInfo = WebDriverAgentBroadcast; + }; /* End PBXContainerItemProxy section */ /* Begin PBXCopyFilesBuildPhase section */ @@ -990,6 +1019,23 @@ FBCAFE000000000000000019 /* FBPixelBufferConverterTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBPixelBufferConverterTests.m; sourceTree = ""; }; FBCAFE00000000000000001B /* FBVideoEncoderTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBVideoEncoderTests.m; sourceTree = ""; }; FBCAFE000000000000004001 /* FBVideoStreamSessionTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBVideoStreamSessionTests.m; sourceTree = ""; }; + FBCAFE000000000000005002 /* WebDriverAgentBroadcast.appex */ = {isa = PBXFileReference; explicitFileType = "wrapper.app-extension"; includeInIndex = 0; path = WebDriverAgentBroadcast.appex; sourceTree = BUILT_PRODUCTS_DIR; }; + FBCAFE00000000000000500C /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + FBCAFE000000000000005101 /* FBBroadcastSampleHandler.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FBBroadcastSampleHandler.h; sourceTree = ""; }; + FBCAFE000000000000005102 /* FBBroadcastSampleHandler.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBBroadcastSampleHandler.m; sourceTree = ""; }; + FBCAFE000000000000005104 /* FBExtBroadcastClient.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FBExtBroadcastClient.h; sourceTree = ""; }; + FBCAFE000000000000005105 /* FBExtBroadcastClient.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBExtBroadcastClient.m; sourceTree = ""; }; + FBCAFE000000000000005107 /* FBExtSessionPipeline.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FBExtSessionPipeline.h; sourceTree = ""; }; + FBCAFE000000000000005108 /* FBExtSessionPipeline.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBExtSessionPipeline.m; sourceTree = ""; }; + FBCAFE00000000000000510A /* FBExtLogging.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FBExtLogging.h; sourceTree = ""; }; + FBCAFE000000000000005300 /* FBBroadcastProtocol.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FBBroadcastProtocol.h; sourceTree = ""; }; + FBCAFE000000000000005301 /* FBBroadcastProtocol.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBBroadcastProtocol.m; sourceTree = ""; }; + FBCAFE000000000000005310 /* FBBroadcastControlServer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FBBroadcastControlServer.h; sourceTree = ""; }; + FBCAFE000000000000005311 /* FBBroadcastControlServer.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBBroadcastControlServer.m; sourceTree = ""; }; + FBCAFE000000000000005320 /* FBBroadcastManager.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FBBroadcastManager.h; sourceTree = ""; }; + FBCAFE000000000000005321 /* FBBroadcastManager.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBBroadcastManager.m; sourceTree = ""; }; + FBCAFE000000000000005330 /* FBBroadcastPickerHost.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FBBroadcastPickerHost.h; sourceTree = ""; }; + FBCAFE000000000000005331 /* FBBroadcastPickerHost.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBBroadcastPickerHost.m; sourceTree = ""; }; 0E0413372DF1E15100AF007C /* XCUIElement+FBMinMax.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = "XCUIElement+FBMinMax.m"; sourceTree = ""; }; 0E04133A2DF1E15900AF007C /* XCUIElement+FBMinMax.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "XCUIElement+FBMinMax.h"; sourceTree = ""; }; 1357E295233D05240054BDB8 /* XCUIHitPointResult.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = XCUIHitPointResult.h; sourceTree = ""; }; @@ -1573,6 +1619,13 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + FBCAFE000000000000005007 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ @@ -1680,6 +1733,7 @@ EEC288F81BF0ED2500B4DC79 /* WebDriverAgentLib */, EE9B75F91CF7964100275851 /* WebDriverAgentTests */, EEF988341C486655005CA669 /* WebDriverAgentRunner */, + FBCAFE00000000000000500B /* WebDriverAgentBroadcast */, EE9AB8021CAEE182008C271F /* Scripts */, 91F9DAEA1B99DBC2001349B2 /* Products */, B6E83A410C45944B036B6B0F /* Frameworks */, @@ -1703,6 +1757,7 @@ 641EE2DA2240BBE300173FCB /* WebDriverAgentRunner_tvOS.xctest */, 641EE6F82240C5CA00173FCB /* WebDriverAgentLib_tvOS.framework */, 64B264F9228C50E0002A5025 /* UnitTests_tvOS.xctest */, + FBCAFE000000000000005002 /* WebDriverAgentBroadcast.appex */, ); name = Products; sourceTree = ""; @@ -2096,6 +2151,14 @@ FBCAFE000000000000000010 /* FBVideoEncoder.m */, FBCAFE000000000000000013 /* FBVideoStreamManager.h */, FBCAFE000000000000000016 /* FBVideoStreamManager.m */, + FBCAFE000000000000005300 /* FBBroadcastProtocol.h */, + FBCAFE000000000000005301 /* FBBroadcastProtocol.m */, + FBCAFE000000000000005310 /* FBBroadcastControlServer.h */, + FBCAFE000000000000005311 /* FBBroadcastControlServer.m */, + FBCAFE000000000000005320 /* FBBroadcastManager.h */, + FBCAFE000000000000005321 /* FBBroadcastManager.m */, + FBCAFE000000000000005330 /* FBBroadcastPickerHost.h */, + FBCAFE000000000000005331 /* FBBroadcastPickerHost.m */, FBCAFE000000000000001001 /* FBVideoStreamSession.h */, FBCAFE000000000000001004 /* FBVideoStreamSession.m */, ); @@ -2398,6 +2461,21 @@ path = XCTUITestRunner; sourceTree = SOURCE_ROOT; }; + FBCAFE00000000000000500B /* WebDriverAgentBroadcast */ = { + isa = PBXGroup; + children = ( + FBCAFE00000000000000500C /* Info.plist */, + FBCAFE000000000000005101 /* FBBroadcastSampleHandler.h */, + FBCAFE000000000000005102 /* FBBroadcastSampleHandler.m */, + FBCAFE000000000000005104 /* FBExtBroadcastClient.h */, + FBCAFE000000000000005105 /* FBExtBroadcastClient.m */, + FBCAFE000000000000005107 /* FBExtSessionPipeline.h */, + FBCAFE000000000000005108 /* FBExtSessionPipeline.m */, + FBCAFE00000000000000510A /* FBExtLogging.h */, + ); + path = WebDriverAgentBroadcast; + sourceTree = ""; + }; /* End PBXGroup section */ /* Begin PBXHeadersBuildPhase section */ @@ -2648,6 +2726,10 @@ FBCAFE000000000000000009 /* FBPixelBufferConverter.h in Headers */, FBCAFE00000000000000000F /* FBVideoEncoder.h in Headers */, FBCAFE000000000000000015 /* FBVideoStreamManager.h in Headers */, + FBCAFE000000000000005302 /* FBBroadcastProtocol.h in Headers */, + FBCAFE000000000000005312 /* FBBroadcastControlServer.h in Headers */, + FBCAFE000000000000005322 /* FBBroadcastManager.h in Headers */, + FBCAFE000000000000005332 /* FBBroadcastPickerHost.h in Headers */, FBCAFE000000000000001003 /* FBVideoStreamSession.h in Headers */, ); runOnlyForDeploymentPostprocessing = 0; @@ -2902,6 +2984,10 @@ FBCAFE000000000000000008 /* FBPixelBufferConverter.h in Headers */, FBCAFE00000000000000000E /* FBVideoEncoder.h in Headers */, FBCAFE000000000000000014 /* FBVideoStreamManager.h in Headers */, + FBCAFE000000000000005303 /* FBBroadcastProtocol.h in Headers */, + FBCAFE000000000000005313 /* FBBroadcastControlServer.h in Headers */, + FBCAFE000000000000005323 /* FBBroadcastManager.h in Headers */, + FBCAFE000000000000005333 /* FBBroadcastPickerHost.h in Headers */, FBCAFE000000000000001002 /* FBVideoStreamSession.h in Headers */, ); runOnlyForDeploymentPostprocessing = 0; @@ -3087,12 +3173,30 @@ ); dependencies = ( EE158B5C1CBD462500A3E3F0 /* PBXTargetDependency */, + FBCAFE00000000000000500A /* PBXTargetDependency */, ); name = WebDriverAgentRunner; productName = XCTUITestRunner; productReference = EEF9882A1C486603005CA669 /* WebDriverAgentRunner.xctest */; productType = "com.apple.product-type.bundle.ui-testing"; }; + FBCAFE000000000000005001 /* WebDriverAgentBroadcast */ = { + isa = PBXNativeTarget; + buildConfigurationList = FBCAFE000000000000005003 /* Build configuration list for PBXNativeTarget "WebDriverAgentBroadcast" */; + buildPhases = ( + FBCAFE000000000000005006 /* Sources */, + FBCAFE000000000000005007 /* Frameworks */, + FBCAFE000000000000005008 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = WebDriverAgentBroadcast; + productName = WebDriverAgentBroadcast; + productReference = FBCAFE000000000000005002 /* WebDriverAgentBroadcast.appex */; + productType = "com.apple.product-type.app-extension"; + }; /* End PBXNativeTarget section */ /* Begin PBXProject section */ @@ -3129,6 +3233,9 @@ EEF988291C486603005CA669 = { CreatedOnToolsVersion = 7.2; }; + FBCAFE000000000000005001 = { + CreatedOnToolsVersion = 15.0; + }; }; }; buildConfigurationList = 91F9DAE41B99DBC2001349B2 /* Build configuration list for PBXProject "WebDriverAgent" */; @@ -3147,6 +3254,7 @@ EE158A981CBD452B00A3E3F0 /* WebDriverAgentLib */, 641EE5D52240C5CA00173FCB /* WebDriverAgentLib_tvOS */, EEF988291C486603005CA669 /* WebDriverAgentRunner */, + FBCAFE000000000000005001 /* WebDriverAgentBroadcast */, 641EE2D92240BBE300173FCB /* WebDriverAgentRunner_tvOS */, EE836C011C0F118600D87246 /* UnitTests */, 64B264F8228C50E0002A5025 /* UnitTests_tvOS */, @@ -3231,6 +3339,13 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + FBCAFE000000000000005008 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXResourcesBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ @@ -3377,6 +3492,10 @@ FBCAFE00000000000000000C /* FBPixelBufferConverter.m in Sources */, FBCAFE000000000000000012 /* FBVideoEncoder.m in Sources */, FBCAFE000000000000000018 /* FBVideoStreamManager.m in Sources */, + FBCAFE000000000000005304 /* FBBroadcastProtocol.m in Sources */, + FBCAFE000000000000005314 /* FBBroadcastControlServer.m in Sources */, + FBCAFE000000000000005324 /* FBBroadcastManager.m in Sources */, + FBCAFE000000000000005334 /* FBBroadcastPickerHost.m in Sources */, FBCAFE000000000000001006 /* FBVideoStreamSession.m in Sources */, ); runOnlyForDeploymentPostprocessing = 0; @@ -3526,6 +3645,10 @@ FBCAFE00000000000000000B /* FBPixelBufferConverter.m in Sources */, FBCAFE000000000000000011 /* FBVideoEncoder.m in Sources */, FBCAFE000000000000000017 /* FBVideoStreamManager.m in Sources */, + FBCAFE000000000000005305 /* FBBroadcastProtocol.m in Sources */, + FBCAFE000000000000005315 /* FBBroadcastControlServer.m in Sources */, + FBCAFE000000000000005325 /* FBBroadcastManager.m in Sources */, + FBCAFE000000000000005335 /* FBBroadcastPickerHost.m in Sources */, FBCAFE000000000000001005 /* FBVideoStreamSession.m in Sources */, ); runOnlyForDeploymentPostprocessing = 0; @@ -3655,6 +3778,19 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + FBCAFE000000000000005006 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + FBCAFE000000000000005103 /* FBBroadcastSampleHandler.m in Sources */, + FBCAFE000000000000005106 /* FBExtBroadcastClient.m in Sources */, + FBCAFE000000000000005109 /* FBExtSessionPipeline.m in Sources */, + FBCAFE000000000000005201 /* FBVideoEncoder.m in Sources */, + FBCAFE000000000000005202 /* GCDAsyncSocket.m in Sources */, + FBCAFE000000000000005203 /* FBBroadcastProtocol.m in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXSourcesBuildPhase section */ /* Begin PBXTargetDependency section */ @@ -3708,6 +3844,11 @@ target = EE158A981CBD452B00A3E3F0 /* WebDriverAgentLib */; targetProxy = EE9B769E1CF79C0A00275851 /* PBXContainerItemProxy */; }; + FBCAFE00000000000000500A /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = FBCAFE000000000000005001 /* WebDriverAgentBroadcast */; + targetProxy = FBCAFE000000000000005009 /* PBXContainerItemProxy */; + }; /* End PBXTargetDependency section */ /* Begin PBXVariantGroup section */ @@ -4594,6 +4735,38 @@ }; name = Release; }; + FBCAFE000000000000005004 /* Debug */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = EEE5CABF1C80361500CBBDD9 /* IOSSettings.xcconfig */; + buildSettings = { + APPLICATION_EXTENSION_API_ONLY = YES; + DEBUG_INFORMATION_FORMAT = dwarf; + GCC_TREAT_WARNINGS_AS_ERRORS = NO; + INFOPLIST_FILE = WebDriverAgentBroadcast/Info.plist; + PRODUCT_BUNDLE_IDENTIFIER = com.facebook.WebDriverAgentRunner.xctrunner.broadcast; + PRODUCT_NAME = "$(TARGET_NAME)"; + SKIP_INSTALL = YES; + SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Debug; + }; + FBCAFE000000000000005005 /* Release */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = EEE5CABF1C80361500CBBDD9 /* IOSSettings.xcconfig */; + buildSettings = { + APPLICATION_EXTENSION_API_ONLY = YES; + GCC_TREAT_WARNINGS_AS_ERRORS = NO; + INFOPLIST_FILE = WebDriverAgentBroadcast/Info.plist; + ONLY_ACTIVE_ARCH = YES; + PRODUCT_BUNDLE_IDENTIFIER = com.facebook.WebDriverAgentRunner.xctrunner.broadcast; + PRODUCT_NAME = "$(TARGET_NAME)"; + SKIP_INSTALL = YES; + SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Release; + }; /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ @@ -4696,6 +4869,15 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; + FBCAFE000000000000005003 /* Build configuration list for PBXNativeTarget "WebDriverAgentBroadcast" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + FBCAFE000000000000005004 /* Debug */, + FBCAFE000000000000005005 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; /* End XCConfigurationList section */ }; rootObject = 91F9DAE11B99DBC2001349B2 /* Project object */; diff --git a/WebDriverAgent.xcodeproj/xcshareddata/xcschemes/WebDriverAgentRunner-nodebug.xcscheme b/WebDriverAgent.xcodeproj/xcshareddata/xcschemes/WebDriverAgentRunner-nodebug.xcscheme index 5ff961179..c6878787c 100644 --- a/WebDriverAgent.xcodeproj/xcshareddata/xcschemes/WebDriverAgentRunner-nodebug.xcscheme +++ b/WebDriverAgent.xcodeproj/xcshareddata/xcschemes/WebDriverAgentRunner-nodebug.xcscheme @@ -5,6 +5,24 @@ + + + + + + + + + + + + + + + + + + + +NS_ASSUME_NONNULL_BEGIN + +/** + The broadcast upload extension's principal class: receives the system screen frames from + ReplayKit and forwards them to the per-session encode pipelines managed by + FBExtBroadcastClient. Audio samples are ignored (audio capture is a planned follow-up). + */ +@interface FBBroadcastSampleHandler : RPBroadcastSampleHandler + +@end + +NS_ASSUME_NONNULL_END diff --git a/WebDriverAgentBroadcast/FBBroadcastSampleHandler.m b/WebDriverAgentBroadcast/FBBroadcastSampleHandler.m new file mode 100644 index 000000000..604f5f956 --- /dev/null +++ b/WebDriverAgentBroadcast/FBBroadcastSampleHandler.m @@ -0,0 +1,138 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import "FBBroadcastSampleHandler.h" + +#import +#import + +#import "FBBroadcastProtocol.h" +#import "FBExtBroadcastClient.h" +#import "FBExtLogging.h" +#import "FBExtSessionPipeline.h" + +static NSString *const FBBroadcastSampleHandlerErrorDomain = @"com.facebook.WebDriverAgent.FBBroadcastSampleHandler"; + +@interface FBBroadcastSampleHandler () + +@property (nonatomic, nullable) FBExtBroadcastClient *client; + +@end + +@implementation FBBroadcastSampleHandler + +- (void)broadcastStartedWithSetupInfo:(nullable NSDictionary *)setupInfo +{ + // setupInfo is nil for broadcasts started from the system picker; nothing to read from it. + FBExtLogInfo("Broadcast started"); + self.client = [[FBExtBroadcastClient alloc] init]; + self.client.delegate = self; + [self.client start]; +} + +- (void)broadcastPaused +{ + FBExtLogInfo("Broadcast paused"); + self.client.paused = YES; + [self sendStatusEvent:@"paused" reason:nil]; +} + +- (void)broadcastResumed +{ + FBExtLogInfo("Broadcast resumed"); + self.client.paused = NO; + [self sendStatusEvent:@"resumed" reason:nil]; +} + +- (void)broadcastFinished +{ + FBExtLogInfo("Broadcast finished"); + [self sendStatusEvent:@"finishing" reason:@"broadcastFinished"]; + [self.client shutdown]; + self.client = nil; +} + +- (void)processSampleBuffer:(CMSampleBufferRef)sampleBuffer + withType:(RPSampleBufferType)sampleBufferType +{ + if (sampleBufferType != RPSampleBufferTypeVideo) { + // Audio capture is a planned follow-up. + return; + } + FBExtBroadcastClient *client = self.client; + if (nil == client) { + return; + } + + client.framesReceived += 1; + + CFTypeRef orientationAttachment = CMGetAttachment(sampleBuffer, + (__bridge CFStringRef)RPVideoSampleOrientationKey, + NULL); + if (NULL != orientationAttachment) { + client.currentOrientation = (uint8_t)[(__bridge NSNumber *)orientationAttachment unsignedIntValue]; + } + + CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + if (NULL != pixelBuffer && client.screenWidth == 0) { + client.screenWidth = CVPixelBufferGetWidth(pixelBuffer); + client.screenHeight = CVPixelBufferGetHeight(pixelBuffer); + } + + NSDictionary *pipelines = client.activePipelines; + for (FBExtSessionPipeline *pipeline in pipelines.allValues) { + [pipeline submitSampleBuffer:sampleBuffer orientation:client.currentOrientation]; + } +} + +- (void)sendStatusEvent:(NSString *)event reason:(nullable NSString *)reason +{ + NSMutableDictionary *payload = [NSMutableDictionary dictionaryWithObject:event + forKey:FBBroadcastKeyEvent]; + if (nil != reason) { + payload[FBBroadcastKeyReason] = reason; + } + NSData *message = FBBroadcastEncodeJSONMessage(FBBroadcastMessageTypeStatus, 0, payload); + if (nil != message) { + [self.client sendProtocolMessage:message isDroppable:NO]; + } +} + +- (void)finishBroadcast +{ + // The public API only offers finishBroadcastWithError:, which makes the system show an error + // alert. The private graceful variant avoids that; fall back to the public one when absent. + SEL gracefulSelector = NSSelectorFromString(@"finishBroadcastGracefully:"); + if ([self respondsToSelector:gracefulSelector]) { +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Warc-performSelector-leaks" + [self performSelector:gracefulSelector withObject:nil]; +#pragma clang diagnostic pop + return; + } + NSError *error = [NSError errorWithDomain:FBBroadcastSampleHandlerErrorDomain + code:1 + userInfo:@{NSLocalizedDescriptionKey: @"WebDriverAgent stopped the broadcast"}]; + [self finishBroadcastWithError:error]; +} + +#pragma mark - + +- (void)broadcastClientDidRequestStop:(FBExtBroadcastClient *)client +{ + [self sendStatusEvent:@"finishing" reason:@"stopRequested"]; + [self finishBroadcast]; +} + +- (void)broadcastClient:(FBExtBroadcastClient *)client didFailPermanently:(NSError *)error +{ + FBExtLogError("Finishing the broadcast: %{public}@", error.description); + [self finishBroadcastWithError:error]; +} + +@end diff --git a/WebDriverAgentBroadcast/FBExtBroadcastClient.h b/WebDriverAgentBroadcast/FBExtBroadcastClient.h new file mode 100644 index 000000000..bfab075c0 --- /dev/null +++ b/WebDriverAgentBroadcast/FBExtBroadcastClient.h @@ -0,0 +1,54 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import + +#import "FBExtSessionPipeline.h" + +NS_ASSUME_NONNULL_BEGIN + +@class FBExtBroadcastClient; + +@protocol FBExtBroadcastClientDelegate + +/** WDA asked the extension to finish the broadcast. */ +- (void)broadcastClientDidRequestStop:(FBExtBroadcastClient *)client; + +/** The connection to WDA could not be (re-)established; the broadcast should be finished. */ +- (void)broadcastClient:(FBExtBroadcastClient *)client didFailPermanently:(NSError *)error; + +@end + +/** + The extension's endpoint of the WDA control connection: connects to WDA's loopback control + port, answers session add/remove/keyframe requests by managing FBExtSessionPipeline instances, + sends the HELLO/heartbeat messages and pushes the pipelines' encoded output. + */ +@interface FBExtBroadcastClient : NSObject + +@property (nonatomic, weak) id delegate; + +/** Stats fed into the heartbeat; updated by the sample handler. */ +@property (atomic) BOOL paused; +@property (atomic) uint64_t framesReceived; +@property (atomic) uint8_t currentOrientation; +@property (atomic) size_t screenWidth; +@property (atomic) size_t screenHeight; + +/** A point-in-time snapshot of the active pipelines, safe to read from any thread. */ +@property (atomic, copy, readonly) NSDictionary *activePipelines; + +/** Starts connecting to the WDA control port (with retries). */ +- (void)start; + +/** Tears down all pipelines and closes the connection (no reconnect attempts). */ +- (void)shutdown; + +@end + +NS_ASSUME_NONNULL_END diff --git a/WebDriverAgentBroadcast/FBExtBroadcastClient.m b/WebDriverAgentBroadcast/FBExtBroadcastClient.m new file mode 100644 index 000000000..adab5cd82 --- /dev/null +++ b/WebDriverAgentBroadcast/FBExtBroadcastClient.m @@ -0,0 +1,302 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import "FBExtBroadcastClient.h" + +#import "FBBroadcastProtocol.h" +#import "FBExtLogging.h" +#import "GCDAsyncSocket.h" + +static const long TAG_HEADER = 1; +static const long TAG_PAYLOAD = 2; + +static const NSUInteger CONNECT_ATTEMPTS = 3; +static const NSTimeInterval CONNECT_RETRY_DELAY = 1.0; +static const NSTimeInterval CONNECT_TIMEOUT = 5.0; +static const NSTimeInterval HEARTBEAT_INTERVAL = 2.0; +// Loopback writes should never back up this far; if they do, WDA is wedged - shed delta frames. +static const size_t MAX_OUTSTANDING_WRITE_BYTES = 4 * 1024 * 1024; + +static NSString *const FBExtClientErrorDomain = @"com.facebook.WebDriverAgent.FBExtBroadcastClient"; + +@interface FBExtBroadcastClient () + +@property (nonatomic) dispatch_queue_t queue; +@property (nonatomic, nullable) GCDAsyncSocket *socket; +@property (nonatomic, nullable) dispatch_source_t heartbeatTimer; +@property (nonatomic) NSMutableDictionary *pipelines; +@property (atomic, copy, readwrite) NSDictionary *activePipelines; +@property (nonatomic) NSUInteger remainingConnectAttempts; +@property (atomic) BOOL stopped; +@property (atomic) BOOL connected; +@property (nonatomic) size_t outstandingWriteBytes; +@property (nonatomic) FBBroadcastMessageHeader pendingHeader; + +@end + +@implementation FBExtBroadcastClient + +- (instancetype)init +{ + if ((self = [super init])) { + _queue = dispatch_queue_create("wda.broadcast.client", DISPATCH_QUEUE_SERIAL); + _pipelines = [NSMutableDictionary dictionary]; + _activePipelines = @{}; + _remainingConnectAttempts = CONNECT_ATTEMPTS; + } + return self; +} + +- (void)start +{ + dispatch_async(self.queue, ^{ + [self connect]; + }); +} + +- (void)connect +{ + if (self.stopped) { + return; + } + self.socket = [[GCDAsyncSocket alloc] initWithDelegate:self delegateQueue:self.queue]; + NSError *error; + if (![self.socket connectToHost:@"127.0.0.1" onPort:FBBroadcastDefaultControlPort + withTimeout:CONNECT_TIMEOUT error:&error]) { + FBExtLogError("Cannot start connecting to WDA: %{public}@", error.description); + [self scheduleReconnectOrGiveUp:error]; + } +} + +- (void)scheduleReconnectOrGiveUp:(nullable NSError *)error +{ + if (self.stopped) { + return; + } + if (self.remainingConnectAttempts == 0) { + NSError *permanentError = error + ?: [NSError errorWithDomain:FBExtClientErrorDomain + code:1 + userInfo:@{NSLocalizedDescriptionKey: @"Lost the connection to WebDriverAgent"}]; + [self.delegate broadcastClient:self didFailPermanently:permanentError]; + return; + } + self.remainingConnectAttempts -= 1; + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(CONNECT_RETRY_DELAY * NSEC_PER_SEC)), + self.queue, ^{ + [self connect]; + }); +} + +- (void)shutdown +{ + self.stopped = YES; + dispatch_async(self.queue, ^{ + [self stopHeartbeat]; + for (FBExtSessionPipeline *pipeline in self.pipelines.allValues) { + [pipeline teardown]; + } + [self.pipelines removeAllObjects]; + self.activePipelines = @{}; + self.socket.delegate = nil; + [self.socket disconnect]; + self.socket = nil; + }); +} + +#pragma mark - + +- (void)sendProtocolMessage:(NSData *)message isDroppable:(BOOL)droppable +{ + dispatch_async(self.queue, ^{ + if (!self.connected || nil == self.socket) { + return; + } + if (droppable && self.outstandingWriteBytes > MAX_OUTSTANDING_WRITE_BYTES) { + return; + } + self.outstandingWriteBytes += message.length; + [self.socket writeData:message withTimeout:-1 tag:(long)message.length]; + }); +} + +#pragma mark - Heartbeat + +- (void)startHeartbeat +{ + [self stopHeartbeat]; + dispatch_source_t timer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, self.queue); + if (nil == timer) { + return; + } + dispatch_source_set_timer(timer, + dispatch_time(DISPATCH_TIME_NOW, (int64_t)(HEARTBEAT_INTERVAL * NSEC_PER_SEC)), + (uint64_t)(HEARTBEAT_INTERVAL * NSEC_PER_SEC), + (uint64_t)(0.2 * NSEC_PER_SEC)); + __weak typeof(self) weakSelf = self; + dispatch_source_set_event_handler(timer, ^{ + [weakSelf sendHeartbeat]; + }); + dispatch_resume(timer); + self.heartbeatTimer = timer; +} + +- (void)stopHeartbeat +{ + dispatch_source_t timer = self.heartbeatTimer; + if (nil != timer) { + dispatch_source_cancel(timer); + self.heartbeatTimer = nil; + } +} + +- (void)sendHeartbeat +{ + NSData *message = FBBroadcastEncodeJSONMessage(FBBroadcastMessageTypeHeartbeat, 0, @{ + FBBroadcastKeyState: self.paused ? @"paused" : @"active", + FBBroadcastKeyFramesReceived: @(self.framesReceived), + FBBroadcastKeyOrientation: @(self.currentOrientation), + FBBroadcastKeyScreenWidth: @(self.screenWidth), + FBBroadcastKeyScreenHeight: @(self.screenHeight), + }); + if (nil != message) { + [self sendProtocolMessage:message isDroppable:NO]; + } +} + +#pragma mark - Incoming messages + +- (void)handleMessageWithHeader:(FBBroadcastMessageHeader)header payload:(NSData *)payload +{ + switch (header.type) { + case FBBroadcastMessageTypeSessionAdd: { + NSDictionary *configuration = FBBroadcastParseJSONPayload(payload); + if (nil == configuration) { + FBExtLogError("Session %u: SESSION_ADD payload is not a JSON object", header.sessionId); + return; + } + [self addSession:header.sessionId configuration:configuration]; + return; + } + case FBBroadcastMessageTypeSessionRemove: + [self removeSession:header.sessionId]; + return; + case FBBroadcastMessageTypeKeyframeRequest: + [self.pipelines[@(header.sessionId)] requestKeyFrame]; + return; + case FBBroadcastMessageTypeStopBroadcast: + FBExtLogInfo("WDA requested the broadcast to stop"); + self.stopped = YES; + [self.delegate broadcastClientDidRequestStop:self]; + return; + default: + FBExtLogError("Ignoring an unexpected message of type 0x%02x", header.type); + return; + } +} + +- (void)addSession:(uint32_t)sessionId configuration:(NSDictionary *)configuration +{ + [self removeSession:sessionId]; + NSError *error; + FBExtSessionPipeline *pipeline = [[FBExtSessionPipeline alloc] initWithSessionId:sessionId + configuration:configuration + sink:self + error:&error]; + if (nil == pipeline) { + FBExtLogError("Session %u: cannot create a pipeline: %{public}@", sessionId, error.description); + NSData *message = FBBroadcastEncodeJSONMessage(FBBroadcastMessageTypeSessionError, sessionId, @{ + FBBroadcastKeyMessage: error.localizedDescription ?: @"Cannot create the session pipeline", + }); + if (nil != message) { + [self sendProtocolMessage:message isDroppable:NO]; + } + return; + } + self.pipelines[@(sessionId)] = pipeline; + self.activePipelines = self.pipelines; + FBExtLogInfo("Session %u: pipeline created (%{public}@)", sessionId, configuration.description); +} + +- (void)removeSession:(uint32_t)sessionId +{ + FBExtSessionPipeline *pipeline = self.pipelines[@(sessionId)]; + if (nil == pipeline) { + return; + } + [pipeline teardown]; + [self.pipelines removeObjectForKey:@(sessionId)]; + self.activePipelines = self.pipelines; + FBExtLogInfo("Session %u: pipeline removed", sessionId); +} + +#pragma mark - + +- (void)socket:(GCDAsyncSocket *)sock didConnectToHost:(NSString *)host port:(uint16_t)port +{ + FBExtLogInfo("Connected to WDA at %{public}@:%d", host, port); + self.connected = YES; + self.remainingConnectAttempts = CONNECT_ATTEMPTS; + NSData *hello = FBBroadcastEncodeJSONMessage(FBBroadcastMessageTypeHello, 0, @{ + FBBroadcastKeyProtocolVersion: @(FBBroadcastProtocolVersion), + FBBroadcastKeyOsVersion: NSProcessInfo.processInfo.operatingSystemVersionString, + }); + if (nil != hello) { + [self sendProtocolMessage:hello isDroppable:NO]; + } + [self startHeartbeat]; + [sock readDataToLength:FBBroadcastHeaderLength withTimeout:-1 tag:TAG_HEADER]; +} + +- (void)socket:(GCDAsyncSocket *)sock didReadData:(NSData *)data withTag:(long)tag +{ + if (tag == TAG_HEADER) { + FBBroadcastMessageHeader header; + if (!FBBroadcastParseHeader(data, &header)) { + FBExtLogError("Malformed control message header; disconnecting"); + [sock disconnect]; + return; + } + self.pendingHeader = header; + if (header.payloadLength == 0) { + [self handleMessageWithHeader:header payload:NSData.data]; + [sock readDataToLength:FBBroadcastHeaderLength withTimeout:-1 tag:TAG_HEADER]; + } else { + [sock readDataToLength:header.payloadLength withTimeout:-1 tag:TAG_PAYLOAD]; + } + return; + } + [self handleMessageWithHeader:self.pendingHeader payload:data]; + [sock readDataToLength:FBBroadcastHeaderLength withTimeout:-1 tag:TAG_HEADER]; +} + +- (void)socket:(GCDAsyncSocket *)sock didWriteDataWithTag:(long)tag +{ + if (tag > 0 && self.outstandingWriteBytes >= (size_t)tag) { + self.outstandingWriteBytes -= (size_t)tag; + } else { + self.outstandingWriteBytes = 0; + } +} + +- (void)socketDidDisconnect:(GCDAsyncSocket *)sock withError:(nullable NSError *)error +{ + if (sock != self.socket) { + return; + } + self.connected = NO; + self.outstandingWriteBytes = 0; + [self stopHeartbeat]; + if (self.stopped) { + return; + } + FBExtLogError("Disconnected from WDA: %{public}@", error.description ?: @"closed"); + [self scheduleReconnectOrGiveUp:error]; +} + +@end diff --git a/WebDriverAgentBroadcast/FBExtLogging.h b/WebDriverAgentBroadcast/FBExtLogging.h new file mode 100644 index 000000000..f898407df --- /dev/null +++ b/WebDriverAgentBroadcast/FBExtLogging.h @@ -0,0 +1,27 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import + +/** + Minimal logging for the broadcast extension. FBLogger cannot be used here because it pulls in + FBConfiguration and, transitively, XCTest, which is forbidden in app extensions. + View with: log stream --predicate 'subsystem == "com.facebook.WebDriverAgent.broadcast"' + */ +static inline os_log_t FBExtLog(void) +{ + static os_log_t logger; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + logger = os_log_create("com.facebook.WebDriverAgent.broadcast", "extension"); + }); + return logger; +} + +#define FBExtLogInfo(...) os_log_info(FBExtLog(), __VA_ARGS__) +#define FBExtLogError(...) os_log_error(FBExtLog(), __VA_ARGS__) diff --git a/WebDriverAgentBroadcast/FBExtSessionPipeline.h b/WebDriverAgentBroadcast/FBExtSessionPipeline.h new file mode 100644 index 000000000..b5b991bb6 --- /dev/null +++ b/WebDriverAgentBroadcast/FBExtSessionPipeline.h @@ -0,0 +1,65 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +/** Receives fully framed protocol messages produced by a pipeline. */ +@protocol FBExtMessageSink + +/** + @param message A complete wire message (header + payload) + @param droppable YES when the message may be dropped under backpressure (delta frames) + */ +- (void)sendProtocolMessage:(NSData *)message isDroppable:(BOOL)droppable; + +@end + +/** + One per WDA capture session: letterbox-scales incoming ReplayKit pixel buffers to the session's + dimensions and encodes them with the session's codec/bitrate/fps, emitting VIDEO_PARAMS and + VIDEO_FRAME protocol messages to the sink. + */ +@interface FBExtSessionPipeline : NSObject + +@property (nonatomic, readonly) uint32_t sessionId; + +/** + @param sessionId The WDA session identifier + @param configuration The SESSION_ADD JSON payload (width, height, codec, bitrate, fps) + @param sink The message sink (held weakly) + @param error Set when the encoder or scaler cannot be created + */ +- (nullable instancetype)initWithSessionId:(uint32_t)sessionId + configuration:(NSDictionary *)configuration + sink:(id)sink + error:(NSError **)error; + +- (instancetype)init NS_UNAVAILABLE; + +/** + Submits a ReplayKit video sample for this session. Called on the ReplayKit sample queue and + never blocks: the frame is dropped when the session is not yet due for a new frame (fps pacing) + or when the previous frame is still being scaled/submitted. + + @param sampleBuffer The video sample from ReplayKit + @param orientation The CGImagePropertyOrientation (1-8) reported for the sample + */ +- (void)submitSampleBuffer:(CMSampleBufferRef)sampleBuffer orientation:(uint8_t)orientation; + +/** Forces the next encoded frame to be a key frame. */ +- (void)requestKeyFrame; + +/** Stops the encoder and releases the scaler and buffer pool. */ +- (void)teardown; + +@end + +NS_ASSUME_NONNULL_END diff --git a/WebDriverAgentBroadcast/FBExtSessionPipeline.m b/WebDriverAgentBroadcast/FBExtSessionPipeline.m new file mode 100644 index 000000000..5361c0ab5 --- /dev/null +++ b/WebDriverAgentBroadcast/FBExtSessionPipeline.m @@ -0,0 +1,272 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import "FBExtSessionPipeline.h" + +#include + +#import +#import + +#import "FBBroadcastProtocol.h" +#import "FBExtLogging.h" +#import "FBVideoEncoder.h" + +static NSString *const FBExtPipelineErrorDomain = @"com.facebook.WebDriverAgent.FBExtSessionPipeline"; + +// Bounds the scaled-buffer pool so a stuck consumer cannot grow the extension past its ~50MB cap. +static const int POOL_ALLOCATION_THRESHOLD = 3; + +@interface FBExtSessionPipeline () + +@property (nonatomic, weak) id sink; +@property (nonatomic) dispatch_queue_t queue; +@property (nonatomic, nullable) FBVideoEncoder *encoder; +@property (nonatomic) VTPixelTransferSessionRef transferSession; +@property (nonatomic) CVPixelBufferPoolRef bufferPool; +@property (nonatomic) NSUInteger fps; +@property (atomic) uint64_t lastSubmitTimeMs; +@property (atomic) BOOL inFlight; +@property (atomic) BOOL active; +@property (atomic) uint8_t currentOrientation; +@property (nonatomic, nullable, copy) NSData *lastSentParameterSets; + +@end + +@implementation FBExtSessionPipeline + +- (nullable instancetype)initWithSessionId:(uint32_t)sessionId + configuration:(NSDictionary *)configuration + sink:(id)sink + error:(NSError **)error +{ + if ((self = [super init])) { + _sessionId = sessionId; + _sink = sink; + _queue = dispatch_queue_create([NSString stringWithFormat:@"wda.broadcast.session.%u", sessionId].UTF8String, + DISPATCH_QUEUE_SERIAL); + + NSUInteger width = [configuration[FBBroadcastKeyWidth] unsignedIntegerValue]; + NSUInteger height = [configuration[FBBroadcastKeyHeight] unsignedIntegerValue]; + // Hardware encoders require even dimensions (same rule as the WDA-side converter). + width -= width % 2; + height -= height % 2; + NSUInteger bitrate = [configuration[FBBroadcastKeyBitrate] unsignedIntegerValue]; + NSUInteger fps = [configuration[FBBroadcastKeyFps] unsignedIntegerValue]; + NSString *codecName = [configuration[FBBroadcastKeyCodec] isKindOfClass:NSString.class] + ? (NSString *)configuration[FBBroadcastKeyCodec] + : @""; + FBVideoCodec codec = [FBBroadcastCodecH265 isEqualToString:codecName] + ? FBVideoCodecH265 + : FBVideoCodecH264; + if (width == 0 || height == 0) { + if (error) { + *error = [NSError errorWithDomain:FBExtPipelineErrorDomain + code:1 + userInfo:@{NSLocalizedDescriptionKey: @"Session configuration is missing positive 'width'/'height'"}]; + } + return nil; + } + _fps = fps; + + OSStatus status = VTPixelTransferSessionCreate(kCFAllocatorDefault, &_transferSession); + if (status != noErr || NULL == _transferSession) { + if (error) { + *error = [NSError errorWithDomain:FBExtPipelineErrorDomain + code:status + userInfo:@{NSLocalizedDescriptionKey: [NSString stringWithFormat:@"Cannot create a pixel transfer session (status %d)", (int)status]}]; + } + return nil; + } + // Letterbox to preserve the aspect ratio, matching the WDA-side screenshot converter. + VTSessionSetProperty(_transferSession, kVTPixelTransferPropertyKey_ScalingMode, kVTScalingMode_Letterbox); + + NSDictionary *pixelBufferAttributes = @{ + (id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange), + (id)kCVPixelBufferWidthKey: @(width), + (id)kCVPixelBufferHeightKey: @(height), + (id)kCVPixelBufferIOSurfacePropertiesKey: @{}, + }; + NSDictionary *poolAttributes = @{(id)kCVPixelBufferPoolMinimumBufferCountKey: @1}; + CVReturn poolStatus = CVPixelBufferPoolCreate(kCFAllocatorDefault, + (__bridge CFDictionaryRef)poolAttributes, + (__bridge CFDictionaryRef)pixelBufferAttributes, + &_bufferPool); + if (poolStatus != kCVReturnSuccess || NULL == _bufferPool) { + VTPixelTransferSessionInvalidate(_transferSession); + CFRelease(_transferSession); + _transferSession = NULL; + if (error) { + *error = [NSError errorWithDomain:FBExtPipelineErrorDomain + code:poolStatus + userInfo:@{NSLocalizedDescriptionKey: [NSString stringWithFormat:@"Cannot create a pixel buffer pool (status %d)", (int)poolStatus]}]; + } + return nil; + } + + FBVideoEncoder *encoder = [[FBVideoEncoder alloc] initWithCodec:codec + width:width + height:height + bitrate:bitrate > 0 ? bitrate : 6000000 + fps:fps > 0 ? fps : 30 + error:error]; + if (nil == encoder) { + [self releaseScalerResources]; + return nil; + } + encoder.delegate = self; + _encoder = encoder; + _active = YES; + // Open every session with an IDR: WDA only switches a stream onto the broadcast source once + // a key frame (with parameter sets) has arrived. + [encoder requestKeyFrame]; + } + return self; +} + +- (void)submitSampleBuffer:(CMSampleBufferRef)sampleBuffer orientation:(uint8_t)orientation +{ + if (!self.active) { + return; + } + self.currentOrientation = orientation; + + // Respect this session's framerate even though ReplayKit may deliver faster. + uint64_t nowMs = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) / NSEC_PER_MSEC; + uint64_t minIntervalMs = self.fps > 0 ? (uint64_t)(1000 / self.fps) : 0; + if (minIntervalMs > 1 && nowMs - self.lastSubmitTimeMs < minIntervalMs - 1) { + return; + } + // Never queue and never block the ReplayKit sample queue: drop when the previous frame is + // still being scaled/submitted. + if (self.inFlight) { + return; + } + self.inFlight = YES; + self.lastSubmitTimeMs = nowMs; + + CFRetain(sampleBuffer); + dispatch_async(self.queue, ^{ + [self processRetainedSampleBuffer:sampleBuffer atTimeMs:nowMs]; + CFRelease(sampleBuffer); + self.inFlight = NO; + }); +} + +- (void)processRetainedSampleBuffer:(CMSampleBufferRef)sampleBuffer atTimeMs:(uint64_t)nowMs +{ + if (!self.active) { + return; + } + CVPixelBufferRef sourceBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + if (NULL == sourceBuffer) { + return; + } + + CVPixelBufferRef scaledBuffer = NULL; + NSDictionary *auxAttributes = @{(id)kCVPixelBufferPoolAllocationThresholdKey: @(POOL_ALLOCATION_THRESHOLD)}; + CVReturn poolStatus = CVPixelBufferPoolCreatePixelBufferWithAuxAttributes(kCFAllocatorDefault, + self.bufferPool, + (__bridge CFDictionaryRef)auxAttributes, + &scaledBuffer); + if (poolStatus != kCVReturnSuccess || NULL == scaledBuffer) { + // The pool is exhausted (encoder still holds the buffers) - drop the frame instead of growing. + return; + } + + OSStatus transferStatus = VTPixelTransferSessionTransferImage(self.transferSession, sourceBuffer, scaledBuffer); + if (transferStatus != noErr) { + FBExtLogError("Session %u: pixel transfer failed (status %d)", self.sessionId, (int)transferStatus); + CVPixelBufferRelease(scaledBuffer); + return; + } + + NSError *error; + if (![self.encoder encodePixelBuffer:scaledBuffer presentationTimeMs:nowMs error:&error]) { + FBExtLogError("Session %u: cannot encode a frame: %{public}@", self.sessionId, error.description); + } + CVPixelBufferRelease(scaledBuffer); +} + +- (void)requestKeyFrame +{ + [self.encoder requestKeyFrame]; +} + +- (void)teardown +{ + self.active = NO; + dispatch_async(self.queue, ^{ + if (nil != self.encoder) { + self.encoder.delegate = nil; + [self.encoder stop]; + self.encoder = nil; + } + [self releaseScalerResources]; + }); +} + +- (void)releaseScalerResources +{ + if (NULL != self.transferSession) { + VTPixelTransferSessionInvalidate(self.transferSession); + CFRelease(self.transferSession); + self.transferSession = NULL; + } + if (NULL != self.bufferPool) { + CVPixelBufferPoolRelease(self.bufferPool); + self.bufferPool = NULL; + } +} + +- (void)dealloc +{ + if (nil != _encoder) { + _encoder.delegate = nil; + [_encoder stop]; + } + [self releaseScalerResources]; +} + +#pragma mark - + +- (void)videoEncoder:(FBVideoEncoder *)encoder + didEncodeFrame:(NSData *)annexBPictureData + isKeyFrame:(BOOL)isKeyFrame + presentationTimeUs:(uint64_t)presentationTimeUs +{ + if (!self.active || annexBPictureData.length == 0) { + return; + } + id sink = self.sink; + if (nil == sink) { + return; + } + + // WDA needs the parameter sets before the first IDR that uses them. + if (isKeyFrame) { + NSData *parameterSets = encoder.parameterSetAnnexB; + NSData *lastSent = self.lastSentParameterSets; + if (parameterSets.length > 0 && (nil == lastSent || ![parameterSets isEqualToData:lastSent])) { + self.lastSentParameterSets = parameterSets; + [sink sendProtocolMessage:FBBroadcastEncodeMessage(FBBroadcastMessageTypeVideoParams, + self.sessionId, + parameterSets) + isDroppable:NO]; + } + } + + NSData *payload = FBBroadcastEncodeVideoFramePayload(presentationTimeUs, + isKeyFrame, + self.currentOrientation, + annexBPictureData); + [sink sendProtocolMessage:FBBroadcastEncodeMessage(FBBroadcastMessageTypeVideoFrame, self.sessionId, payload) + isDroppable:!isKeyFrame]; +} + +@end diff --git a/WebDriverAgentBroadcast/Info.plist b/WebDriverAgentBroadcast/Info.plist new file mode 100644 index 000000000..acc5d9fe0 --- /dev/null +++ b/WebDriverAgentBroadcast/Info.plist @@ -0,0 +1,35 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleDisplayName + WebDriverAgent Broadcast + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + XPC! + CFBundleShortVersionString + 1.0 + CFBundleSignature + ???? + CFBundleVersion + 1 + NSExtension + + NSExtensionPointIdentifier + com.apple.broadcast-services-upload + NSExtensionPrincipalClass + FBBroadcastSampleHandler + RPBroadcastProcessMode + RPBroadcastProcessModeSampleBuffer + + + diff --git a/WebDriverAgentLib/Commands/FBScreenCaptureCommands.m b/WebDriverAgentLib/Commands/FBScreenCaptureCommands.m index d4f79a0c9..0b77267d7 100644 --- a/WebDriverAgentLib/Commands/FBScreenCaptureCommands.m +++ b/WebDriverAgentLib/Commands/FBScreenCaptureCommands.m @@ -8,6 +8,7 @@ #import "FBScreenCaptureCommands.h" +#import "FBBroadcastManager.h" #import "FBConfiguration.h" #import "FBRouteRequest.h" #import "FBVideoStreamManager.h" @@ -23,6 +24,16 @@ + (NSArray *)routes { return @[ + // The broadcast routes must be registered before the '/:id' routes: RoutingHTTPServer + // matches routes in registration order, so 'GET /mobilerun/screencapture/broadcast' would + // otherwise be swallowed by 'GET /mobilerun/screencapture/:id'. + [[FBRoute POST:@"/mobilerun/screencapture/broadcast/start"] respondWithTarget:self action:@selector(handleStartBroadcast:)], + [[FBRoute POST:@"/mobilerun/screencapture/broadcast/stop"] respondWithTarget:self action:@selector(handleStopBroadcast:)], + [[FBRoute GET:@"/mobilerun/screencapture/broadcast"] respondWithTarget:self action:@selector(handleGetBroadcastStatus:)], + [[FBRoute POST:@"/mobilerun/screencapture/broadcast/start"].withoutSession respondWithTarget:self action:@selector(handleStartBroadcast:)], + [[FBRoute POST:@"/mobilerun/screencapture/broadcast/stop"].withoutSession respondWithTarget:self action:@selector(handleStopBroadcast:)], + [[FBRoute GET:@"/mobilerun/screencapture/broadcast"].withoutSession respondWithTarget:self action:@selector(handleGetBroadcastStatus:)], + [[FBRoute POST:@"/mobilerun/screencapture/start"] respondWithTarget:self action:@selector(handleStartScreenCapture:)], [[FBRoute POST:@"/mobilerun/screencapture/stop"] respondWithTarget:self action:@selector(handleStopAllScreenCapture:)], [[FBRoute GET:@"/mobilerun/screencapture"] respondWithTarget:self action:@selector(handleListScreenCapture:)], @@ -41,6 +52,59 @@ + (NSArray *)routes #pragma mark - Commands ++ (id)handleStartBroadcast:(FBRouteRequest *)request +{ + NSTimeInterval timeout = 30.0; + NSNumber *timeoutArg = request.arguments[@"timeout"]; + if ([timeoutArg isKindOfClass:NSNumber.class] && timeoutArg.doubleValue > 0) { + timeout = timeoutArg.doubleValue; + } + NSMutableArray *confirmButtonLabels = [NSMutableArray array]; + id labelsArg = request.arguments[@"confirmButtonLabels"]; + if ([labelsArg isKindOfClass:NSArray.class]) { + for (id label in (NSArray *)labelsArg) { + if ([label isKindOfClass:NSString.class] && [(NSString *)label length] > 0) { + [confirmButtonLabels addObject:label]; + } + } + } + NSNumber *restoreArg = request.arguments[@"restoreForegroundApp"]; + BOOL restoreForegroundApp = [restoreArg isKindOfClass:NSNumber.class] ? restoreArg.boolValue : YES; + + NSError *error; + if (![FBBroadcastManager.sharedInstance startBroadcastWithTimeout:timeout + confirmButtonLabels:confirmButtonLabels + restoreForegroundApp:restoreForegroundApp + error:&error]) { + if ([error.domain isEqualToString:FBBroadcastManagerErrorDomain]) { + switch (error.code) { + case FBBroadcastManagerErrorUnsupported: + return FBResponseWithStatus([FBCommandStatus unsupportedOperationErrorWithMessage:error.localizedDescription traceback:nil]); + case FBBroadcastManagerErrorTimeout: + return FBResponseWithStatus([FBCommandStatus timeoutErrorWithMessage:error.localizedDescription traceback:nil]); + default: + break; + } + } + return FBResponseWithUnknownError(error); + } + return FBResponseWithObject([FBBroadcastManager.sharedInstance statusDictionary]); +} + ++ (id)handleStopBroadcast:(FBRouteRequest *)request +{ + NSError *error; + if (![FBBroadcastManager.sharedInstance stopBroadcastWithError:&error]) { + return FBResponseWithStatus([FBCommandStatus timeoutErrorWithMessage:error.localizedDescription traceback:nil]); + } + return FBResponseWithObject([FBBroadcastManager.sharedInstance statusDictionary]); +} + ++ (id)handleGetBroadcastStatus:(FBRouteRequest *)request +{ + return FBResponseWithObject([FBBroadcastManager.sharedInstance statusDictionary]); +} + + (id)handleStartScreenCapture:(FBRouteRequest *)request { FBVideoCodec codec; diff --git a/WebDriverAgentLib/Routing/FBWebServer.m b/WebDriverAgentLib/Routing/FBWebServer.m index 9c37d2f5b..29bdaa6f6 100644 --- a/WebDriverAgentLib/Routing/FBWebServer.m +++ b/WebDriverAgentLib/Routing/FBWebServer.m @@ -11,6 +11,7 @@ #import "RoutingConnection.h" #import "RoutingHTTPServer.h" +#import "FBBroadcastManager.h" #import "FBCommandHandler.h" #import "FBErrorBuilder.h" #import "FBExceptionHandler.h" @@ -79,6 +80,8 @@ - (void)startServing self.exceptionHandler = [FBExceptionHandler new]; [self startHTTPServer]; [self initScreenshotsBroadcaster]; + // Listen permanently so broadcasts started from Control Center attach as well. + [FBBroadcastManager.sharedInstance startListening]; self.keepAlive = YES; NSRunLoop *runLoop = [NSRunLoop mainRunLoop]; @@ -179,6 +182,7 @@ - (void)stopServing { [FBSession.activeSession kill]; [FBVideoStreamManager.sharedInstance stopAllSessions]; + [FBBroadcastManager.sharedInstance stopListening]; [self stopScreenshotsBroadcaster]; if (self.server.isRunning) { [self.server stop:NO]; diff --git a/WebDriverAgentLib/Utilities/FBBroadcastControlServer.h b/WebDriverAgentLib/Utilities/FBBroadcastControlServer.h new file mode 100644 index 000000000..7d2635dc3 --- /dev/null +++ b/WebDriverAgentLib/Utilities/FBBroadcastControlServer.h @@ -0,0 +1,84 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import + +#import "FBBroadcastProtocol.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + Callbacks are invoked on the server's internal serial queue. + */ +@protocol FBBroadcastControlServerDelegate + +/** The extension connected and sent its HELLO message. */ +- (void)broadcastServerDidConnect:(NSDictionary *)helloInfo; + +/** A periodic heartbeat with the extension's stats. */ +- (void)broadcastServerDidReceiveHeartbeat:(NSDictionary *)heartbeat; + +/** A broadcast lifecycle event (paused/resumed/finishing). */ +- (void)broadcastServerDidReceiveStatus:(NSDictionary *)status; + +/** The extension could not serve the given session (e.g. its encoder failed). */ +- (void)broadcastServerDidReceiveSessionError:(NSString *)message forSession:(uint32_t)sessionId; + +/** Fresh Annex-B parameter sets for the given session. */ +- (void)broadcastServerDidReceiveParameterSets:(NSData *)parameterSets forSession:(uint32_t)sessionId; + +/** An encoded picture for the given session. */ +- (void)broadcastServerDidReceiveFrame:(NSData *)annexBPictureData + isKeyFrame:(BOOL)isKeyFrame + ptsUs:(uint64_t)ptsUs + orientation:(uint8_t)orientation + forSession:(uint32_t)sessionId; + +/** The extension disconnected or went silent (watchdog). */ +- (void)broadcastServerDidDisconnect; + +@end + +/** + The WDA endpoint of the broadcast-extension control connection: listens on the loopback + control port, accepts a single extension connection at a time and speaks the + FBBroadcastProtocol wire format in both directions. The existing FBTCPSocket cannot be used + here because its delegate API discards the received bytes. + */ +@interface FBBroadcastControlServer : NSObject + +@property (nonatomic, weak) id delegate; + +/** YES while an extension connection is established (HELLO received, watchdog content). */ +@property (atomic, readonly) BOOL isExtensionConnected; + +- (instancetype)initWithPort:(uint16_t)port; + +- (instancetype)init NS_UNAVAILABLE; + +/** Starts listening on 127.0.0.1:port. */ +- (BOOL)startWithError:(NSError **)error; + +/** Stops listening and drops any extension connection. */ +- (void)stop; + +/** Sends SESSION_ADD for the given session id with a JSON configuration payload. */ +- (void)sendSessionAdd:(uint32_t)sessionId configuration:(NSDictionary *)configuration; + +/** Sends SESSION_REMOVE for the given session id. */ +- (void)sendSessionRemove:(uint32_t)sessionId; + +/** Sends KEYFRAME_REQUEST for the given session id. */ +- (void)sendKeyframeRequest:(uint32_t)sessionId; + +/** Asks the extension to finish the broadcast. */ +- (void)sendStopBroadcast; + +@end + +NS_ASSUME_NONNULL_END diff --git a/WebDriverAgentLib/Utilities/FBBroadcastControlServer.m b/WebDriverAgentLib/Utilities/FBBroadcastControlServer.m new file mode 100644 index 000000000..95d6a6783 --- /dev/null +++ b/WebDriverAgentLib/Utilities/FBBroadcastControlServer.m @@ -0,0 +1,274 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import "FBBroadcastControlServer.h" + +#include + +#import "FBLogger.h" +#import "GCDAsyncSocket.h" + +static const long TAG_HEADER = 1; +static const long TAG_PAYLOAD = 2; + +// The extension heartbeats every 2s; treat the broadcast as dead after 6s of silence. +static const NSTimeInterval WATCHDOG_INTERVAL = 3.0; +static const NSTimeInterval STALENESS_TIMEOUT = 6.0; + +@interface FBBroadcastControlServer () + +@property (nonatomic, readonly) uint16_t port; +@property (nonatomic) dispatch_queue_t queue; +@property (nonatomic, nullable) GCDAsyncSocket *listenSocket; +@property (nonatomic, nullable) GCDAsyncSocket *extensionSocket; +@property (nonatomic, nullable) dispatch_source_t watchdogTimer; +@property (atomic, readwrite) BOOL isExtensionConnected; +@property (nonatomic) uint64_t lastMessageAtMs; +@property (nonatomic) FBBroadcastMessageHeader pendingHeader; + +@end + +@implementation FBBroadcastControlServer + +- (instancetype)initWithPort:(uint16_t)port +{ + if ((self = [super init])) { + _port = port; + _queue = dispatch_queue_create("wda.broadcast.control", DISPATCH_QUEUE_SERIAL); + } + return self; +} + +- (BOOL)startWithError:(NSError **)error +{ + self.listenSocket = [[GCDAsyncSocket alloc] initWithDelegate:self delegateQueue:self.queue]; + // Loopback only: the control port must never be reachable from the network. + if (![self.listenSocket acceptOnInterface:@"127.0.0.1" port:self.port error:error]) { + self.listenSocket = nil; + return NO; + } + [FBLogger logFmt:@"Broadcast control server is listening on 127.0.0.1:%d", self.port]; + return YES; +} + +- (void)stop +{ + dispatch_sync(self.queue, ^{ + [self stopWatchdog]; + [self dropExtensionSocketLocked]; + self.listenSocket.delegate = nil; + [self.listenSocket disconnect]; + self.listenSocket = nil; + }); +} + +#pragma mark - Outgoing messages + +- (void)sendMessage:(nullable NSData *)message +{ + if (nil == message) { + return; + } + dispatch_async(self.queue, ^{ + [self.extensionSocket writeData:(NSData *)message withTimeout:-1 tag:0]; + }); +} + +- (void)sendSessionAdd:(uint32_t)sessionId configuration:(NSDictionary *)configuration +{ + [self sendMessage:FBBroadcastEncodeJSONMessage(FBBroadcastMessageTypeSessionAdd, sessionId, configuration)]; +} + +- (void)sendSessionRemove:(uint32_t)sessionId +{ + [self sendMessage:FBBroadcastEncodeMessage(FBBroadcastMessageTypeSessionRemove, sessionId, nil)]; +} + +- (void)sendKeyframeRequest:(uint32_t)sessionId +{ + [self sendMessage:FBBroadcastEncodeMessage(FBBroadcastMessageTypeKeyframeRequest, sessionId, nil)]; +} + +- (void)sendStopBroadcast +{ + [self sendMessage:FBBroadcastEncodeMessage(FBBroadcastMessageTypeStopBroadcast, 0, nil)]; +} + +#pragma mark - Watchdog + +- (void)startWatchdog +{ + [self stopWatchdog]; + dispatch_source_t timer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, self.queue); + if (nil == timer) { + return; + } + dispatch_source_set_timer(timer, + dispatch_time(DISPATCH_TIME_NOW, (int64_t)(WATCHDOG_INTERVAL * NSEC_PER_SEC)), + (uint64_t)(WATCHDOG_INTERVAL * NSEC_PER_SEC), + (uint64_t)(0.5 * NSEC_PER_SEC)); + __weak typeof(self) weakSelf = self; + dispatch_source_set_event_handler(timer, ^{ + [weakSelf checkStaleness]; + }); + dispatch_resume(timer); + self.watchdogTimer = timer; +} + +- (void)stopWatchdog +{ + dispatch_source_t timer = self.watchdogTimer; + if (nil != timer) { + dispatch_source_cancel(timer); + self.watchdogTimer = nil; + } +} + +- (void)checkStaleness +{ + if (nil == self.extensionSocket) { + return; + } + uint64_t nowMs = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) / NSEC_PER_MSEC; + if (nowMs - self.lastMessageAtMs > (uint64_t)(STALENESS_TIMEOUT * 1000)) { + [FBLogger log:@"The broadcast extension went silent; dropping the connection"]; + [self dropExtensionSocketLocked]; + [self.delegate broadcastServerDidDisconnect]; + } +} + +- (void)dropExtensionSocketLocked +{ + if (nil == self.extensionSocket) { + return; + } + GCDAsyncSocket *socket = self.extensionSocket; + self.extensionSocket = nil; + self.isExtensionConnected = NO; + socket.delegate = nil; + [socket disconnect]; + [self stopWatchdog]; +} + +#pragma mark - Incoming messages + +- (void)handleMessageWithHeader:(FBBroadcastMessageHeader)header payload:(NSData *)payload +{ + id delegate = self.delegate; + switch (header.type) { + case FBBroadcastMessageTypeHello: { + NSDictionary *helloInfo = FBBroadcastParseJSONPayload(payload) ?: @{}; + self.isExtensionConnected = YES; + [delegate broadcastServerDidConnect:helloInfo]; + return; + } + case FBBroadcastMessageTypeHeartbeat: { + NSDictionary *heartbeat = FBBroadcastParseJSONPayload(payload); + if (nil != heartbeat) { + [delegate broadcastServerDidReceiveHeartbeat:heartbeat]; + } + return; + } + case FBBroadcastMessageTypeStatus: { + NSDictionary *status = FBBroadcastParseJSONPayload(payload); + if (nil != status) { + [delegate broadcastServerDidReceiveStatus:status]; + } + return; + } + case FBBroadcastMessageTypeSessionError: { + NSDictionary *info = FBBroadcastParseJSONPayload(payload); + NSString *message = [info[FBBroadcastKeyMessage] isKindOfClass:NSString.class] + ? (NSString *)info[FBBroadcastKeyMessage] + : @"The extension reported a session failure"; + [delegate broadcastServerDidReceiveSessionError:message forSession:header.sessionId]; + return; + } + case FBBroadcastMessageTypeVideoParams: + [delegate broadcastServerDidReceiveParameterSets:payload forSession:header.sessionId]; + return; + case FBBroadcastMessageTypeVideoFrame: { + uint64_t ptsUs = 0; + BOOL isKeyFrame = NO; + uint8_t orientation = 0; + NSData *annexB = nil; + if (!FBBroadcastParseVideoFramePayload(payload, &ptsUs, &isKeyFrame, &orientation, &annexB)) { + [FBLogger logFmt:@"Session %u: malformed VIDEO_FRAME payload", header.sessionId]; + return; + } + [delegate broadcastServerDidReceiveFrame:(NSData *)annexB + isKeyFrame:isKeyFrame + ptsUs:ptsUs + orientation:orientation + forSession:header.sessionId]; + return; + } + default: + [FBLogger logFmt:@"Ignoring an unexpected broadcast control message of type 0x%02x", header.type]; + return; + } +} + +#pragma mark - + +- (void)socket:(GCDAsyncSocket *)sock didAcceptNewSocket:(GCDAsyncSocket *)newSocket +{ + // A reconnecting extension supersedes the previous connection. + if (nil != self.extensionSocket) { + [FBLogger log:@"A new broadcast extension connection replaces the previous one"]; + [self dropExtensionSocketLocked]; + [self.delegate broadcastServerDidDisconnect]; + } + [FBLogger logFmt:@"The broadcast extension connected from %@:%d", newSocket.connectedHost, newSocket.connectedPort]; + self.extensionSocket = newSocket; + self.lastMessageAtMs = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) / NSEC_PER_MSEC; + [self startWatchdog]; + [newSocket readDataToLength:FBBroadcastHeaderLength withTimeout:-1 tag:TAG_HEADER]; +} + +- (void)socket:(GCDAsyncSocket *)sock didReadData:(NSData *)data withTag:(long)tag +{ + if (sock != self.extensionSocket) { + return; + } + self.lastMessageAtMs = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) / NSEC_PER_MSEC; + + if (tag == TAG_HEADER) { + FBBroadcastMessageHeader header; + if (!FBBroadcastParseHeader(data, &header)) { + [FBLogger log:@"Malformed broadcast control message header; dropping the connection"]; + [self dropExtensionSocketLocked]; + [self.delegate broadcastServerDidDisconnect]; + return; + } + self.pendingHeader = header; + if (header.payloadLength == 0) { + [self handleMessageWithHeader:header payload:NSData.data]; + [sock readDataToLength:FBBroadcastHeaderLength withTimeout:-1 tag:TAG_HEADER]; + } else { + [sock readDataToLength:header.payloadLength withTimeout:-1 tag:TAG_PAYLOAD]; + } + return; + } + [self handleMessageWithHeader:self.pendingHeader payload:data]; + [sock readDataToLength:FBBroadcastHeaderLength withTimeout:-1 tag:TAG_HEADER]; +} + +- (void)socketDidDisconnect:(GCDAsyncSocket *)sock withError:(nullable NSError *)error +{ + if (sock != self.extensionSocket) { + return; + } + [FBLogger logFmt:@"The broadcast extension disconnected: %@", error.description ?: @"closed"]; + self.extensionSocket = nil; + self.isExtensionConnected = NO; + [self stopWatchdog]; + [self.delegate broadcastServerDidDisconnect]; +} + +@end diff --git a/WebDriverAgentLib/Utilities/FBBroadcastManager.h b/WebDriverAgentLib/Utilities/FBBroadcastManager.h new file mode 100644 index 000000000..99e65733c --- /dev/null +++ b/WebDriverAgentLib/Utilities/FBBroadcastManager.h @@ -0,0 +1,86 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import + +#import "FBVideoStreamSession.h" + +NS_ASSUME_NONNULL_BEGIN + +extern NSErrorDomain const FBBroadcastManagerErrorDomain; + +typedef NS_ERROR_ENUM(FBBroadcastManagerErrorDomain, FBBroadcastManagerError) { + /** ReplayKit broadcasts are not available in this environment (Simulator/tvOS). */ + FBBroadcastManagerErrorUnsupported = 1, + /** The broadcast did not reach the connected state within the allotted time. */ + FBBroadcastManagerErrorTimeout = 2, + /** The system broadcast picker could not be driven. */ + FBBroadcastManagerErrorPicker = 3, +}; + +/** + Coordinates the ReplayKit broadcast extension: owns the loopback control server the extension + connects to, drives the system broadcast picker UI to start a broadcast, and routes the + extension's pre-encoded frames into the matching FBVideoStreamSession instances. + + The control server listens permanently (started from FBWebServer), so broadcasts started + manually from Control Center attach exactly like ones started via the HTTP endpoint. + */ +@interface FBBroadcastManager : NSObject + ++ (instancetype)sharedInstance; + +/** YES while the broadcast extension is connected to the control server. */ +@property (nonatomic, readonly) BOOL isExtensionConnected; + +/** Starts the loopback control server. Safe to call multiple times. */ +- (void)startListening; + +/** Stops the control server and drops the extension connection. */ +- (void)stopListening; + +/** @return A dictionary describing the broadcast state for the status endpoint. */ +- (NSDictionary *)statusDictionary; + +/** + Starts a system broadcast targeting the bundled extension by foregrounding the runner app, + triggering the broadcast picker and confirming the system sheet via UI automation, then waits + for the extension to connect. Must be called on the main thread. Idempotent while connected. + + @param timeout The overall time budget in seconds for the broadcast to reach the connected state + @param confirmButtonLabels Labels to look for on the system confirmation sheet + @param restoreForegroundApp YES to re-activate the previously active application afterwards + @param error If there is an error, upon return contains an NSError describing the problem + @return NO in case of a failure + */ +- (BOOL)startBroadcastWithTimeout:(NSTimeInterval)timeout + confirmButtonLabels:(NSArray *)confirmButtonLabels + restoreForegroundApp:(BOOL)restoreForegroundApp + error:(NSError **)error; + +/** + Asks the extension to finish the broadcast and waits for it to disconnect. + Idempotent when no broadcast is running. + + @param error If there is an error, upon return contains an NSError describing the problem + @return NO in case of a failure + */ +- (BOOL)stopBroadcastWithError:(NSError **)error; + +/** Notifies the manager that a capture session started (sends SESSION_ADD when connected). */ +- (void)notifySessionAdded:(FBVideoStreamSession *)session; + +/** Notifies the manager that a capture session stopped (sends SESSION_REMOVE when connected). */ +- (void)notifySessionRemoved:(NSUInteger)identifier; + +/** Forwards a key frame request for the given session to the extension. */ +- (void)requestKeyFrameForSession:(NSUInteger)identifier; + +@end + +NS_ASSUME_NONNULL_END diff --git a/WebDriverAgentLib/Utilities/FBBroadcastManager.m b/WebDriverAgentLib/Utilities/FBBroadcastManager.m new file mode 100644 index 000000000..711c5507d --- /dev/null +++ b/WebDriverAgentLib/Utilities/FBBroadcastManager.m @@ -0,0 +1,350 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import "FBBroadcastManager.h" + +#import "FBBroadcastControlServer.h" +#import "FBBroadcastPickerHost.h" +#import "FBBroadcastProtocol.h" +#import "FBConfiguration.h" +#import "FBLogger.h" +#import "FBRunLoopSpinner.h" +#import "FBVideoStreamManager.h" +#import "XCUIApplication.h" +#import "XCUIApplication+FBHelpers.h" + +NSErrorDomain const FBBroadcastManagerErrorDomain = @"com.facebook.WebDriverAgent.FBBroadcastManager"; + +#if !TARGET_OS_SIMULATOR && !TARGET_OS_TV +static const NSTimeInterval FOREGROUND_TIMEOUT = 5.0; +static const NSTimeInterval CONFIRM_BUTTON_TIMEOUT = 10.0; +#endif +static const NSTimeInterval STOP_TIMEOUT = 5.0; + +@interface FBBroadcastManager () + +@property (nonatomic, nullable) FBBroadcastControlServer *controlServer; +@property (atomic, nullable, copy) NSDictionary *helloInfo; +@property (atomic, nullable, copy) NSDictionary *lastHeartbeat; +@property (atomic, nullable) NSDate *connectedAt; +@property (atomic, nullable) NSDate *lastHeartbeatAt; +@property (atomic) BOOL paused; + +@end + +@implementation FBBroadcastManager + ++ (instancetype)sharedInstance +{ + static FBBroadcastManager *instance; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + instance = [[self alloc] init]; + }); + return instance; +} + +- (BOOL)isExtensionConnected +{ + return self.controlServer.isExtensionConnected; +} + +#pragma mark - Control server lifecycle + +- (void)startListening +{ + if (nil != self.controlServer) { + return; + } + uint16_t port = (uint16_t)FBConfiguration.broadcastControlPort; + FBBroadcastControlServer *server = [[FBBroadcastControlServer alloc] initWithPort:port]; + server.delegate = self; + NSError *error; + if (![server startWithError:&error]) { + [FBLogger logFmt:@"Cannot start the broadcast control server on port %d: %@", port, error.description]; + return; + } + self.controlServer = server; +} + +- (void)stopListening +{ + [self.controlServer stop]; + self.controlServer = nil; + [self resetConnectionState]; +} + +- (void)resetConnectionState +{ + self.helloInfo = nil; + self.lastHeartbeat = nil; + self.connectedAt = nil; + self.lastHeartbeatAt = nil; + self.paused = NO; +} + +#pragma mark - Status + +- (NSDictionary *)statusDictionary +{ + NSString *state = @"idle"; + if (self.isExtensionConnected) { + state = self.paused ? @"paused" : @"connected"; + } + NSDictionary *heartbeat = self.lastHeartbeat; + return @{ + @"state": state, + @"controlPort": @(FBConfiguration.broadcastControlPort), + @"preferredExtension": FBConfiguration.broadcastExtensionBundleId, + @"connectedAt": self.connectedAt ? @((uint64_t)(self.connectedAt.timeIntervalSince1970 * 1000)) : NSNull.null, + @"lastHeartbeatAt": self.lastHeartbeatAt ? @((uint64_t)(self.lastHeartbeatAt.timeIntervalSince1970 * 1000)) : NSNull.null, + @"hello": self.helloInfo ?: NSNull.null, + @"heartbeat": heartbeat ?: NSNull.null, + @"sessions": [FBVideoStreamManager.sharedInstance activeSessionsInfo], + }; +} + +#pragma mark - Broadcast start/stop + +- (BOOL)startBroadcastWithTimeout:(NSTimeInterval)timeout + confirmButtonLabels:(NSArray *)confirmButtonLabels + restoreForegroundApp:(BOOL)restoreForegroundApp + error:(NSError **)error +{ +#if TARGET_OS_SIMULATOR || TARGET_OS_TV + if (error) { + *error = [NSError errorWithDomain:FBBroadcastManagerErrorDomain + code:FBBroadcastManagerErrorUnsupported + userInfo:@{NSLocalizedDescriptionKey: @"ReplayKit broadcasts are only supported on physical iOS devices"}]; + } + return NO; +#else + if (self.isExtensionConnected) { + return YES; + } + if (nil == self.controlServer) { + [self startListening]; + } + + XCUIApplication *runner = [[XCUIApplication alloc] initWithBundleIdentifier:(NSString *)NSBundle.mainBundle.bundleIdentifier]; + XCUIApplication *previousApp = nil; + if (restoreForegroundApp) { + XCUIApplication *active = XCUIApplication.fb_activeApplication; + if (nil != active && ![active.bundleID isEqualToString:runner.bundleID]) { + previousApp = active; + } + } + + // The picker can only present from a foreground app, so bring the runner up first. + [runner activate]; + BOOL foregrounded = [[[[FBRunLoopSpinner new] timeout:FOREGROUND_TIMEOUT] interval:0.1] spinUntilTrue:^BOOL{ + return runner.state == XCUIApplicationStateRunningForeground; + }]; + if (!foregrounded) { + if (error) { + *error = [NSError errorWithDomain:FBBroadcastManagerErrorDomain + code:FBBroadcastManagerErrorTimeout + userInfo:@{NSLocalizedDescriptionKey: @"The runner app could not be brought to the foreground to present the broadcast picker"}]; + } + return NO; + } + + NSError *pickerError; + if (![FBBroadcastPickerHost triggerPickerWithPreferredExtension:FBConfiguration.broadcastExtensionBundleId + error:&pickerError]) { + if (error) { + *error = [NSError errorWithDomain:FBBroadcastManagerErrorDomain + code:FBBroadcastManagerErrorPicker + userInfo:@{NSLocalizedDescriptionKey: pickerError.localizedDescription ?: @"Cannot trigger the broadcast picker"}]; + } + return NO; + } + + // The confirmation sheet is hosted by different processes depending on the iOS version, so + // look for the confirm button in both the system app and the runner itself. + NSArray *labels = confirmButtonLabels.count > 0 ? confirmButtonLabels : @[@"Start Broadcast"]; + __block XCUIElement *confirmButton = nil; + [[[[FBRunLoopSpinner new] timeout:CONFIRM_BUTTON_TIMEOUT] interval:0.3] spinUntilTrue:^BOOL{ + for (XCUIApplication *app in @[XCUIApplication.fb_systemApplication, runner]) { + for (NSString *label in labels) { + XCUIElement *candidate = app.buttons[label]; + if (candidate.exists) { + confirmButton = candidate; + return YES; + } + } + XCUIElement *prefixMatch = [app.buttons matchingPredicate:[NSPredicate predicateWithFormat:@"label BEGINSWITH[c] 'Start'"]].firstMatch; + if (prefixMatch.exists) { + confirmButton = prefixMatch; + return YES; + } + } + return NO; + }]; + if (nil == confirmButton) { + [FBBroadcastPickerHost dismiss]; + if (error) { + *error = [NSError errorWithDomain:FBBroadcastManagerErrorDomain + code:FBBroadcastManagerErrorTimeout + userInfo:@{NSLocalizedDescriptionKey: [NSString stringWithFormat:@"The broadcast confirmation sheet did not show a button labeled %@ within %.0fs. Pass 'confirmButtonLabels' if the device language is not English", [labels componentsJoinedByString:@"/"], CONFIRM_BUTTON_TIMEOUT]}]; + } + return NO; + } + [confirmButton tap]; + + // Cover the system's 3-2-1 countdown plus the extension's connect/HELLO round trip. + BOOL connected = [[[[FBRunLoopSpinner new] timeout:(timeout > 0 ? timeout : 30.0)] interval:0.3] spinUntilTrue:^BOOL{ + return self.isExtensionConnected; + }]; + [FBBroadcastPickerHost dismiss]; + if (!connected) { + if (error) { + *error = [NSError errorWithDomain:FBBroadcastManagerErrorDomain + code:FBBroadcastManagerErrorTimeout + userInfo:@{NSLocalizedDescriptionKey: @"The broadcast was confirmed but the extension did not connect in time. Check that the extension is embedded and signed correctly (see docs/broadcast-extension.md)"}]; + } + return NO; + } + + if (nil != previousApp) { + [previousApp activate]; + } + return YES; +#endif +} + +- (BOOL)stopBroadcastWithError:(NSError **)error +{ + if (!self.isExtensionConnected) { + return YES; + } + [self.controlServer sendStopBroadcast]; + BOOL stopped = [[[[FBRunLoopSpinner new] timeout:STOP_TIMEOUT] interval:0.2] spinUntilTrue:^BOOL{ + return !self.isExtensionConnected; + }]; + if (!stopped) { + if (error) { + *error = [NSError errorWithDomain:FBBroadcastManagerErrorDomain + code:FBBroadcastManagerErrorTimeout + userInfo:@{NSLocalizedDescriptionKey: @"The broadcast extension did not finish the broadcast in time"}]; + } + return NO; + } + return YES; +} + +#pragma mark - Session bridging + ++ (NSDictionary *)sessionAddPayloadForConfiguration:(FBScreenCaptureConfiguration *)configuration +{ + return @{ + FBBroadcastKeyWidth: @(configuration.width), + FBBroadcastKeyHeight: @(configuration.height), + FBBroadcastKeyCodec: configuration.codec == FBVideoCodecH265 ? FBBroadcastCodecH265 : FBBroadcastCodecH264, + FBBroadcastKeyBitrate: @(configuration.bitrate), + FBBroadcastKeyFps: @(configuration.fps), + }; +} + +- (void)notifySessionAdded:(FBVideoStreamSession *)session +{ + if (!self.isExtensionConnected) { + return; + } + [self.controlServer sendSessionAdd:(uint32_t)session.identifier + configuration:[self.class sessionAddPayloadForConfiguration:session.configuration]]; +} + +- (void)notifySessionRemoved:(NSUInteger)identifier +{ + if (!self.isExtensionConnected) { + return; + } + [self.controlServer sendSessionRemove:(uint32_t)identifier]; +} + +- (void)requestKeyFrameForSession:(NSUInteger)identifier +{ + [self.controlServer sendKeyframeRequest:(uint32_t)identifier]; +} + +#pragma mark - + +- (void)broadcastServerDidConnect:(NSDictionary *)helloInfo +{ + [FBLogger logFmt:@"The broadcast extension connected: %@", helloInfo]; + self.helloInfo = helloInfo; + self.connectedAt = NSDate.date; + self.paused = NO; + // Attach every live capture session to the broadcast source. + for (FBVideoStreamSession *session in [FBVideoStreamManager.sharedInstance activeSessions]) { + [self.controlServer sendSessionAdd:(uint32_t)session.identifier + configuration:[self.class sessionAddPayloadForConfiguration:session.configuration]]; + } +} + +- (void)broadcastServerDidReceiveHeartbeat:(NSDictionary *)heartbeat +{ + self.lastHeartbeat = heartbeat; + self.lastHeartbeatAt = NSDate.date; + self.paused = [@"paused" isEqualToString:(NSString *)(heartbeat[FBBroadcastKeyState] ?: @"")]; +} + +- (void)broadcastServerDidReceiveStatus:(NSDictionary *)status +{ + NSString *event = status[FBBroadcastKeyEvent]; + [FBLogger logFmt:@"Broadcast status event: %@", status]; + if ([@"paused" isEqualToString:event]) { + self.paused = YES; + } else if ([@"resumed" isEqualToString:event]) { + self.paused = NO; + } +} + +- (void)broadcastServerDidReceiveSessionError:(NSString *)message forSession:(uint32_t)sessionId +{ + [FBLogger logFmt:@"The broadcast extension cannot serve session %u: %@", sessionId, message]; + FBVideoStreamSession *session = [FBVideoStreamManager.sharedInstance sessionWithIdentifier:sessionId]; + [session detachBroadcastSourceAndForceKeyFrame]; +} + +- (void)broadcastServerDidReceiveParameterSets:(NSData *)parameterSets forSession:(uint32_t)sessionId +{ + FBVideoStreamSession *session = [FBVideoStreamManager.sharedInstance sessionWithIdentifier:sessionId]; + if (nil == session) { + [self.controlServer sendSessionRemove:sessionId]; + return; + } + [session ingestBroadcastParameterSets:parameterSets]; +} + +- (void)broadcastServerDidReceiveFrame:(NSData *)annexBPictureData + isKeyFrame:(BOOL)isKeyFrame + ptsUs:(uint64_t)ptsUs + orientation:(uint8_t)orientation + forSession:(uint32_t)sessionId +{ + FBVideoStreamSession *session = [FBVideoStreamManager.sharedInstance sessionWithIdentifier:sessionId]; + if (nil == session) { + // The session is gone (stale extension pipeline); ask the extension to drop it. + [self.controlServer sendSessionRemove:sessionId]; + return; + } + [session ingestBroadcastFrame:annexBPictureData isKeyFrame:isKeyFrame]; +} + +- (void)broadcastServerDidDisconnect +{ + [FBLogger log:@"The broadcast extension disconnected; reverting sessions to the screenshot source"]; + [self resetConnectionState]; + for (FBVideoStreamSession *session in [FBVideoStreamManager.sharedInstance activeSessions]) { + [session detachBroadcastSourceAndForceKeyFrame]; + } +} + +@end diff --git a/WebDriverAgentLib/Utilities/FBBroadcastPickerHost.h b/WebDriverAgentLib/Utilities/FBBroadcastPickerHost.h new file mode 100644 index 000000000..837bbddc9 --- /dev/null +++ b/WebDriverAgentLib/Utilities/FBBroadcastPickerHost.h @@ -0,0 +1,35 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import + +NS_ASSUME_NONNULL_BEGIN + +/** + Hosts an (effectively invisible) RPSystemBroadcastPickerView inside the runner app and + triggers its button programmatically, which makes the system present the broadcast + confirmation sheet. Main thread only. + */ +@interface FBBroadcastPickerHost : NSObject + +/** + Creates the hosting window if needed and taps the picker's internal button. + + @param preferredExtension The bundle identifier of the broadcast upload extension to preselect + @param error Set when the picker cannot be hosted or its button cannot be found + @return NO in case of a failure + */ ++ (BOOL)triggerPickerWithPreferredExtension:(NSString *)preferredExtension + error:(NSError **)error; + +/** Hides and releases the hosting window. */ ++ (void)dismiss; + +@end + +NS_ASSUME_NONNULL_END diff --git a/WebDriverAgentLib/Utilities/FBBroadcastPickerHost.m b/WebDriverAgentLib/Utilities/FBBroadcastPickerHost.m new file mode 100644 index 000000000..26766eaa6 --- /dev/null +++ b/WebDriverAgentLib/Utilities/FBBroadcastPickerHost.m @@ -0,0 +1,129 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import "FBBroadcastPickerHost.h" + +#import +#if !TARGET_OS_TV +#import +#endif + +#import "FBLogger.h" + +static NSString *const FBBroadcastPickerHostErrorDomain = @"com.facebook.WebDriverAgent.FBBroadcastPickerHost"; + +#if !TARGET_OS_TV +static UIWindow *pickerWindow = nil; +#endif + +@implementation FBBroadcastPickerHost + +#if TARGET_OS_TV + ++ (BOOL)triggerPickerWithPreferredExtension:(NSString *)preferredExtension + error:(NSError **)error +{ + if (error) { + *error = [NSError errorWithDomain:FBBroadcastPickerHostErrorDomain + code:1 + userInfo:@{NSLocalizedDescriptionKey: @"The ReplayKit broadcast picker is not available on tvOS"}]; + } + return NO; +} + ++ (void)dismiss +{ +} + +#else + ++ (BOOL)triggerPickerWithPreferredExtension:(NSString *)preferredExtension + error:(NSError **)error +{ + NSAssert(NSThread.isMainThread, @"The broadcast picker must be triggered on the main thread"); + + if (nil == pickerWindow) { + // The picker view must live in a visible window for the system sheet to present, but the + // window can be tiny and nearly transparent so it never interferes with the runner's UI. + UIWindowScene *foregroundScene = nil; + for (UIScene *scene in UIApplication.sharedApplication.connectedScenes) { + if ([scene isKindOfClass:UIWindowScene.class] + && scene.activationState == UISceneActivationStateForegroundActive) { + foregroundScene = (UIWindowScene *)scene; + break; + } + } + UIWindow *window = nil == foregroundScene + ? [[UIWindow alloc] initWithFrame:CGRectMake(0, 0, 2, 2)] + : [[UIWindow alloc] initWithWindowScene:foregroundScene]; + window.frame = CGRectMake(0, 0, 2, 2); + window.windowLevel = UIWindowLevelNormal; + window.alpha = 0.02; + window.rootViewController = [[UIViewController alloc] init]; + + RPSystemBroadcastPickerView *pickerView = [[RPSystemBroadcastPickerView alloc] initWithFrame:CGRectMake(0, 0, 2, 2)]; + pickerView.preferredExtension = preferredExtension; + pickerView.showsMicrophoneButton = NO; + [window.rootViewController.view addSubview:pickerView]; + + pickerWindow = window; + } + [pickerWindow makeKeyAndVisible]; + + RPSystemBroadcastPickerView *pickerView = nil; + for (UIView *subview in pickerWindow.rootViewController.view.subviews) { + if ([subview isKindOfClass:RPSystemBroadcastPickerView.class]) { + pickerView = (RPSystemBroadcastPickerView *)subview; + break; + } + } + pickerView.preferredExtension = preferredExtension; + + UIButton *button = [self findButtonInView:pickerView]; + if (nil == button) { + if (error) { + *error = [NSError errorWithDomain:FBBroadcastPickerHostErrorDomain + code:2 + userInfo:@{NSLocalizedDescriptionKey: @"Cannot locate the button inside RPSystemBroadcastPickerView. The system view layout may have changed in this iOS version"}]; + } + return NO; + } + [FBLogger log:@"Triggering the system broadcast picker"]; + [button sendActionsForControlEvents:UIControlEventTouchUpInside]; + return YES; +} + ++ (nullable UIButton *)findButtonInView:(nullable UIView *)view +{ + if (nil == view) { + return nil; + } + if ([view isKindOfClass:UIButton.class]) { + return (UIButton *)view; + } + for (UIView *subview in view.subviews) { + UIButton *button = [self findButtonInView:subview]; + if (nil != button) { + return button; + } + } + return nil; +} + ++ (void)dismiss +{ + if (nil == pickerWindow) { + return; + } + pickerWindow.hidden = YES; + pickerWindow = nil; +} + +#endif + +@end diff --git a/WebDriverAgentLib/Utilities/FBBroadcastProtocol.h b/WebDriverAgentLib/Utilities/FBBroadcastProtocol.h new file mode 100644 index 000000000..b9159c67c --- /dev/null +++ b/WebDriverAgentLib/Utilities/FBBroadcastProtocol.h @@ -0,0 +1,140 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import + +NS_ASSUME_NONNULL_BEGIN + +/** + Wire protocol shared between WebDriverAgent and the ReplayKit broadcast upload extension. + + This file is compiled into both WebDriverAgentLib and the WebDriverAgentBroadcast extension + target, so it must only depend on Foundation. + + Transport: a single TCP connection on loopback. The extension is the client, WDA is the + server. Every message is a fixed 16-byte big-endian header followed by a payload: + + offset 0 uint32 magic = 'WDAB' + offset 4 uint8 protocolVersion + offset 5 uint8 messageType + offset 6 uint16 reserved (0) + offset 8 uint32 sessionId (0 when not applicable) + offset 12 uint32 payloadLength + offset 16 ... payload + + Control payloads are UTF-8 JSON; the high-rate VIDEO_FRAME payload is fixed binary + (see FBBroadcastEncodeVideoFramePayload). + */ + +/** 'WDAB' */ +extern const uint32_t FBBroadcastProtocolMagic; +extern const uint8_t FBBroadcastProtocolVersion; +/** The fixed message header length in bytes. */ +extern const NSUInteger FBBroadcastHeaderLength; +/** The loopback TCP port WDA listens on for the extension connection. */ +extern const uint16_t FBBroadcastDefaultControlPort; + +typedef NS_ENUM(uint8_t, FBBroadcastMessageType) { + // WDA -> extension + /** JSON {width, height, codec, bitrate, fps}; sessionId in the header. */ + FBBroadcastMessageTypeSessionAdd = 0x01, + /** Empty payload; sessionId in the header. */ + FBBroadcastMessageTypeSessionRemove = 0x02, + /** Empty payload; sessionId in the header. */ + FBBroadcastMessageTypeKeyframeRequest = 0x03, + /** Empty payload. Asks the extension to finish the broadcast. */ + FBBroadcastMessageTypeStopBroadcast = 0x04, + + // extension -> WDA + /** JSON {protocolVersion, osVersion}. Sent once right after connecting. */ + FBBroadcastMessageTypeHello = 0x81, + /** JSON {state, framesReceived, orientation, screenWidth, screenHeight}. Sent every 2s. */ + FBBroadcastMessageTypeHeartbeat = 0x82, + /** Raw Annex-B parameter sets (VPS/SPS/PPS) for sessionId. Sent before the IDR that uses them. */ + FBBroadcastMessageTypeVideoParams = 0x83, + /** Binary: [8B ptsUs BE][1B flags][Annex-B VCL NAL units]; sessionId in the header. */ + FBBroadcastMessageTypeVideoFrame = 0x84, + /** JSON {event: "paused"|"resumed"|"finishing", reason}. */ + FBBroadcastMessageTypeStatus = 0x85, + /** JSON {message} for sessionId (e.g. the per-session encoder could not be created). */ + FBBroadcastMessageTypeSessionError = 0x86, +}; + +/** A parsed message header. */ +typedef struct { + uint8_t version; + uint8_t type; + uint32_t sessionId; + uint32_t payloadLength; +} FBBroadcastMessageHeader; + +/** VIDEO_FRAME flags bit 0: the frame is a key (IDR) frame. */ +extern const uint8_t FBBroadcastFrameFlagKeyFrame; +/** VIDEO_FRAME flags bits 1-3: CGImagePropertyOrientation (1-8) of the captured frame. */ +extern const uint8_t FBBroadcastFrameOrientationShift; +extern const uint8_t FBBroadcastFrameOrientationMask; + +/** JSON keys for SESSION_ADD / HELLO / HEARTBEAT / STATUS payloads. */ +extern NSString *const FBBroadcastKeyWidth; +extern NSString *const FBBroadcastKeyHeight; +extern NSString *const FBBroadcastKeyCodec; +extern NSString *const FBBroadcastKeyBitrate; +extern NSString *const FBBroadcastKeyFps; +extern NSString *const FBBroadcastKeyProtocolVersion; +extern NSString *const FBBroadcastKeyOsVersion; +extern NSString *const FBBroadcastKeyState; +extern NSString *const FBBroadcastKeyFramesReceived; +extern NSString *const FBBroadcastKeyOrientation; +extern NSString *const FBBroadcastKeyScreenWidth; +extern NSString *const FBBroadcastKeyScreenHeight; +extern NSString *const FBBroadcastKeyEvent; +extern NSString *const FBBroadcastKeyReason; +extern NSString *const FBBroadcastKeyMessage; + +/** Codec string values used in SESSION_ADD, matching the HTTP API. */ +extern NSString *const FBBroadcastCodecH264; +extern NSString *const FBBroadcastCodecH265; + +/** Builds a complete wire message (header + payload). */ +NSData *FBBroadcastEncodeMessage(FBBroadcastMessageType type, + uint32_t sessionId, + NSData *_Nullable payload); + +/** Builds a complete wire message with a JSON payload. Returns nil if serialization fails. */ +NSData *_Nullable FBBroadcastEncodeJSONMessage(FBBroadcastMessageType type, + uint32_t sessionId, + NSDictionary *payload); + +/** + Parses and validates a 16-byte header. + + @return NO when the data is too short, the magic does not match or the version is unsupported. + */ +BOOL FBBroadcastParseHeader(NSData *headerData, FBBroadcastMessageHeader *outHeader); + +/** Deserializes a JSON control payload. Returns nil when the payload is not a JSON object. */ +NSDictionary *_Nullable FBBroadcastParseJSONPayload(NSData *payload); + +/** Builds a VIDEO_FRAME payload: [8B ptsUs BE][1B flags][annexB]. */ +NSData *FBBroadcastEncodeVideoFramePayload(uint64_t ptsUs, + BOOL isKeyFrame, + uint8_t orientation, + NSData *annexBPictureData); + +/** + Parses a VIDEO_FRAME payload. + + @return NO when the payload is shorter than its fixed prefix. + */ +BOOL FBBroadcastParseVideoFramePayload(NSData *payload, + uint64_t *outPtsUs, + BOOL *outIsKeyFrame, + uint8_t *outOrientation, + NSData *_Nullable __autoreleasing *_Nonnull outAnnexB); + +NS_ASSUME_NONNULL_END diff --git a/WebDriverAgentLib/Utilities/FBBroadcastProtocol.m b/WebDriverAgentLib/Utilities/FBBroadcastProtocol.m new file mode 100644 index 000000000..301f33a79 --- /dev/null +++ b/WebDriverAgentLib/Utilities/FBBroadcastProtocol.m @@ -0,0 +1,147 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import "FBBroadcastProtocol.h" + +const uint32_t FBBroadcastProtocolMagic = 0x57444142; // 'WDAB' +const uint8_t FBBroadcastProtocolVersion = 1; +const NSUInteger FBBroadcastHeaderLength = 16; +const uint16_t FBBroadcastDefaultControlPort = 9300; + +const uint8_t FBBroadcastFrameFlagKeyFrame = 1 << 0; +const uint8_t FBBroadcastFrameOrientationShift = 1; +const uint8_t FBBroadcastFrameOrientationMask = 0x07; + +NSString *const FBBroadcastKeyWidth = @"width"; +NSString *const FBBroadcastKeyHeight = @"height"; +NSString *const FBBroadcastKeyCodec = @"codec"; +NSString *const FBBroadcastKeyBitrate = @"bitrate"; +NSString *const FBBroadcastKeyFps = @"fps"; +NSString *const FBBroadcastKeyProtocolVersion = @"protocolVersion"; +NSString *const FBBroadcastKeyOsVersion = @"osVersion"; +NSString *const FBBroadcastKeyState = @"state"; +NSString *const FBBroadcastKeyFramesReceived = @"framesReceived"; +NSString *const FBBroadcastKeyOrientation = @"orientation"; +NSString *const FBBroadcastKeyScreenWidth = @"screenWidth"; +NSString *const FBBroadcastKeyScreenHeight = @"screenHeight"; +NSString *const FBBroadcastKeyEvent = @"event"; +NSString *const FBBroadcastKeyReason = @"reason"; +NSString *const FBBroadcastKeyMessage = @"message"; + +NSString *const FBBroadcastCodecH264 = @"h264"; +NSString *const FBBroadcastCodecH265 = @"h265"; + +NSData *FBBroadcastEncodeMessage(FBBroadcastMessageType type, + uint32_t sessionId, + NSData *_Nullable payload) +{ + NSUInteger payloadLength = payload.length; + NSMutableData *message = [NSMutableData dataWithCapacity:FBBroadcastHeaderLength + payloadLength]; + + uint32_t bigMagic = CFSwapInt32HostToBig(FBBroadcastProtocolMagic); + [message appendBytes:&bigMagic length:sizeof(bigMagic)]; + uint8_t version = FBBroadcastProtocolVersion; + [message appendBytes:&version length:sizeof(version)]; + uint8_t messageType = (uint8_t)type; + [message appendBytes:&messageType length:sizeof(messageType)]; + uint16_t reserved = 0; + [message appendBytes:&reserved length:sizeof(reserved)]; + uint32_t bigSessionId = CFSwapInt32HostToBig(sessionId); + [message appendBytes:&bigSessionId length:sizeof(bigSessionId)]; + uint32_t bigPayloadLength = CFSwapInt32HostToBig((uint32_t)payloadLength); + [message appendBytes:&bigPayloadLength length:sizeof(bigPayloadLength)]; + + if (payloadLength > 0) { + [message appendData:(NSData *)payload]; + } + return message; +} + +NSData *_Nullable FBBroadcastEncodeJSONMessage(FBBroadcastMessageType type, + uint32_t sessionId, + NSDictionary *payload) +{ + NSData *json = [NSJSONSerialization dataWithJSONObject:payload options:(NSJSONWritingOptions)0 error:NULL]; + if (nil == json) { + return nil; + } + return FBBroadcastEncodeMessage(type, sessionId, json); +} + +BOOL FBBroadcastParseHeader(NSData *headerData, FBBroadcastMessageHeader *outHeader) +{ + if (headerData.length < FBBroadcastHeaderLength) { + return NO; + } + const uint8_t *bytes = (const uint8_t *)headerData.bytes; + + uint32_t magic; + memcpy(&magic, bytes, sizeof(magic)); + if (CFSwapInt32BigToHost(magic) != FBBroadcastProtocolMagic) { + return NO; + } + uint8_t version = bytes[4]; + if (version != FBBroadcastProtocolVersion) { + return NO; + } + + outHeader->version = version; + outHeader->type = bytes[5]; + uint32_t sessionId; + memcpy(&sessionId, bytes + 8, sizeof(sessionId)); + outHeader->sessionId = CFSwapInt32BigToHost(sessionId); + uint32_t payloadLength; + memcpy(&payloadLength, bytes + 12, sizeof(payloadLength)); + outHeader->payloadLength = CFSwapInt32BigToHost(payloadLength); + return YES; +} + +NSDictionary *_Nullable FBBroadcastParseJSONPayload(NSData *payload) +{ + if (payload.length == 0) { + return nil; + } + id parsed = [NSJSONSerialization JSONObjectWithData:payload options:(NSJSONReadingOptions)0 error:NULL]; + return [parsed isKindOfClass:NSDictionary.class] ? parsed : nil; +} + +NSData *FBBroadcastEncodeVideoFramePayload(uint64_t ptsUs, + BOOL isKeyFrame, + uint8_t orientation, + NSData *annexBPictureData) +{ + NSMutableData *payload = [NSMutableData dataWithCapacity:sizeof(uint64_t) + sizeof(uint8_t) + annexBPictureData.length]; + uint64_t bigPts = CFSwapInt64HostToBig(ptsUs); + [payload appendBytes:&bigPts length:sizeof(bigPts)]; + uint8_t flags = (isKeyFrame ? FBBroadcastFrameFlagKeyFrame : 0) + | (uint8_t)((orientation & FBBroadcastFrameOrientationMask) << FBBroadcastFrameOrientationShift); + [payload appendBytes:&flags length:sizeof(flags)]; + [payload appendData:annexBPictureData]; + return payload; +} + +BOOL FBBroadcastParseVideoFramePayload(NSData *payload, + uint64_t *outPtsUs, + BOOL *outIsKeyFrame, + uint8_t *outOrientation, + NSData *_Nullable __autoreleasing *_Nonnull outAnnexB) +{ + static const NSUInteger prefixLength = sizeof(uint64_t) + sizeof(uint8_t); + if (payload.length < prefixLength) { + return NO; + } + const uint8_t *bytes = (const uint8_t *)payload.bytes; + uint64_t bigPts; + memcpy(&bigPts, bytes, sizeof(bigPts)); + *outPtsUs = CFSwapInt64BigToHost(bigPts); + uint8_t flags = bytes[sizeof(uint64_t)]; + *outIsKeyFrame = (flags & FBBroadcastFrameFlagKeyFrame) != 0; + *outOrientation = (flags >> FBBroadcastFrameOrientationShift) & FBBroadcastFrameOrientationMask; + *outAnnexB = [payload subdataWithRange:NSMakeRange(prefixLength, payload.length - prefixLength)]; + return YES; +} diff --git a/WebDriverAgentLib/Utilities/FBConfiguration.h b/WebDriverAgentLib/Utilities/FBConfiguration.h index 25704d93d..4a2647c45 100644 --- a/WebDriverAgentLib/Utilities/FBConfiguration.h +++ b/WebDriverAgentLib/Utilities/FBConfiguration.h @@ -139,6 +139,20 @@ extern NSString *const FBSnapshotMaxDepthKey; */ + (NSInteger)screenCaptureServerPort; +/** + The loopback TCP port the ReplayKit broadcast extension connects to. The default value is 9300. + It can be overridden via the BROADCAST_CONTROL_PORT environment variable (the extension itself + always uses the compile-time default, so overriding requires rebuilding the extension too). + */ ++ (NSInteger)broadcastControlPort; + +/** + The bundle identifier of the embedded ReplayKit broadcast upload extension. Defaults to the + runner app's bundle identifier with a '.broadcast' suffix; can be overridden via the + BROADCAST_EXT_BUNDLE_ID environment variable. + */ ++ (NSString *)broadcastExtensionBundleId; + /** The scaling factor for frames of the mjpeg stream. The default (and maximum) value is 100, which does not perform any scaling. The minimum value must be greater than zero. diff --git a/WebDriverAgentLib/Utilities/FBConfiguration.m b/WebDriverAgentLib/Utilities/FBConfiguration.m index 1e57095af..56704f7ee 100644 --- a/WebDriverAgentLib/Utilities/FBConfiguration.m +++ b/WebDriverAgentLib/Utilities/FBConfiguration.m @@ -17,6 +17,7 @@ #import #include "TargetConditionals.h" +#import "FBBroadcastProtocol.h" #import "FBXCodeCompatibility.h" #import "XCAXClient_iOS+FBSnapshotReqParams.h" #import "XCTestPrivateSymbols.h" @@ -175,6 +176,25 @@ + (NSInteger)screenCaptureServerPort return DefaultScreenCaptureServerPort; } ++ (NSInteger)broadcastControlPort +{ + if (NSProcessInfo.processInfo.environment[@"BROADCAST_CONTROL_PORT"] && + [NSProcessInfo.processInfo.environment[@"BROADCAST_CONTROL_PORT"] length] > 0) { + return [NSProcessInfo.processInfo.environment[@"BROADCAST_CONTROL_PORT"] integerValue]; + } + + return FBBroadcastDefaultControlPort; +} + ++ (NSString *)broadcastExtensionBundleId +{ + NSString *fromEnv = NSProcessInfo.processInfo.environment[@"BROADCAST_EXT_BUNDLE_ID"]; + if (fromEnv.length > 0) { + return fromEnv; + } + return [NSBundle.mainBundle.bundleIdentifier stringByAppendingString:@".broadcast"]; +} + + (CGFloat)mjpegScalingFactor { return FBMjpegScalingFactor; diff --git a/WebDriverAgentLib/Utilities/FBVideoStreamManager.h b/WebDriverAgentLib/Utilities/FBVideoStreamManager.h index a2f6064dc..e9e34bcd8 100644 --- a/WebDriverAgentLib/Utilities/FBVideoStreamManager.h +++ b/WebDriverAgentLib/Utilities/FBVideoStreamManager.h @@ -50,6 +50,9 @@ NS_ASSUME_NONNULL_BEGIN /** @return The session with the given id, or nil. */ - (nullable FBVideoStreamSession *)sessionWithIdentifier:(NSUInteger)identifier; +/** @return A snapshot of all active session objects. */ +- (NSArray *)activeSessions; + /** @return An array of dictionaries describing all active sessions, ordered by id. */ - (NSArray *)activeSessionsInfo; diff --git a/WebDriverAgentLib/Utilities/FBVideoStreamManager.m b/WebDriverAgentLib/Utilities/FBVideoStreamManager.m index 9dc31b38b..81b4aa1be 100644 --- a/WebDriverAgentLib/Utilities/FBVideoStreamManager.m +++ b/WebDriverAgentLib/Utilities/FBVideoStreamManager.m @@ -12,6 +12,7 @@ #import @import UniformTypeIdentifiers; +#import "FBBroadcastManager.h" #import "FBConfiguration.h" #import "FBImageUtils.h" #import "FBLogger.h" @@ -134,6 +135,12 @@ - (nullable FBVideoStreamSession *)startSessionWithConfiguration:(FBScreenCaptur [weakSelf captureFrameWithGeneration:generation]; }); } + // Attach the session to the broadcast extension (if connected): the session keeps serving + // locally encoded screenshot frames until the extension's first key frame arrives. + session.onBroadcastKeyFrameNeeded = ^(NSUInteger sessionIdentifier) { + [FBBroadcastManager.sharedInstance requestKeyFrameForSession:sessionIdentifier]; + }; + [FBBroadcastManager.sharedInstance notifySessionAdded:session]; [FBLogger logFmt:@"Started screen capture session %@ (%@ %@x%@) on port %@", @(identifier), session.toDictionary[@"codec"], @(configuration.width), @(configuration.height), @(configuration.port)]; return session; @@ -193,6 +200,7 @@ - (BOOL)stopSessionWithIdentifier:(NSUInteger)identifier } } [session stop]; + [FBBroadcastManager.sharedInstance notifySessionRemoved:identifier]; [FBLogger logFmt:@"Stopped screen capture session %@", @(identifier)]; return YES; } @@ -240,6 +248,14 @@ - (void)stopAllSessions } for (FBVideoStreamSession *session in snapshot) { [session stop]; + [FBBroadcastManager.sharedInstance notifySessionRemoved:session.identifier]; + } +} + +- (NSArray *)activeSessions +{ + @synchronized (self.sessions) { + return self.sessions.allValues; } } @@ -283,7 +299,9 @@ - (void)captureFrameWithGeneration:(NSUInteger)generation BOOL anyClients = NO; for (FBVideoStreamSession *session in snapshot) { loopFps = MAX(loopFps, session.configuration.fps); - if ([session hasClients]) { + // Sessions fed by the broadcast extension do not need (expensive) XCTest screenshots; the + // loop keeps ticking cheaply so it picks the screenshot capture back up if they detach. + if ([session requiresLocalFrames]) { anyClients = YES; } } diff --git a/WebDriverAgentLib/Utilities/FBVideoStreamSession.h b/WebDriverAgentLib/Utilities/FBVideoStreamSession.h index 350211c0d..bef042813 100644 --- a/WebDriverAgentLib/Utilities/FBVideoStreamSession.h +++ b/WebDriverAgentLib/Utilities/FBVideoStreamSession.h @@ -21,6 +21,14 @@ typedef NS_ENUM(NSUInteger, FBVideoFraming) { FBVideoFramingScrcpy, }; +/** The origin of the encoded frames a session is currently serving. */ +typedef NS_ENUM(NSUInteger, FBVideoStreamSource) { + /** Frames are captured via XCTest screenshots and encoded locally (default). */ + FBVideoStreamSourceScreenshot, + /** Pre-encoded frames are received from the ReplayKit broadcast extension. */ + FBVideoStreamSourceBroadcast, +}; + /** Describes a single screen-capture streaming request. */ @interface FBScreenCaptureConfiguration : NSObject @@ -53,6 +61,13 @@ typedef NS_ENUM(NSUInteger, FBVideoFraming) { @property (nonatomic, readonly) NSUInteger identifier; /** The configuration this session was started with. */ @property (nonatomic, readonly) FBScreenCaptureConfiguration *configuration; +/** The origin of the frames this session currently serves. */ +@property (atomic, readonly) FBVideoStreamSource activeSource; +/** + Invoked (with the session identifier) when a key frame is needed while the session serves + broadcast frames, so the request can be forwarded to the extension's encoder. + */ +@property (nonatomic, nullable, copy) void (^onBroadcastKeyFrameNeeded)(NSUInteger sessionIdentifier); - (instancetype)initWithIdentifier:(NSUInteger)identifier configuration:(FBScreenCaptureConfiguration *)configuration; @@ -86,6 +101,29 @@ typedef NS_ENUM(NSUInteger, FBVideoFraming) { */ - (void)maybeEncodeCGImage:(CGImageRef)image atTimeMs:(uint64_t)nowMs; +/** + YES while this session needs frames from the shared screenshot capture loop, i.e. it is active, + has at least one client and is not being fed by the broadcast extension. + */ +- (BOOL)requiresLocalFrames; + +/** Stores fresh Annex-B parameter sets received from the broadcast extension. */ +- (void)ingestBroadcastParameterSets:(NSData *)parameterSets; + +/** + Broadcasts a pre-encoded picture received from the broadcast extension. While the session is + still on the screenshot source it switches to the broadcast source on the first key frame that + has parameter sets available (delta frames before that are dropped, so clients always resync + at an IDR). + */ +- (void)ingestBroadcastFrame:(NSData *)annexBPictureData isKeyFrame:(BOOL)isKeyFrame; + +/** + Reverts the session to the local screenshot source (e.g. because the broadcast stopped) and + forces the local encoder to open with a key frame so clients can resync without reconnecting. + */ +- (void)detachBroadcastSourceAndForceKeyFrame; + /** @return A dictionary describing this session. */ - (NSDictionary *)toDictionary; diff --git a/WebDriverAgentLib/Utilities/FBVideoStreamSession.m b/WebDriverAgentLib/Utilities/FBVideoStreamSession.m index 807ce7e61..2c381480e 100644 --- a/WebDriverAgentLib/Utilities/FBVideoStreamSession.m +++ b/WebDriverAgentLib/Utilities/FBVideoStreamSession.m @@ -41,6 +41,9 @@ @interface FBVideoStreamSession () /** The parameter sets most recently broadcast as a scrcpy config packet (for change detection). */ @property (nonatomic, nullable, copy) NSData *lastSentParameterSets; @property (atomic, getter=isActive) BOOL active; +@property (atomic, readwrite) FBVideoStreamSource activeSource; +/** The parameter sets most recently received from the broadcast extension. */ +@property (atomic, nullable, copy) NSData *broadcastParameterSets; @end @@ -55,6 +58,7 @@ - (instancetype)initWithIdentifier:(NSUInteger)identifier _configuration = configuration; _listeningClients = [NSMutableArray array]; _active = NO; + _activeSource = FBVideoStreamSourceScreenshot; } return self; } @@ -117,13 +121,30 @@ - (BOOL)hasClients - (void)requestKeyFrame { + // While the broadcast extension feeds this session the local encoder is idle, so the request + // must reach the extension's encoder instead. + if (self.activeSource == FBVideoStreamSourceBroadcast) { + void (^onKeyFrameNeeded)(NSUInteger) = self.onBroadcastKeyFrameNeeded; + if (nil != onKeyFrameNeeded) { + onKeyFrameNeeded(self.identifier); + return; + } + } @synchronized (self) { [self.encoder requestKeyFrame]; } } +- (BOOL)requiresLocalFrames +{ + return self.isActive && self.activeSource == FBVideoStreamSourceScreenshot && [self hasClients]; +} + - (void)maybeEncodeCGImage:(CGImageRef)image atTimeMs:(uint64_t)nowMs { + if (self.activeSource == FBVideoStreamSourceBroadcast) { + return; + } @synchronized (self) { if (!self.isActive || nil == self.encoder || ![self hasClients]) { return; @@ -159,12 +180,88 @@ - (uint64_t)nextPresentationTimeMs return candidate; } +#pragma mark - Broadcast (ReplayKit) source + +- (void)ingestBroadcastParameterSets:(NSData *)parameterSets +{ + if (parameterSets.length > 0) { + self.broadcastParameterSets = parameterSets; + } +} + +- (void)ingestBroadcastFrame:(NSData *)annexBPictureData isKeyFrame:(BOOL)isKeyFrame +{ + if (!self.isActive || annexBPictureData.length == 0) { + return; + } + uint64_t presentationTimeUs; + @synchronized (self) { + if (self.activeSource == FBVideoStreamSourceScreenshot) { + // Only take over at an IDR with parameter sets available, so connected clients can + // resync at the source switch without reconnecting. + if (!isKeyFrame || nil == self.broadcastParameterSets) { + return; + } + self.activeSource = FBVideoStreamSourceBroadcast; + // Force a fresh scrcpy config packet for the new elementary stream. + self.lastSentParameterSets = nil; + [FBLogger logFmt:@"Screen capture session %@: switched to the ReplayKit broadcast source", @(self.identifier)]; + } + // Re-stamp with the session's own monotonic clock so one time base survives source switches; + // the extension's timestamp is intentionally ignored. + presentationTimeUs = [self nextPresentationTimeMs] * 1000; + } + [self emitEncodedPicture:annexBPictureData + isKeyFrame:isKeyFrame + presentationTimeUs:presentationTimeUs + parameterSets:self.broadcastParameterSets]; +} + +- (void)detachBroadcastSourceAndForceKeyFrame +{ + @synchronized (self) { + if (self.activeSource != FBVideoStreamSourceBroadcast) { + return; + } + self.activeSource = FBVideoStreamSourceScreenshot; + self.broadcastParameterSets = nil; + // Force a fresh config packet and an IDR from the local encoder so clients resync. + self.lastSentParameterSets = nil; + [self.encoder requestKeyFrame]; + } + [FBLogger logFmt:@"Screen capture session %@: reverted to the screenshot source", @(self.identifier)]; +} + +/** The parameter sets of whichever source currently feeds this session. */ +- (nullable NSData *)currentParameterSets +{ + return self.activeSource == FBVideoStreamSourceBroadcast + ? self.broadcastParameterSets + : self.encoder.parameterSetAnnexB; +} + #pragma mark - - (void)videoEncoder:(FBVideoEncoder *)encoder didEncodeFrame:(NSData *)annexBPictureData isKeyFrame:(BOOL)isKeyFrame presentationTimeUs:(uint64_t)presentationTimeUs +{ + // While the broadcast source is active the local encoder's (at most one in-flight) output is + // discarded; clients resync at the broadcast IDR that triggered the switch. + if (self.activeSource == FBVideoStreamSourceBroadcast) { + return; + } + [self emitEncodedPicture:annexBPictureData + isKeyFrame:isKeyFrame + presentationTimeUs:presentationTimeUs + parameterSets:encoder.parameterSetAnnexB]; +} + +- (void)emitEncodedPicture:(NSData *)annexBPictureData + isKeyFrame:(BOOL)isKeyFrame + presentationTimeUs:(uint64_t)presentationTimeUs + parameterSets:(nullable NSData *)parameterSets { if (annexBPictureData.length == 0) { return; @@ -175,7 +272,6 @@ - (void)videoEncoder:(FBVideoEncoder *)encoder // packet must carry picture data only. Emit a separate config packet whenever the parameter // sets change. if (isKeyFrame) { - NSData *parameterSets = encoder.parameterSetAnnexB; if (parameterSets.length > 0 && ![parameterSets isEqualToData:self.lastSentParameterSets]) { self.lastSentParameterSets = parameterSets; [self broadcastData:[self.class scrcpyPacketWithPayload:parameterSets @@ -192,7 +288,6 @@ - (void)videoEncoder:(FBVideoEncoder *)encoder // Annex-B mode: prepend the parameter sets to key frames so each IDR is independently decodable. if (isKeyFrame) { - NSData *parameterSets = encoder.parameterSetAnnexB; if (parameterSets.length > 0) { NSMutableData *keyFrame = [NSMutableData dataWithCapacity:parameterSets.length + annexBPictureData.length]; [keyFrame appendData:parameterSets]; @@ -249,7 +344,7 @@ - (void)didClientConnect:(GCDAsyncSocket *)newClient } // Hand the latest parameter sets to the new client and force a key frame so it can start // decoding immediately. In scrcpy mode the parameter sets are wrapped as a config packet. - NSData *parameterSets = self.encoder.parameterSetAnnexB; + NSData *parameterSets = [self currentParameterSets]; if (parameterSets.length > 0) { NSData *payload = self.configuration.framing == FBVideoFramingScrcpy ? [self.class scrcpyPacketWithPayload:parameterSets flags:FBScrcpyFlagConfig presentationTimeUs:0] @@ -293,6 +388,7 @@ - (NSDictionary *)toDictionary @"bitrate": @(self.configuration.bitrate), @"port": @(self.configuration.port), @"clients": @(clientCount), + @"source": self.activeSource == FBVideoStreamSourceBroadcast ? @"replaykit" : @"screenshot", }; } diff --git a/docs/broadcast-extension.md b/docs/broadcast-extension.md new file mode 100644 index 000000000..b8da67032 --- /dev/null +++ b/docs/broadcast-extension.md @@ -0,0 +1,113 @@ +# ReplayKit Broadcast Extension + +WebDriverAgent embeds a ReplayKit **broadcast upload extension** +(`WebDriverAgentBroadcast.appex`) into the generated +`WebDriverAgentRunner-Runner.app`. When a broadcast is running, the extension receives the +system's screen frames as pixel buffers (up to 60 fps, no polling), encodes them with one +hardware H.264/H.265 encoder per capture session and ships the elementary stream to WDA over +a loopback TCP connection. `/mobilerun/screencapture` sessions then serve those frames instead +of the XCTest screenshot loop, raising the achievable frame rate from ~10-20 fps to 30-60 fps. + +The extension only works on **physical iOS devices** (ReplayKit broadcasts are unavailable on +the Simulator and tvOS). Without a running broadcast every capture session transparently uses +the legacy screenshot pipeline; each session reports its current origin via the `source` field +(`"replaykit"` or `"screenshot"`). + +## HTTP endpoints + +| Endpoint | Method | Description | +|---|---|---| +| `/mobilerun/screencapture/broadcast/start` | POST | Starts a system broadcast targeting the bundled extension. Foregrounds the runner app, triggers `RPSystemBroadcastPickerView` and confirms the system sheet via UI automation, then waits for the extension to connect. Idempotent while connected. | +| `/mobilerun/screencapture/broadcast` | GET | Broadcast status: `state` (`idle`/`connected`/`paused`), control port, extension id, last heartbeat (frames received, orientation, screen size) and the capture sessions with their active `source`. | +| `/mobilerun/screencapture/broadcast/stop` | POST | Asks the extension to finish the broadcast. Live sessions fall back to the screenshot source with a forced key frame; clients do not need to reconnect. | + +`broadcast/start` body (all optional): + +```json +{ + "timeout": 30, + "confirmButtonLabels": ["Start Broadcast"], + "restoreForegroundApp": true +} +``` + +- `timeout` — seconds to wait for the extension to connect (covers the system's 3-2-1 countdown). +- `confirmButtonLabels` — labels to look for on the system confirmation sheet. Pass the + localized label when the device language is not English (a button starting with "Start" is + used as fallback). +- `restoreForegroundApp` — re-activate the previously active app after the broadcast starts + (the start dance briefly foregrounds the runner app, ~2-3 s). + +Notes: + +- `/mobilerun/screencapture/start` never starts a broadcast by itself. Sessions started while + the broadcast is connected attach to it automatically; sessions started before it pick it up + on the extension's first key frame. +- Broadcasts started manually from Control Center attach exactly the same way (WDA listens + permanently). +- If the broadcast stops (endpoint, status-bar pill, extension crash), sessions revert to the + screenshot source within ~6 s (heartbeat staleness) without dropping client connections. +- Frames are encoded in the native ReplayKit orientation (not rotated upright like the + screenshot path). The current `orientation` (CGImagePropertyOrientation 1-8) is exposed via + the status endpoint and heartbeat. + +## Environment variables + +| Variable | Default | Description | +|---|---|---| +| `BROADCAST_CONTROL_PORT` | `9300` | Loopback port WDA listens on for the extension. The extension always connects to the compile-time default, so overriding this requires rebuilding the extension with a matching `FBBroadcastDefaultControlPort`. | +| `BROADCAST_EXT_BUNDLE_ID` | `.broadcast` | Bundle id preselected in the broadcast picker. Only needed if a re-signing pipeline renames the appex. | + +## Build integration + +- The `WebDriverAgentBroadcast` target is a dependency of `WebDriverAgentRunner`, so any + scheme-based build (`xcodebuild build-for-testing -scheme WebDriverAgentRunner`, Fastlane, + the npm bundle scripts) builds the appex into `BUILT_PRODUCTS_DIR`. +- A scheme **build post-action** (`Scripts/embed-broadcast-extension.sh`) copies the appex + into `Runner.app/PlugIns/`, rewrites its `CFBundleIdentifier` to + `.broadcast` (the host id gets the `.xctrunner` suffix and may + be overridden by downstream tooling, so it is derived at embed time) and re-signs + inner-first. Post-actions run for scheme-based command-line builds too — the same mechanism + that embeds the runner icon today. +- Builds that bypass the scheme (`xcodebuild -target ...`) must run the embed step manually: + + ```bash + BUILT_PRODUCTS_DIR= PRODUCT_NAME=WebDriverAgentRunner \ + Scripts/embed-broadcast-extension.sh + ``` + +## Re-signing (device farms / prebuilt WDA) + +Pipelines that re-sign `WebDriverAgentRunner-Runner.app` must keep the nested appex valid: + +1. Sign `PlugIns/WebDriverAgentBroadcast.appex` first with the **same team/identity** as the + app (or use `codesign --deep` on the app, which handles nesting). Signing the outer app does + not invalidate an already-valid nested signature. +2. The appex `CFBundleIdentifier` must remain `.broadcast` — installd + rejects extensions whose bundle id is not prefixed by the host app's, or whose team differs. + If the pipeline changes the runner's bundle id, patch the appex id accordingly (the embed + script shows how) and set `BROADCAST_EXT_BUNDLE_ID` if the suffix differs. +3. Provisioning: the embedded profile must cover the appex bundle id; a wildcard development + profile (`TEAM.*`) is the low-friction option. The extension needs no special entitlements + (no app groups — IPC is loopback TCP). + +## Architecture + +``` +WebDriverAgentRunner-Runner.app +├── PlugIns/WebDriverAgentRunner.xctest WDA +│ FBBroadcastManager ─ FBBroadcastControlServer (127.0.0.1:9300) +│ FBVideoStreamManager → FBVideoStreamSession (TCP fan-out :9200+, +│ screenshot-encode or broadcast-passthrough per session) +└── PlugIns/WebDriverAgentBroadcast.appex extension + FBBroadcastSampleHandler (RPBroadcastSampleHandler) + → FBExtSessionPipeline × N (VTPixelTransfer letterbox scale + + FBVideoEncoder per session, 420v end to end) + → FBExtBroadcastClient ──TCP──► WDA control port +``` + +The wire protocol (16-byte framed messages, JSON control payloads, binary video frames) is +defined in `WebDriverAgentLib/Utilities/FBBroadcastProtocol.h`, which is compiled into both +the framework and the extension. + +Audio capture (Opus) is a planned follow-up; audio sample buffers are currently ignored. diff --git a/package.json b/package.json index cbea980b8..b7ce78b6f 100644 --- a/package.json +++ b/package.json @@ -89,10 +89,12 @@ "build/lib", "Scripts/build.sh", "Scripts/embed-runner-icon.sh", + "Scripts/embed-broadcast-extension.sh", "Scripts/*.mjs", "Configurations", "PrivateHeaders", "WebDriverAgent.xcodeproj", + "WebDriverAgentBroadcast", "WebDriverAgentLib", "WebDriverAgentRunner", "WebDriverAgentTests", From 51888231cc5a52b89325ce7c66a768d1e7b883d7 Mon Sep 17 00:00:00 2001 From: timo <44401485+Timo972@users.noreply.github.com> Date: Wed, 10 Jun 2026 22:16:15 +0200 Subject: [PATCH 02/11] feat(screencapture): optimize broadcast capture frame pacing --- .../FBExtSessionPipeline.m | 151 +++++++++++++++--- .../Utilities/FBBroadcastProtocol.h | 7 + .../Utilities/FBBroadcastProtocol.m | 32 ++++ WebDriverAgentLib/Utilities/FBVideoEncoder.m | 28 ++-- 4 files changed, 186 insertions(+), 32 deletions(-) diff --git a/WebDriverAgentBroadcast/FBExtSessionPipeline.m b/WebDriverAgentBroadcast/FBExtSessionPipeline.m index 5361c0ab5..2e2f2b573 100644 --- a/WebDriverAgentBroadcast/FBExtSessionPipeline.m +++ b/WebDriverAgentBroadcast/FBExtSessionPipeline.m @@ -20,22 +20,36 @@ static NSString *const FBExtPipelineErrorDomain = @"com.facebook.WebDriverAgent.FBExtSessionPipeline"; // Bounds the scaled-buffer pool so a stuck consumer cannot grow the extension past its ~50MB cap. -static const int POOL_ALLOCATION_THRESHOLD = 3; +static const int POOL_ALLOCATION_THRESHOLD = 4; -@interface FBExtSessionPipeline () +@interface FBExtSessionPipeline () { + CMSampleBufferRef _pendingSampleBuffer; +} @property (nonatomic, weak) id sink; @property (nonatomic) dispatch_queue_t queue; @property (nonatomic, nullable) FBVideoEncoder *encoder; @property (nonatomic) VTPixelTransferSessionRef transferSession; @property (nonatomic) CVPixelBufferPoolRef bufferPool; +@property (nonatomic) NSUInteger width; +@property (nonatomic) NSUInteger height; @property (nonatomic) NSUInteger fps; -@property (atomic) uint64_t lastSubmitTimeMs; -@property (atomic) BOOL inFlight; +@property (nonatomic) uint64_t lastSubmitTimeMs; +@property (nonatomic) BOOL inFlight; +@property (nonatomic) uint64_t pendingSubmitTimeMs; +@property (nonatomic) uint8_t pendingOrientation; @property (atomic) BOOL active; @property (atomic) uint8_t currentOrientation; +@property (nonatomic) BOOL directSourceEncodingDisabled; @property (nonatomic, nullable, copy) NSData *lastSentParameterSets; +- (void)replacePendingSampleBuffer:(CMSampleBufferRef)sampleBuffer + atTimeMs:(uint64_t)nowMs + orientation:(uint8_t)orientation; +- (nullable CMSampleBufferRef)copyPendingSampleBufferAtTimeMs:(uint64_t *)timeMs + orientation:(uint8_t *)orientation; +- (void)clearPendingSampleBufferLocked; + @end @implementation FBExtSessionPipeline @@ -73,6 +87,8 @@ - (nullable instancetype)initWithSessionId:(uint32_t)sessionId return nil; } _fps = fps; + _width = width; + _height = height; OSStatus status = VTPixelTransferSessionCreate(kCFAllocatorDefault, &_transferSession); if (status != noErr || NULL == _transferSession) { @@ -134,40 +150,125 @@ - (void)submitSampleBuffer:(CMSampleBufferRef)sampleBuffer orientation:(uint8_t) if (!self.active) { return; } - self.currentOrientation = orientation; // Respect this session's framerate even though ReplayKit may deliver faster. uint64_t nowMs = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) / NSEC_PER_MSEC; uint64_t minIntervalMs = self.fps > 0 ? (uint64_t)(1000 / self.fps) : 0; - if (minIntervalMs > 1 && nowMs - self.lastSubmitTimeMs < minIntervalMs - 1) { - return; - } - // Never queue and never block the ReplayKit sample queue: drop when the previous frame is - // still being scaled/submitted. - if (self.inFlight) { - return; + @synchronized (self) { + if (minIntervalMs > 1 && nowMs - self.lastSubmitTimeMs < minIntervalMs - 1) { + return; + } + + // Never block the ReplayKit sample queue. If a frame is already being scaled/submitted, + // retain only the newest due sample and replace any older pending sample. + if (self.inFlight) { + [self replacePendingSampleBuffer:sampleBuffer atTimeMs:nowMs orientation:orientation]; + return; + } + self.inFlight = YES; + self.lastSubmitTimeMs = nowMs; } - self.inFlight = YES; - self.lastSubmitTimeMs = nowMs; CFRetain(sampleBuffer); dispatch_async(self.queue, ^{ - [self processRetainedSampleBuffer:sampleBuffer atTimeMs:nowMs]; + [self drainRetainedSampleBuffer:sampleBuffer atTimeMs:nowMs orientation:orientation]; CFRelease(sampleBuffer); - self.inFlight = NO; }); } -- (void)processRetainedSampleBuffer:(CMSampleBufferRef)sampleBuffer atTimeMs:(uint64_t)nowMs +- (void)replacePendingSampleBuffer:(CMSampleBufferRef)sampleBuffer + atTimeMs:(uint64_t)nowMs + orientation:(uint8_t)orientation +{ + CMSampleBufferRef retainedSampleBuffer = (CMSampleBufferRef)CFRetain(sampleBuffer); + CMSampleBufferRef previous = _pendingSampleBuffer; + _pendingSampleBuffer = retainedSampleBuffer; + self.pendingSubmitTimeMs = nowMs; + self.pendingOrientation = orientation; + if (NULL != previous) { + CFRelease(previous); + } +} + +- (nullable CMSampleBufferRef)copyPendingSampleBufferAtTimeMs:(uint64_t *)timeMs + orientation:(uint8_t *)orientation +{ + @synchronized (self) { + if (!self.active) { + [self clearPendingSampleBufferLocked]; + self.inFlight = NO; + return NULL; + } + CMSampleBufferRef pending = _pendingSampleBuffer; + if (NULL == pending) { + self.inFlight = NO; + return NULL; + } + _pendingSampleBuffer = NULL; + *timeMs = self.pendingSubmitTimeMs; + *orientation = self.pendingOrientation; + self.lastSubmitTimeMs = self.pendingSubmitTimeMs; + return pending; + } +} + +- (void)clearPendingSampleBufferLocked +{ + CMSampleBufferRef pending = _pendingSampleBuffer; + _pendingSampleBuffer = NULL; + if (NULL != pending) { + CFRelease(pending); + } +} + +- (void)drainRetainedSampleBuffer:(CMSampleBufferRef)sampleBuffer + atTimeMs:(uint64_t)nowMs + orientation:(uint8_t)orientation +{ + CMSampleBufferRef currentSampleBuffer = sampleBuffer; + uint64_t currentTimeMs = nowMs; + uint8_t currentOrientation = orientation; + BOOL shouldReleaseCurrentSampleBuffer = NO; + + while (NULL != currentSampleBuffer) { + [self processRetainedSampleBuffer:currentSampleBuffer + atTimeMs:currentTimeMs + orientation:currentOrientation]; + if (shouldReleaseCurrentSampleBuffer) { + CFRelease(currentSampleBuffer); + } + currentSampleBuffer = [self copyPendingSampleBufferAtTimeMs:¤tTimeMs + orientation:¤tOrientation]; + shouldReleaseCurrentSampleBuffer = (NULL != currentSampleBuffer); + } +} + +- (void)processRetainedSampleBuffer:(CMSampleBufferRef)sampleBuffer + atTimeMs:(uint64_t)nowMs + orientation:(uint8_t)orientation { if (!self.active) { return; } + self.currentOrientation = orientation; CVPixelBufferRef sourceBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); if (NULL == sourceBuffer) { return; } + if (!self.directSourceEncodingDisabled + && CVPixelBufferGetWidth(sourceBuffer) == self.width + && CVPixelBufferGetHeight(sourceBuffer) == self.height) { + NSError *error; + if ([self.encoder encodePixelBuffer:sourceBuffer presentationTimeMs:nowMs error:&error]) { + return; + } + self.directSourceEncodingDisabled = YES; + FBExtLogError("Session %u: direct source encode failed; falling back to pixel transfer: %{public}@", + self.sessionId, + error.description); + } + CVPixelBufferRef scaledBuffer = NULL; NSDictionary *auxAttributes = @{(id)kCVPixelBufferPoolAllocationThresholdKey: @(POOL_ALLOCATION_THRESHOLD)}; CVReturn poolStatus = CVPixelBufferPoolCreatePixelBufferWithAuxAttributes(kCFAllocatorDefault, @@ -201,6 +302,9 @@ - (void)requestKeyFrame - (void)teardown { self.active = NO; + @synchronized (self) { + [self clearPendingSampleBufferLocked]; + } dispatch_async(self.queue, ^{ if (nil != self.encoder) { self.encoder.delegate = nil; @@ -226,6 +330,9 @@ - (void)releaseScalerResources - (void)dealloc { + @synchronized (self) { + [self clearPendingSampleBufferLocked]; + } if (nil != _encoder) { _encoder.delegate = nil; [_encoder stop]; @@ -261,11 +368,11 @@ - (void)videoEncoder:(FBVideoEncoder *)encoder } } - NSData *payload = FBBroadcastEncodeVideoFramePayload(presentationTimeUs, - isKeyFrame, - self.currentOrientation, - annexBPictureData); - [sink sendProtocolMessage:FBBroadcastEncodeMessage(FBBroadcastMessageTypeVideoFrame, self.sessionId, payload) + [sink sendProtocolMessage:FBBroadcastEncodeVideoFrameMessage(self.sessionId, + presentationTimeUs, + isKeyFrame, + self.currentOrientation, + annexBPictureData) isDroppable:!isKeyFrame]; } diff --git a/WebDriverAgentLib/Utilities/FBBroadcastProtocol.h b/WebDriverAgentLib/Utilities/FBBroadcastProtocol.h index b9159c67c..ebd2d98f8 100644 --- a/WebDriverAgentLib/Utilities/FBBroadcastProtocol.h +++ b/WebDriverAgentLib/Utilities/FBBroadcastProtocol.h @@ -126,6 +126,13 @@ NSData *FBBroadcastEncodeVideoFramePayload(uint64_t ptsUs, uint8_t orientation, NSData *annexBPictureData); +/** Builds a complete VIDEO_FRAME wire message without first materializing a separate payload. */ +NSData *FBBroadcastEncodeVideoFrameMessage(uint32_t sessionId, + uint64_t ptsUs, + BOOL isKeyFrame, + uint8_t orientation, + NSData *annexBPictureData); + /** Parses a VIDEO_FRAME payload. diff --git a/WebDriverAgentLib/Utilities/FBBroadcastProtocol.m b/WebDriverAgentLib/Utilities/FBBroadcastProtocol.m index 301f33a79..1f32a4842 100644 --- a/WebDriverAgentLib/Utilities/FBBroadcastProtocol.m +++ b/WebDriverAgentLib/Utilities/FBBroadcastProtocol.m @@ -125,6 +125,38 @@ BOOL FBBroadcastParseHeader(NSData *headerData, FBBroadcastMessageHeader *outHea return payload; } +NSData *FBBroadcastEncodeVideoFrameMessage(uint32_t sessionId, + uint64_t ptsUs, + BOOL isKeyFrame, + uint8_t orientation, + NSData *annexBPictureData) +{ + static const NSUInteger prefixLength = sizeof(uint64_t) + sizeof(uint8_t); + NSUInteger payloadLength = prefixLength + annexBPictureData.length; + NSMutableData *message = [NSMutableData dataWithCapacity:FBBroadcastHeaderLength + payloadLength]; + + uint32_t bigMagic = CFSwapInt32HostToBig(FBBroadcastProtocolMagic); + [message appendBytes:&bigMagic length:sizeof(bigMagic)]; + uint8_t version = FBBroadcastProtocolVersion; + [message appendBytes:&version length:sizeof(version)]; + uint8_t messageType = (uint8_t)FBBroadcastMessageTypeVideoFrame; + [message appendBytes:&messageType length:sizeof(messageType)]; + uint16_t reserved = 0; + [message appendBytes:&reserved length:sizeof(reserved)]; + uint32_t bigSessionId = CFSwapInt32HostToBig(sessionId); + [message appendBytes:&bigSessionId length:sizeof(bigSessionId)]; + uint32_t bigPayloadLength = CFSwapInt32HostToBig((uint32_t)payloadLength); + [message appendBytes:&bigPayloadLength length:sizeof(bigPayloadLength)]; + + uint64_t bigPts = CFSwapInt64HostToBig(ptsUs); + [message appendBytes:&bigPts length:sizeof(bigPts)]; + uint8_t flags = (isKeyFrame ? FBBroadcastFrameFlagKeyFrame : 0) + | (uint8_t)((orientation & FBBroadcastFrameOrientationMask) << FBBroadcastFrameOrientationShift); + [message appendBytes:&flags length:sizeof(flags)]; + [message appendData:annexBPictureData]; + return message; +} + BOOL FBBroadcastParseVideoFramePayload(NSData *payload, uint64_t *outPtsUs, BOOL *outIsKeyFrame, diff --git a/WebDriverAgentLib/Utilities/FBVideoEncoder.m b/WebDriverAgentLib/Utilities/FBVideoEncoder.m index 18b3c13c4..4230ec9a7 100644 --- a/WebDriverAgentLib/Utilities/FBVideoEncoder.m +++ b/WebDriverAgentLib/Utilities/FBVideoEncoder.m @@ -88,6 +88,8 @@ - (void)configureSession { VTSessionSetProperty(self.session, kVTCompressionPropertyKey_RealTime, kCFBooleanTrue); VTSessionSetProperty(self.session, kVTCompressionPropertyKey_AllowFrameReordering, kCFBooleanFalse); + CFNumberRef maxFrameDelayCount = (__bridge CFNumberRef)@(1); + VTSessionSetProperty(self.session, kVTCompressionPropertyKey_MaxFrameDelayCount, maxFrameDelayCount); CFStringRef profileLevel = (self.codec == FBVideoCodecH265) ? kVTProfileLevel_HEVC_Main_AutoLevel @@ -254,27 +256,33 @@ - (nullable NSData *)annexBPictureDataFromSampleBuffer:(CMSampleBufferRef)sample return nil; } - NSMutableData *avccData = [NSMutableData dataWithLength:totalLength]; - OSStatus status = CMBlockBufferCopyDataBytes(blockBuffer, 0, totalLength, avccData.mutableBytes); - if (status != kCMBlockBufferNoErr) { - return nil; - } - - const uint8_t *bytes = (const uint8_t *)avccData.bytes; size_t headerLength = self.nalUnitHeaderLength == 0 ? 4 : self.nalUnitHeaderLength; - NSMutableData *annexBData = [NSMutableData data]; + uint8_t headerBytes[8]; + NSMutableData *annexBData = [NSMutableData dataWithCapacity:totalLength + sizeof(FBAnnexBStartCode)]; size_t offset = 0; while (offset + headerLength <= totalLength) { + OSStatus status = CMBlockBufferCopyDataBytes(blockBuffer, offset, headerLength, headerBytes); + if (status != kCMBlockBufferNoErr) { + return nil; + } uint64_t nalLength = 0; for (size_t i = 0; i < headerLength; i++) { - nalLength = (nalLength << 8) | bytes[offset + i]; + nalLength = (nalLength << 8) | headerBytes[i]; } offset += headerLength; if (nalLength == 0 || offset + nalLength > totalLength) { break; } [annexBData appendBytes:FBAnnexBStartCode length:sizeof(FBAnnexBStartCode)]; - [annexBData appendBytes:bytes + offset length:(NSUInteger)nalLength]; + NSUInteger previousLength = annexBData.length; + [annexBData setLength:previousLength + (NSUInteger)nalLength]; + status = CMBlockBufferCopyDataBytes(blockBuffer, + offset, + (size_t)nalLength, + (uint8_t *)annexBData.mutableBytes + previousLength); + if (status != kCMBlockBufferNoErr) { + return nil; + } offset += nalLength; } return annexBData.length > 0 ? annexBData : nil; From 48d5f19fc94adac092347d9da015316debd0da15 Mon Sep 17 00:00:00 2001 From: timo <44401485+Timo972@users.noreply.github.com> Date: Wed, 10 Jun 2026 22:18:43 +0200 Subject: [PATCH 03/11] feat(screencapture): tune broadcast scaler for realtime --- WebDriverAgentBroadcast/FBExtSessionPipeline.m | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/WebDriverAgentBroadcast/FBExtSessionPipeline.m b/WebDriverAgentBroadcast/FBExtSessionPipeline.m index 2e2f2b573..4a3d3e562 100644 --- a/WebDriverAgentBroadcast/FBExtSessionPipeline.m +++ b/WebDriverAgentBroadcast/FBExtSessionPipeline.m @@ -101,6 +101,7 @@ - (nullable instancetype)initWithSessionId:(uint32_t)sessionId } // Letterbox to preserve the aspect ratio, matching the WDA-side screenshot converter. VTSessionSetProperty(_transferSession, kVTPixelTransferPropertyKey_ScalingMode, kVTScalingMode_Letterbox); + VTSessionSetProperty(_transferSession, kVTPixelTransferPropertyKey_RealTime, kCFBooleanTrue); NSDictionary *pixelBufferAttributes = @{ (id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange), @@ -108,7 +109,7 @@ - (nullable instancetype)initWithSessionId:(uint32_t)sessionId (id)kCVPixelBufferHeightKey: @(height), (id)kCVPixelBufferIOSurfacePropertiesKey: @{}, }; - NSDictionary *poolAttributes = @{(id)kCVPixelBufferPoolMinimumBufferCountKey: @1}; + NSDictionary *poolAttributes = @{(id)kCVPixelBufferPoolMinimumBufferCountKey: @(POOL_ALLOCATION_THRESHOLD)}; CVReturn poolStatus = CVPixelBufferPoolCreate(kCFAllocatorDefault, (__bridge CFDictionaryRef)poolAttributes, (__bridge CFDictionaryRef)pixelBufferAttributes, From 9d92f36396fee97ed8bc086a4e543cfb259eb9a8 Mon Sep 17 00:00:00 2001 From: timo <44401485+Timo972@users.noreply.github.com> Date: Wed, 10 Jun 2026 22:35:59 +0200 Subject: [PATCH 04/11] feat(screencapture): add capture quality option --- WebDriverAgentLib/Commands/FBScreenCaptureCommands.m | 9 +++++++++ WebDriverAgentLib/Utilities/FBVideoStreamManager.m | 10 ++++++++-- WebDriverAgentLib/Utilities/FBVideoStreamSession.h | 2 ++ WebDriverAgentLib/Utilities/FBVideoStreamSession.m | 11 +++++++++++ docs/mobilerun-screencapture.md | 7 ++++++- 5 files changed, 36 insertions(+), 3 deletions(-) diff --git a/WebDriverAgentLib/Commands/FBScreenCaptureCommands.m b/WebDriverAgentLib/Commands/FBScreenCaptureCommands.m index 0b77267d7..fe984e5fe 100644 --- a/WebDriverAgentLib/Commands/FBScreenCaptureCommands.m +++ b/WebDriverAgentLib/Commands/FBScreenCaptureCommands.m @@ -15,6 +15,7 @@ static const NSUInteger DEFAULT_CAPTURE_FPS = 30; static const NSUInteger DEFAULT_CAPTURE_BITRATE = 6000000; +static const CGFloat DEFAULT_CAPTURE_QUALITY = 0.8; @implementation FBScreenCaptureCommands @@ -133,6 +134,14 @@ + (NSArray *)routes configuration.height = (NSUInteger)(height - (height % 2)); NSNumber *bitrate = request.arguments[@"bitrate"]; configuration.bitrate = (nil != bitrate && bitrate.integerValue > 0) ? bitrate.unsignedIntegerValue : DEFAULT_CAPTURE_BITRATE; + id quality = request.arguments[@"quality"]; + if (nil == quality) { + configuration.quality = DEFAULT_CAPTURE_QUALITY; + } else if (![quality isKindOfClass:NSNumber.class] || ((NSNumber *)quality).doubleValue < 0.0 || ((NSNumber *)quality).doubleValue > 1.0) { + return FBResponseWithStatus([FBCommandStatus invalidArgumentErrorWithMessage:@"'quality' must be a number in the range 0.0..1.0" traceback:nil]); + } else { + configuration.quality = (CGFloat)((NSNumber *)quality).doubleValue; + } NSNumber *fps = request.arguments[@"fps"]; configuration.fps = (nil != fps && fps.integerValue > 0) ? fps.unsignedIntegerValue : DEFAULT_CAPTURE_FPS; NSNumber *port = request.arguments[@"port"]; diff --git a/WebDriverAgentLib/Utilities/FBVideoStreamManager.m b/WebDriverAgentLib/Utilities/FBVideoStreamManager.m index 81b4aa1be..172fafa2b 100644 --- a/WebDriverAgentLib/Utilities/FBVideoStreamManager.m +++ b/WebDriverAgentLib/Utilities/FBVideoStreamManager.m @@ -28,7 +28,6 @@ static const NSTimeInterval FRAME_TIMEOUT = 1.0; static const NSTimeInterval FAILURE_BACKOFF_MIN = 1.0; static const NSTimeInterval FAILURE_BACKOFF_MAX = 10.0; -static const CGFloat CAPTURE_COMPRESSION_QUALITY = 0.8; static const char *QUEUE_NAME = "Screen Capture Encoder Queue"; @interface FBVideoStreamManager () @@ -317,8 +316,15 @@ - (void)captureFrameWithGeneration:(NSUInteger)generation } NSError *error; + CGFloat captureQuality = 1.0; + for (FBVideoStreamSession *session in snapshot) { + if ([session requiresLocalFrames]) { + captureQuality = MIN(captureQuality, session.configuration.quality); + } + } + NSData *screenshotData = [FBScreenshot takeInOriginalResolutionWithScreenID:self.mainScreenID - compressionQuality:CAPTURE_COMPRESSION_QUALITY + compressionQuality:captureQuality uti:UTTypeJPEG timeout:FRAME_TIMEOUT error:&error]; diff --git a/WebDriverAgentLib/Utilities/FBVideoStreamSession.h b/WebDriverAgentLib/Utilities/FBVideoStreamSession.h index bef042813..e656cdec3 100644 --- a/WebDriverAgentLib/Utilities/FBVideoStreamSession.h +++ b/WebDriverAgentLib/Utilities/FBVideoStreamSession.h @@ -40,6 +40,8 @@ typedef NS_ENUM(NSUInteger, FBVideoStreamSource) { @property (nonatomic) NSUInteger height; /** The target average bitrate in bits per second. */ @property (nonatomic) NSUInteger bitrate; +/** JPEG compression quality used when capturing XCTest screenshot frames before video encoding. */ +@property (nonatomic) CGFloat quality; /** The capture/encode framerate in frames per second. */ @property (nonatomic) NSUInteger fps; /** The framing of the broadcast byte stream (raw Annex-B by default). */ diff --git a/WebDriverAgentLib/Utilities/FBVideoStreamSession.m b/WebDriverAgentLib/Utilities/FBVideoStreamSession.m index 2c381480e..00908b9f2 100644 --- a/WebDriverAgentLib/Utilities/FBVideoStreamSession.m +++ b/WebDriverAgentLib/Utilities/FBVideoStreamSession.m @@ -25,8 +25,18 @@ static const uint64_t FBScrcpyFlagConfig = (uint64_t)1 << 63; static const uint64_t FBScrcpyFlagKeyFrame = (uint64_t)1 << 62; static const uint64_t FBScrcpyPtsMask = ~((uint64_t)3 << 62); +static const CGFloat FBDefaultScreenCaptureQuality = 0.8; @implementation FBScreenCaptureConfiguration + +- (instancetype)init +{ + if ((self = [super init])) { + _quality = FBDefaultScreenCaptureQuality; + } + return self; +} + @end @@ -386,6 +396,7 @@ - (NSDictionary *)toDictionary @"height": @(self.configuration.height), @"fps": @(self.configuration.fps), @"bitrate": @(self.configuration.bitrate), + @"quality": @(self.configuration.quality), @"port": @(self.configuration.port), @"clients": @(clientCount), @"source": self.activeSource == FBVideoStreamSourceBroadcast ? @"replaykit" : @"screenshot", diff --git a/docs/mobilerun-screencapture.md b/docs/mobilerun-screencapture.md index f5ac961bf..0c80b320b 100644 --- a/docs/mobilerun-screencapture.md +++ b/docs/mobilerun-screencapture.md @@ -42,6 +42,7 @@ not a WDA automation session is active. | `codec` | string | no | `"h264"` | `h264`/`avc` or `h265`/`hevc`. | | `framing` | string | no | `"annexb"` | `annexb`/`annex-b`/`raw`, or `scrcpy`/`packet`/`packetized`. See [Wire formats](#wire-formats). | | `bitrate` | int | no | `6000000` | Target average bits/sec. | +| `quality` | float | no | `0.8` | JPEG quality (`0.0`–`1.0`) used for XCTest screenshot capture before local H.264/H.265 encoding. Lower values can reduce screenshot capture/decode cost. Does not affect ReplayKit/broadcast-source frames. | | `fps` | int | no | `30` | Capture/encode frame rate. | | `port` | int | no | auto | `0` or omitted → auto-assign from **9200** (env `SCREEN_CAPTURE_SERVER_PORT` overrides the base), scanning forward up to 64 ports. An explicit port (1–65535) is tried once and surfaces a bind failure. | @@ -58,6 +59,7 @@ Returned by `start` and `GET …/:id`; also the array items of the list response "height": 1334, "fps": 30, "bitrate": 6000000, + "quality": 0.8, "port": 9200, "clients": 0 } @@ -100,7 +102,7 @@ Start (raw, default), against `http://:8100`: ```bash curl -s -X POST http://localhost:8100/mobilerun/screencapture/start \ -H 'Content-Type: application/json' \ - -d '{"width":750,"height":1334,"codec":"h264","fps":30}' + -d '{"width":750,"height":1334,"codec":"h264","fps":30,"quality":0.5}' # → { "id":1, "framing":"annexb", "port":9200, ... } ``` @@ -132,5 +134,8 @@ curl -s -X POST http://localhost:8100/mobilerun/screencapture/1/stop than buffered (1 s write timeout), and `TCP_NODELAY` is on for low latency. - A late joiner won't decode until the next key frame — hit the `/keyframe` endpoint (connecting already forces one) if you need an immediate resync. +- If multiple screenshot-source sessions request different `quality` values, WDA captures the + shared local screenshot frame at the lowest requested quality and fans it out to all local + encoders. - For `scrcpy` framing you must parse the 12-byte header yourself (or reuse `ReadFrame` from `h264reader.go`); you can't pipe it straight into ffmpeg the way you can with `annexb`. From 161b640702b824da8d376b9432e6bfc1904c1e11 Mon Sep 17 00:00:00 2001 From: timo <44401485+Timo972@users.noreply.github.com> Date: Wed, 10 Jun 2026 22:49:00 +0200 Subject: [PATCH 05/11] feat(screencapture): show the broadcast picker in under a second POST /mobilerun/screencapture/broadcast/start used to sit on a black screen for ~5s before the system sheet appeared: it foregrounded the runner via XCUIApplication.activate, whose self-quiescence wait can only ever time out because the waiting thread is the very main thread whose idleness is being awaited. Foreground via LSApplicationWorkspace (FBUnattachedAppLauncher) instead and poll the in-process application state, keeping the XCTest activation only as a fallback. Measured on an iPhone XS (iOS 18.7): runner foreground after 485ms, picker triggered after 531ms, broadcast connected after 8.4s total. Also skip the active-app lookup when the runner is already frontmost, re-fire the picker press every 2s until the confirmation sheet shows up (the system drops presses that arrive before the scene is fully active), and log per-stage timings for the whole dance. Co-Authored-By: Claude Fable 5 --- .../xcschemes/WebDriverAgentRunner.xcscheme | 2 +- .../Utilities/FBBroadcastManager.m | 70 +++++++++++++++---- 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/WebDriverAgent.xcodeproj/xcshareddata/xcschemes/WebDriverAgentRunner.xcscheme b/WebDriverAgent.xcodeproj/xcshareddata/xcschemes/WebDriverAgentRunner.xcscheme index 913833d0f..4937e6a7d 100644 --- a/WebDriverAgent.xcodeproj/xcshareddata/xcschemes/WebDriverAgentRunner.xcscheme +++ b/WebDriverAgent.xcodeproj/xcshareddata/xcschemes/WebDriverAgentRunner.xcscheme @@ -1,7 +1,7 @@ + version = "1.7"> diff --git a/WebDriverAgentLib/Utilities/FBBroadcastManager.m b/WebDriverAgentLib/Utilities/FBBroadcastManager.m index 711c5507d..3947c512d 100644 --- a/WebDriverAgentLib/Utilities/FBBroadcastManager.m +++ b/WebDriverAgentLib/Utilities/FBBroadcastManager.m @@ -8,12 +8,16 @@ #import "FBBroadcastManager.h" +#include +#import + #import "FBBroadcastControlServer.h" #import "FBBroadcastPickerHost.h" #import "FBBroadcastProtocol.h" #import "FBConfiguration.h" #import "FBLogger.h" #import "FBRunLoopSpinner.h" +#import "FBUnattachedAppLauncher.h" #import "FBVideoStreamManager.h" #import "XCUIApplication.h" #import "XCUIApplication+FBHelpers.h" @@ -23,6 +27,14 @@ #if !TARGET_OS_SIMULATOR && !TARGET_OS_TV static const NSTimeInterval FOREGROUND_TIMEOUT = 5.0; static const NSTimeInterval CONFIRM_BUTTON_TIMEOUT = 10.0; +// The picker press is dropped silently by the system when it fires before the scene is fully +// active, so it is re-fired periodically until the confirmation sheet shows up. +static const uint64_t PICKER_RETRIGGER_INTERVAL_MS = 2000; + +static uint64_t FBBroadcastNowMs(void) +{ + return clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) / NSEC_PER_MSEC; +} #endif static const NSTimeInterval STOP_TIMEOUT = 5.0; @@ -131,28 +143,47 @@ - (BOOL)startBroadcastWithTimeout:(NSTimeInterval)timeout [self startListening]; } + uint64_t startedMs = FBBroadcastNowMs(); XCUIApplication *runner = [[XCUIApplication alloc] initWithBundleIdentifier:(NSString *)NSBundle.mainBundle.bundleIdentifier]; XCUIApplication *previousApp = nil; - if (restoreForegroundApp) { + BOOL runnerIsActive = UIApplication.sharedApplication.applicationState == UIApplicationStateActive; + // When the runner is already frontmost there is neither an app to restore nor a need for the + // (slow) active-app lookup. + if (restoreForegroundApp && !runnerIsActive) { XCUIApplication *active = XCUIApplication.fb_activeApplication; if (nil != active && ![active.bundleID isEqualToString:runner.bundleID]) { previousApp = active; } + [FBLogger logFmt:@"broadcast/start: active-app lookup finished after %llums", FBBroadcastNowMs() - startedMs]; } - // The picker can only present from a foreground app, so bring the runner up first. - [runner activate]; - BOOL foregrounded = [[[[FBRunLoopSpinner new] timeout:FOREGROUND_TIMEOUT] interval:0.1] spinUntilTrue:^BOOL{ - return runner.state == XCUIApplicationStateRunningForeground; - }]; - if (!foregrounded) { - if (error) { - *error = [NSError errorWithDomain:FBBroadcastManagerErrorDomain - code:FBBroadcastManagerErrorTimeout - userInfo:@{NSLocalizedDescriptionKey: @"The runner app could not be brought to the foreground to present the broadcast picker"}]; + // The picker can only present from a foreground app, so bring the runner up first. The + // LaunchServices route is used instead of XCUIApplication.activate because activating the + // runner from inside itself blocks on a self-quiescence wait that can only ever time out: + // the waiting thread is the very main thread whose idleness is being awaited. + if (!runnerIsActive) { + BOOL launched = [FBUnattachedAppLauncher launchAppWithBundleId:(NSString *)NSBundle.mainBundle.bundleIdentifier]; + BOOL foregrounded = launched && [[[[FBRunLoopSpinner new] timeout:2.0] interval:0.05] spinUntilTrue:^BOOL{ + return UIApplication.sharedApplication.applicationState == UIApplicationStateActive; + }]; + if (!foregrounded) { + // Reliable but slow fallback: XCTest's activation waits out its quiescence timeout. + [FBLogger log:@"broadcast/start: LaunchServices foregrounding failed; falling back to XCUIApplication.activate"]; + [runner activate]; + foregrounded = [[[[FBRunLoopSpinner new] timeout:FOREGROUND_TIMEOUT] interval:0.05] spinUntilTrue:^BOOL{ + return UIApplication.sharedApplication.applicationState == UIApplicationStateActive; + }]; + } + if (!foregrounded) { + if (error) { + *error = [NSError errorWithDomain:FBBroadcastManagerErrorDomain + code:FBBroadcastManagerErrorTimeout + userInfo:@{NSLocalizedDescriptionKey: @"The runner app could not be brought to the foreground to present the broadcast picker"}]; + } + return NO; } - return NO; } + [FBLogger logFmt:@"broadcast/start: runner foreground after %llums", FBBroadcastNowMs() - startedMs]; NSError *pickerError; if (![FBBroadcastPickerHost triggerPickerWithPreferredExtension:FBConfiguration.broadcastExtensionBundleId @@ -164,13 +195,16 @@ - (BOOL)startBroadcastWithTimeout:(NSTimeInterval)timeout } return NO; } + [FBLogger logFmt:@"broadcast/start: picker triggered after %llums", FBBroadcastNowMs() - startedMs]; // The confirmation sheet is hosted by different processes depending on the iOS version, so // look for the confirm button in both the system app and the runner itself. NSArray *labels = confirmButtonLabels.count > 0 ? confirmButtonLabels : @[@"Start Broadcast"]; + NSArray *candidateApps = @[XCUIApplication.fb_systemApplication, runner]; __block XCUIElement *confirmButton = nil; - [[[[FBRunLoopSpinner new] timeout:CONFIRM_BUTTON_TIMEOUT] interval:0.3] spinUntilTrue:^BOOL{ - for (XCUIApplication *app in @[XCUIApplication.fb_systemApplication, runner]) { + __block uint64_t lastTriggerMs = FBBroadcastNowMs(); + [[[[FBRunLoopSpinner new] timeout:CONFIRM_BUTTON_TIMEOUT] interval:0.25] spinUntilTrue:^BOOL{ + for (XCUIApplication *app in candidateApps) { for (NSString *label in labels) { XCUIElement *candidate = app.buttons[label]; if (candidate.exists) { @@ -184,6 +218,12 @@ - (BOOL)startBroadcastWithTimeout:(NSTimeInterval)timeout return YES; } } + if (FBBroadcastNowMs() - lastTriggerMs >= PICKER_RETRIGGER_INTERVAL_MS) { + lastTriggerMs = FBBroadcastNowMs(); + [FBLogger log:@"broadcast/start: confirmation sheet not visible yet; re-triggering the picker"]; + [FBBroadcastPickerHost triggerPickerWithPreferredExtension:FBConfiguration.broadcastExtensionBundleId + error:nil]; + } return NO; }]; if (nil == confirmButton) { @@ -196,11 +236,13 @@ - (BOOL)startBroadcastWithTimeout:(NSTimeInterval)timeout return NO; } [confirmButton tap]; + [FBLogger logFmt:@"broadcast/start: confirmation tapped after %llums", FBBroadcastNowMs() - startedMs]; // Cover the system's 3-2-1 countdown plus the extension's connect/HELLO round trip. BOOL connected = [[[[FBRunLoopSpinner new] timeout:(timeout > 0 ? timeout : 30.0)] interval:0.3] spinUntilTrue:^BOOL{ return self.isExtensionConnected; }]; + [FBLogger logFmt:@"broadcast/start: %@ after %llums", connected ? @"extension connected" : @"extension connect timeout", FBBroadcastNowMs() - startedMs]; [FBBroadcastPickerHost dismiss]; if (!connected) { if (error) { From e9adf6b25ebaa593ffa8421c664c4e7eabf96d0b Mon Sep 17 00:00:00 2001 From: timo <44401485+Timo972@users.noreply.github.com> Date: Wed, 10 Jun 2026 23:10:09 +0200 Subject: [PATCH 06/11] feat(screencapture): cache screenshot bitmap contexts --- .../Utilities/FBPixelBufferConverter.m | 67 ++++++++++++++----- 1 file changed, 50 insertions(+), 17 deletions(-) diff --git a/WebDriverAgentLib/Utilities/FBPixelBufferConverter.m b/WebDriverAgentLib/Utilities/FBPixelBufferConverter.m index f948ac508..a8251ac03 100644 --- a/WebDriverAgentLib/Utilities/FBPixelBufferConverter.m +++ b/WebDriverAgentLib/Utilities/FBPixelBufferConverter.m @@ -11,6 +11,7 @@ #import NSErrorDomain const FBPixelBufferConverterErrorDomain = @"com.facebook.WebDriverAgent.FBPixelBufferConverter"; +static CFStringRef const FBPixelBufferBitmapContextAttachmentKey = CFSTR("com.facebook.WebDriverAgent.FBPixelBufferConverter.BitmapContext"); static size_t FBEvenDimension(size_t value) { @@ -23,6 +24,9 @@ static size_t FBEvenDimension(size_t value) @interface FBPixelBufferConverter () @property (nonatomic) CVPixelBufferPoolRef pixelBufferPool; @property (nonatomic) CGColorSpaceRef colorSpace; + +- (nullable CGContextRef)copyBitmapContextForPixelBuffer:(CVPixelBufferRef)pixelBuffer + error:(NSError **)error CF_RETURNS_RETAINED; @end @implementation FBPixelBufferConverter @@ -107,29 +111,14 @@ - (nullable CVPixelBufferRef)copyPixelBufferFromCGImage:(CGImageRef)image } CVPixelBufferLockBaseAddress(pixelBuffer, 0); - void *baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer); - CGContextRef context = CGBitmapContextCreate(baseAddress, - self.width, - self.height, - 8, - CVPixelBufferGetBytesPerRow(pixelBuffer), - self.colorSpace, - kCGImageAlphaNoneSkipFirst | kCGBitmapByteOrder32Little); + CGContextRef context = [self copyBitmapContextForPixelBuffer:pixelBuffer + error:error]; if (NULL == context) { CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); CVPixelBufferRelease(pixelBuffer); - if (error) { - *error = [NSError errorWithDomain:FBPixelBufferConverterErrorDomain - code:5 - userInfo:@{NSLocalizedDescriptionKey: @"Cannot create a bitmap context for the pixel buffer"}]; - } return NULL; } - // Fill the whole frame with black to produce the letterbox/pillarbox padding. - CGContextSetRGBFillColor(context, 0.0, 0.0, 0.0, 1.0); - CGContextFillRect(context, CGRectMake(0, 0, self.width, self.height)); - // Scale to fit while preserving the aspect ratio and center the result. CGFloat imageWidth = (CGFloat)CGImageGetWidth(image); CGFloat imageHeight = (CGFloat)CGImageGetHeight(image); @@ -139,6 +128,16 @@ - (nullable CVPixelBufferRef)copyPixelBufferFromCGImage:(CGImageRef)image CGFloat targetHeight = imageHeight * scale; CGFloat originX = ((CGFloat)self.width - targetWidth) / 2.0; CGFloat originY = ((CGFloat)self.height - targetHeight) / 2.0; + BOOL needsPadding = originX > 0.5 + || originY > 0.5 + || targetWidth < (CGFloat)self.width - 0.5 + || targetHeight < (CGFloat)self.height - 0.5; + if (needsPadding) { + // Fill only when the frame needs letterbox/pillarbox padding; a full-frame draw overwrites + // every pixel and does not need the clear. + CGContextSetRGBFillColor(context, 0.0, 0.0, 0.0, 1.0); + CGContextFillRect(context, CGRectMake(0, 0, self.width, self.height)); + } CGContextDrawImage(context, CGRectMake(originX, originY, targetWidth, targetHeight), image); } @@ -147,6 +146,40 @@ - (nullable CVPixelBufferRef)copyPixelBufferFromCGImage:(CGImageRef)image return pixelBuffer; } +- (nullable CGContextRef)copyBitmapContextForPixelBuffer:(CVPixelBufferRef)pixelBuffer + error:(NSError **)error +{ + CGContextRef cachedContext = (CGContextRef)CVBufferCopyAttachment(pixelBuffer, + FBPixelBufferBitmapContextAttachmentKey, + NULL); + if (NULL != cachedContext) { + return cachedContext; + } + + void *baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer); + CGContextRef context = CGBitmapContextCreate(baseAddress, + self.width, + self.height, + 8, + CVPixelBufferGetBytesPerRow(pixelBuffer), + self.colorSpace, + kCGImageAlphaNoneSkipFirst | kCGBitmapByteOrder32Little); + if (NULL == context) { + if (error) { + *error = [NSError errorWithDomain:FBPixelBufferConverterErrorDomain + code:5 + userInfo:@{NSLocalizedDescriptionKey: @"Cannot create a bitmap context for the pixel buffer"}]; + } + return NULL; + } + + CVBufferSetAttachment(pixelBuffer, + FBPixelBufferBitmapContextAttachmentKey, + context, + kCVAttachmentMode_ShouldNotPropagate); + return context; +} + - (void)dealloc { if (NULL != _pixelBufferPool) { From 51aab8975ff835bbb84e7bc6a380273ca157c80e Mon Sep 17 00:00:00 2001 From: timo <44401485+Timo972@users.noreply.github.com> Date: Wed, 10 Jun 2026 23:20:13 +0200 Subject: [PATCH 07/11] feat(screencapture): make broadcast/start safe to call concurrently and repeatedly Starting a second broadcast while one is live or still launching makes iOS kill both. Three gaps allowed exactly that: - The start dance spins the main run loop, so a second /broadcast/start request could be dispatched re-entrantly mid-dance and drive the picker again. A startInProgress gate now serializes starts; followers await the leader's outcome and return its result. - A live broadcast whose extension is momentarily disconnected (crash, TCP reconnect window) passed the isExtensionConnected idempotency check. The dance now checks UIScreen.isCaptured first and waits for the extension instead of stacking a second broadcast. - A missed XCUIElement tap on the confirmation button (sheet dismissed in between) was recorded by XCTest as a test failure, tearing down the whole WDA session. The tap now goes through WDA's own event synthesis at the button's coordinates, so a miss surfaces as a plain connect timeout. Side effect: no springboard-idle wait, which cuts the confirmation tap from ~4.8s to ~1.9s and time-to-connected from ~8.4s to ~5.5s on an iPhone XS. Verified on device: two concurrent starts produce one broadcast and two "connected" responses; a repeat start while connected returns in ~30ms. Co-Authored-By: Claude Fable 5 --- .../Utilities/FBBroadcastManager.m | 105 +++++++++++++++++- 1 file changed, 99 insertions(+), 6 deletions(-) diff --git a/WebDriverAgentLib/Utilities/FBBroadcastManager.m b/WebDriverAgentLib/Utilities/FBBroadcastManager.m index 3947c512d..47e304995 100644 --- a/WebDriverAgentLib/Utilities/FBBroadcastManager.m +++ b/WebDriverAgentLib/Utilities/FBBroadcastManager.m @@ -17,8 +17,10 @@ #import "FBConfiguration.h" #import "FBLogger.h" #import "FBRunLoopSpinner.h" +#import "FBScreen.h" #import "FBUnattachedAppLauncher.h" #import "FBVideoStreamManager.h" +#import "XCUIApplication+FBTouchAction.h" #import "XCUIApplication.h" #import "XCUIApplication+FBHelpers.h" @@ -46,6 +48,15 @@ @interface FBBroadcastManager () @property (atomic, nullable) NSDate *connectedAt; @property (atomic, nullable) NSDate *lastHeartbeatAt; @property (atomic) BOOL paused; +/** YES while a start dance is driving the system UI (used to serialize concurrent starts). */ +@property (atomic) BOOL startInProgress; + +#if !TARGET_OS_SIMULATOR && !TARGET_OS_TV +- (BOOL)performBroadcastStartWithTimeout:(NSTimeInterval)timeout + confirmButtonLabels:(NSArray *)confirmButtonLabels + restoreForegroundApp:(BOOL)restoreForegroundApp + error:(NSError **)error; +#endif @end @@ -143,6 +154,66 @@ - (BOOL)startBroadcastWithTimeout:(NSTimeInterval)timeout [self startListening]; } + // Serialize concurrent starts: the dance below spins the main run loop, so another + // /broadcast/start request can be dispatched re-entrantly while the first one is still driving + // the system UI. Starting a second broadcast while one is launching makes iOS kill both, so + // followers just await the leader's outcome. + if (self.startInProgress) { + [FBLogger log:@"broadcast/start: another start attempt is already in progress; awaiting its outcome"]; + [[[[FBRunLoopSpinner new] timeout:(timeout > 0 ? timeout : 30.0)] interval:0.3] spinUntilTrue:^BOOL{ + return self.isExtensionConnected || !self.startInProgress; + }]; + if (self.isExtensionConnected) { + return YES; + } + if (error) { + *error = [NSError errorWithDomain:FBBroadcastManagerErrorDomain + code:FBBroadcastManagerErrorTimeout + userInfo:@{NSLocalizedDescriptionKey: @"A concurrent broadcast start attempt finished without the extension connecting"}]; + } + return NO; + } + + self.startInProgress = YES; + @try { + return [self performBroadcastStartWithTimeout:timeout + confirmButtonLabels:confirmButtonLabels + restoreForegroundApp:restoreForegroundApp + error:error]; + } @finally { + self.startInProgress = NO; + } +#endif +} + +#if !TARGET_OS_SIMULATOR && !TARGET_OS_TV +- (BOOL)performBroadcastStartWithTimeout:(NSTimeInterval)timeout + confirmButtonLabels:(NSArray *)confirmButtonLabels + restoreForegroundApp:(BOOL)restoreForegroundApp + error:(NSError **)error +{ + // The screen may already be captured by a live broadcast even though the extension is not + // connected (it crashed, or it is between TCP reconnect attempts). Driving the picker on top + // of a live broadcast makes iOS kill both, so wait for the extension instead. + if (UIScreen.mainScreen.isCaptured) { + [FBLogger log:@"broadcast/start: the screen is already being captured; waiting for the extension to connect instead of starting another broadcast"]; + [[[[FBRunLoopSpinner new] timeout:5.0] interval:0.2] spinUntilTrue:^BOOL{ + return self.isExtensionConnected || !UIScreen.mainScreen.isCaptured; + }]; + if (self.isExtensionConnected) { + return YES; + } + if (UIScreen.mainScreen.isCaptured) { + if (error) { + *error = [NSError errorWithDomain:FBBroadcastManagerErrorDomain + code:FBBroadcastManagerErrorTimeout + userInfo:@{NSLocalizedDescriptionKey: @"The screen is already being captured (an active broadcast or recording), but the WebDriverAgent broadcast extension did not connect. Stop the existing capture (e.g. via the status bar pill) and retry"}]; + } + return NO; + } + // The capture ended while waiting; fall through and start a fresh broadcast. + } + uint64_t startedMs = FBBroadcastNowMs(); XCUIApplication *runner = [[XCUIApplication alloc] initWithBundleIdentifier:(NSString *)NSBundle.mainBundle.bundleIdentifier]; XCUIApplication *previousApp = nil; @@ -201,20 +272,23 @@ - (BOOL)startBroadcastWithTimeout:(NSTimeInterval)timeout // look for the confirm button in both the system app and the runner itself. NSArray *labels = confirmButtonLabels.count > 0 ? confirmButtonLabels : @[@"Start Broadcast"]; NSArray *candidateApps = @[XCUIApplication.fb_systemApplication, runner]; - __block XCUIElement *confirmButton = nil; + __block BOOL confirmButtonFound = NO; + __block CGRect confirmFrame = CGRectZero; __block uint64_t lastTriggerMs = FBBroadcastNowMs(); [[[[FBRunLoopSpinner new] timeout:CONFIRM_BUTTON_TIMEOUT] interval:0.25] spinUntilTrue:^BOOL{ for (XCUIApplication *app in candidateApps) { for (NSString *label in labels) { XCUIElement *candidate = app.buttons[label]; if (candidate.exists) { - confirmButton = candidate; + confirmFrame = candidate.frame; + confirmButtonFound = YES; return YES; } } XCUIElement *prefixMatch = [app.buttons matchingPredicate:[NSPredicate predicateWithFormat:@"label BEGINSWITH[c] 'Start'"]].firstMatch; if (prefixMatch.exists) { - confirmButton = prefixMatch; + confirmFrame = prefixMatch.frame; + confirmButtonFound = YES; return YES; } } @@ -226,7 +300,7 @@ - (BOOL)startBroadcastWithTimeout:(NSTimeInterval)timeout } return NO; }]; - if (nil == confirmButton) { + if (!confirmButtonFound || CGRectIsEmpty(confirmFrame)) { [FBBroadcastPickerHost dismiss]; if (error) { *error = [NSError errorWithDomain:FBBroadcastManagerErrorDomain @@ -235,7 +309,26 @@ - (BOOL)startBroadcastWithTimeout:(NSTimeInterval)timeout } return NO; } - [confirmButton tap]; + // Tap via WDA's own event synthesis instead of XCUIElement.tap: a missed XCUIElement tap + // (e.g. the sheet dismissed in between) records an XCTest failure that tears down the whole + // test session, whereas a missed synthesized tap is harmless and surfaces as a connect timeout. + CGFloat scale = (CGFloat)[FBScreen scale]; + CGPoint center = CGPointMake(CGRectGetMidX(confirmFrame) * scale, CGRectGetMidY(confirmFrame) * scale); + NSArray *tapActions = @[ + @{@"type": @"pointerDown", @"x": @(center.x), @"y": @(center.y)}, + @{@"type": @"pause", @"duration": @60}, + @{@"type": @"pointerUp", @"x": @(center.x), @"y": @(center.y)}, + ]; + NSError *tapError; + if (![runner fb_performMobilerunActions:tapActions scale:scale error:&tapError]) { + [FBBroadcastPickerHost dismiss]; + if (error) { + *error = [NSError errorWithDomain:FBBroadcastManagerErrorDomain + code:FBBroadcastManagerErrorPicker + userInfo:@{NSLocalizedDescriptionKey: [NSString stringWithFormat:@"Cannot tap the broadcast confirmation button: %@", tapError.localizedDescription]}]; + } + return NO; + } [FBLogger logFmt:@"broadcast/start: confirmation tapped after %llums", FBBroadcastNowMs() - startedMs]; // Cover the system's 3-2-1 countdown plus the extension's connect/HELLO round trip. @@ -257,8 +350,8 @@ - (BOOL)startBroadcastWithTimeout:(NSTimeInterval)timeout [previousApp activate]; } return YES; -#endif } +#endif - (BOOL)stopBroadcastWithError:(NSError **)error { From b988bcbd1747f75181338a587c8ed56aee3a3a8f Mon Sep 17 00:00:00 2001 From: timo <44401485+Timo972@users.noreply.github.com> Date: Thu, 11 Jun 2026 01:12:37 +0200 Subject: [PATCH 08/11] feat(screencapture): add pipeline metrics to the heartbeat and fix the fps gate Each extension heartbeat now carries per-session counters (samplesIn, accepted, encoded, droppedFpsGate, droppedReplaced, droppedPool, encode submit-to-callback latency last/avg) plus derived per-second rates, the ReplayKit delivery rate and the loopback socket's outstanding bytes. They surface verbatim under 'heartbeat' in GET /mobilerun/screencapture/broadcast, so a low consumer-side fps can be attributed to delivery, a specific pipeline stage or backpressure without reproducing locally. The first measurement immediately located the loss: ReplayKit delivered 42-44 fps while only ~22 fps passed the fps gate - the gate paced by minimum gap from the last accepted frame, which beats against jittery ~30-60Hz delivery (a frame arriving 1ms early is dropped and the next accepted gap doubles). Replace it with a due-time accumulator that admits exactly one frame per interval on average regardless of arrival jitter, with a re-anchor clamp so stalls do not admit bursts. Measured on an iPhone XS: accepted rate went from a flat ~22/s to 30/s whenever delivery sustains it; encode latency ~10ms, all other drop counters zero. Co-Authored-By: Claude Fable 5 --- .../FBExtBroadcastClient.m | 49 ++++++++++++++- .../FBExtSessionPipeline.h | 9 +++ .../FBExtSessionPipeline.m | 60 ++++++++++++++++++- 3 files changed, 112 insertions(+), 6 deletions(-) diff --git a/WebDriverAgentBroadcast/FBExtBroadcastClient.m b/WebDriverAgentBroadcast/FBExtBroadcastClient.m index adab5cd82..c15db353e 100644 --- a/WebDriverAgentBroadcast/FBExtBroadcastClient.m +++ b/WebDriverAgentBroadcast/FBExtBroadcastClient.m @@ -8,6 +8,8 @@ #import "FBExtBroadcastClient.h" +#include + #import "FBBroadcastProtocol.h" #import "FBExtLogging.h" #import "GCDAsyncSocket.h" @@ -36,9 +38,18 @@ @interface FBExtBroadcastClient () @property (atomic) BOOL connected; @property (nonatomic) size_t outstandingWriteBytes; @property (nonatomic) FBBroadcastMessageHeader pendingHeader; +// Previous heartbeat totals used to derive per-second rates (queue-confined). +@property (nonatomic, nullable) NSDictionary *previousPipelineTotals; +@property (nonatomic) uint64_t previousFramesReceived; +@property (nonatomic) uint64_t lastHeartbeatAtMs; @end +static double FBExtRatePerSec(double delta, double intervalSec) +{ + return round(delta / intervalSec * 10) / 10; +} + @implementation FBExtBroadcastClient - (instancetype)init @@ -157,13 +168,45 @@ - (void)stopHeartbeat - (void)sendHeartbeat { - NSData *message = FBBroadcastEncodeJSONMessage(FBBroadcastMessageTypeHeartbeat, 0, @{ + uint64_t nowMs = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) / NSEC_PER_MSEC; + double intervalSec = self.lastHeartbeatAtMs > 0 + ? MAX(0.001, (double)(nowMs - self.lastHeartbeatAtMs) / 1000.0) + : HEARTBEAT_INTERVAL; + uint64_t framesReceived = self.framesReceived; + + NSMutableDictionary *payload = [NSMutableDictionary dictionaryWithDictionary:@{ FBBroadcastKeyState: self.paused ? @"paused" : @"active", - FBBroadcastKeyFramesReceived: @(self.framesReceived), + FBBroadcastKeyFramesReceived: @(framesReceived), FBBroadcastKeyOrientation: @(self.currentOrientation), FBBroadcastKeyScreenWidth: @(self.screenWidth), FBBroadcastKeyScreenHeight: @(self.screenHeight), - }); + }]; + // ReplayKit delivery rate plus loopback backpressure, so a low consumer-side fps can be + // attributed to delivery, a pipeline stage (see the per-pipeline counters) or the socket. + payload[@"framesReceivedPerSec"] = @(FBExtRatePerSec((double)(framesReceived - self.previousFramesReceived), intervalSec)); + payload[@"socketOutstandingBytes"] = @(self.outstandingWriteBytes); + + NSMutableDictionary *pipelinesJson = [NSMutableDictionary dictionary]; + NSMutableDictionary *newTotals = [NSMutableDictionary dictionary]; + [self.pipelines enumerateKeysAndObjectsUsingBlock:^(NSNumber *sessionId, FBExtSessionPipeline *pipeline, BOOL *stop) { + NSDictionary *metrics = [pipeline metricsSnapshot]; + NSMutableDictionary *entry = [metrics mutableCopy]; + NSDictionary *previous = self.previousPipelineTotals[sessionId.stringValue]; + for (NSString *key in @[@"samplesIn", @"accepted", @"encoded"]) { + double delta = metrics[key].doubleValue - [previous[key] doubleValue]; + entry[[key stringByAppendingString:@"PerSec"]] = @(FBExtRatePerSec(delta, intervalSec)); + } + pipelinesJson[sessionId.stringValue] = entry; + newTotals[sessionId.stringValue] = metrics; + }]; + if (pipelinesJson.count > 0) { + payload[@"pipelines"] = pipelinesJson; + } + self.previousPipelineTotals = newTotals; + self.previousFramesReceived = framesReceived; + self.lastHeartbeatAtMs = nowMs; + + NSData *message = FBBroadcastEncodeJSONMessage(FBBroadcastMessageTypeHeartbeat, 0, payload); if (nil != message) { [self sendProtocolMessage:message isDroppable:NO]; } diff --git a/WebDriverAgentBroadcast/FBExtSessionPipeline.h b/WebDriverAgentBroadcast/FBExtSessionPipeline.h index b5b991bb6..b90a3c7a0 100644 --- a/WebDriverAgentBroadcast/FBExtSessionPipeline.h +++ b/WebDriverAgentBroadcast/FBExtSessionPipeline.h @@ -57,6 +57,15 @@ NS_ASSUME_NONNULL_BEGIN /** Forces the next encoded frame to be a key frame. */ - (void)requestKeyFrame; +/** + A point-in-time snapshot of this pipeline's counters, safe to call from any thread: + samplesIn (frames offered by ReplayKit), accepted (passed the fps gate), encoded (encoder + callbacks), droppedFpsGate, droppedReplaced (latched frame superseded before processing), + droppedPool (scaled-buffer pool exhausted), encodeLatencyMsLast/encodeLatencyMsAvg + (submit-to-encoder-callback time). + */ +- (NSDictionary *)metricsSnapshot; + /** Stops the encoder and releases the scaler and buffer pool. */ - (void)teardown; diff --git a/WebDriverAgentBroadcast/FBExtSessionPipeline.m b/WebDriverAgentBroadcast/FBExtSessionPipeline.m index 4a3d3e562..46702650f 100644 --- a/WebDriverAgentBroadcast/FBExtSessionPipeline.m +++ b/WebDriverAgentBroadcast/FBExtSessionPipeline.m @@ -35,6 +35,8 @@ @interface FBExtSessionPipeline () { @property (nonatomic) NSUInteger height; @property (nonatomic) NSUInteger fps; @property (nonatomic) uint64_t lastSubmitTimeMs; +/** The next monotonic timestamp at which a frame is due (fps gate accumulator). */ +@property (nonatomic) uint64_t nextDueMs; @property (nonatomic) BOOL inFlight; @property (nonatomic) uint64_t pendingSubmitTimeMs; @property (nonatomic) uint8_t pendingOrientation; @@ -43,6 +45,17 @@ @interface FBExtSessionPipeline () { @property (nonatomic) BOOL directSourceEncodingDisabled; @property (nonatomic, nullable, copy) NSData *lastSentParameterSets; +// Lightweight diagnostics, exposed via metricsSnapshot/the heartbeat. Increments are not +// strictly atomic RMW, which is acceptable for metrics. +@property (atomic) uint64_t samplesInCount; +@property (atomic) uint64_t acceptedCount; +@property (atomic) uint64_t encodedCount; +@property (atomic) uint64_t droppedFpsGateCount; +@property (atomic) uint64_t droppedReplacedCount; +@property (atomic) uint64_t droppedPoolCount; +@property (atomic) uint64_t lastEncodeLatencyMs; +@property (atomic) double avgEncodeLatencyMs; + - (void)replacePendingSampleBuffer:(CMSampleBufferRef)sampleBuffer atTimeMs:(uint64_t)nowMs orientation:(uint8_t)orientation; @@ -148,17 +161,30 @@ - (nullable instancetype)initWithSessionId:(uint32_t)sessionId - (void)submitSampleBuffer:(CMSampleBufferRef)sampleBuffer orientation:(uint8_t)orientation { + self.samplesInCount += 1; if (!self.active) { return; } - // Respect this session's framerate even though ReplayKit may deliver faster. + // Respect this session's framerate even though ReplayKit may deliver faster. The gate is a + // due-time accumulator rather than a minimum gap from the last accepted frame: gap-based + // pacing beats against jittery ~30Hz delivery (a frame arriving 1ms early is dropped and the + // next accepted gap doubles), which measured ~22fps out of a 42fps input. The accumulator + // admits exactly one frame per interval on average regardless of arrival jitter. uint64_t nowMs = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) / NSEC_PER_MSEC; uint64_t minIntervalMs = self.fps > 0 ? (uint64_t)(1000 / self.fps) : 0; @synchronized (self) { - if (minIntervalMs > 1 && nowMs - self.lastSubmitTimeMs < minIntervalMs - 1) { - return; + if (minIntervalMs > 1) { + if (nowMs + 1 < self.nextDueMs) { + self.droppedFpsGateCount += 1; + return; + } + // Re-anchor after stalls so accumulated due time does not admit a burst. + self.nextDueMs = (self.nextDueMs == 0 || nowMs > self.nextDueMs + minIntervalMs) + ? nowMs + minIntervalMs + : self.nextDueMs + minIntervalMs; } + self.acceptedCount += 1; // Never block the ReplayKit sample queue. If a frame is already being scaled/submitted, // retain only the newest due sample and replace any older pending sample. @@ -188,6 +214,7 @@ - (void)replacePendingSampleBuffer:(CMSampleBufferRef)sampleBuffer self.pendingOrientation = orientation; if (NULL != previous) { CFRelease(previous); + self.droppedReplacedCount += 1; } } @@ -278,6 +305,7 @@ - (void)processRetainedSampleBuffer:(CMSampleBufferRef)sampleBuffer &scaledBuffer); if (poolStatus != kCVReturnSuccess || NULL == scaledBuffer) { // The pool is exhausted (encoder still holds the buffers) - drop the frame instead of growing. + self.droppedPoolCount += 1; return; } @@ -300,6 +328,20 @@ - (void)requestKeyFrame [self.encoder requestKeyFrame]; } +- (NSDictionary *)metricsSnapshot +{ + return @{ + @"samplesIn": @(self.samplesInCount), + @"accepted": @(self.acceptedCount), + @"encoded": @(self.encodedCount), + @"droppedFpsGate": @(self.droppedFpsGateCount), + @"droppedReplaced": @(self.droppedReplacedCount), + @"droppedPool": @(self.droppedPoolCount), + @"encodeLatencyMsLast": @(self.lastEncodeLatencyMs), + @"encodeLatencyMsAvg": @(round(self.avgEncodeLatencyMs * 10) / 10), + }; +} + - (void)teardown { self.active = NO; @@ -348,6 +390,18 @@ - (void)videoEncoder:(FBVideoEncoder *)encoder isKeyFrame:(BOOL)isKeyFrame presentationTimeUs:(uint64_t)presentationTimeUs { + self.encodedCount += 1; + // The frame pts is the monotonic submit time, so callback-time minus pts is the + // scale+encode latency. + uint64_t nowMs = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) / NSEC_PER_MSEC; + uint64_t submitMs = presentationTimeUs / 1000; + if (nowMs >= submitMs) { + uint64_t latencyMs = nowMs - submitMs; + self.lastEncodeLatencyMs = latencyMs; + double avg = self.avgEncodeLatencyMs; + self.avgEncodeLatencyMs = avg <= 0 ? (double)latencyMs : avg * 0.9 + (double)latencyMs * 0.1; + } + if (!self.active || annexBPictureData.length == 0) { return; } From 824338f3b754089bc4116853db57199f4a1d9886 Mon Sep 17 00:00:00 2001 From: timo <44401485+Timo972@users.noreply.github.com> Date: Thu, 11 Jun 2026 01:12:37 +0200 Subject: [PATCH 09/11] fix(webserver): use-after-free of the HTTP server during teardown Three on-device crash reports share one signature: SIGSEGV (pointer authentication failure) in objc_msgSend inside -[RoutingConnection setHeadersForResponse:isError:] on an HTTPConnection thread. The vendored CocoaHTTPServer/RoutingHTTPServer never retain the server from the connection side (HTTPConfig.server and RoutingConnection's http ivar are both __unsafe_unretained), so GET /wda/shutdown - which tears the server down via stopServing while other keep-alive connections are still replying on their own GCD queues - leaves those replies dereferencing a freed RoutingHTTPServer. Continuous endpoint polling (e.g. the droidrun devicekit) makes hitting the race likely. Make HTTPConfig.server strong (connections keep the server alive until their replies finish; the server->connections->config cycle breaks when connections die) and RoutingConnection's server reference weak, and snapshot the route response's mutable headers dictionary instead of aliasing it. Co-Authored-By: Claude Fable 5 --- WebDriverAgentLib/Vendor/CocoaHTTPServer/HTTPConnection.h | 7 +++++-- .../Vendor/RoutingHTTPServer/RoutingConnection.m | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/WebDriverAgentLib/Vendor/CocoaHTTPServer/HTTPConnection.h b/WebDriverAgentLib/Vendor/CocoaHTTPServer/HTTPConnection.h index e1868532f..f2d7b396a 100644 --- a/WebDriverAgentLib/Vendor/CocoaHTTPServer/HTTPConnection.h +++ b/WebDriverAgentLib/Vendor/CocoaHTTPServer/HTTPConnection.h @@ -15,7 +15,10 @@ @interface HTTPConfig : NSObject { - HTTPServer __unsafe_unretained *server; + // Strong on purpose: connections reply on their own queues, so an unretained server is a + // use-after-free once the server is torn down (e.g. via /wda/shutdown) while replies are in + // flight. The server->connections->config->server cycle is broken when connections die. + HTTPServer __strong *server; NSString __strong *documentRoot; dispatch_queue_t queue; } @@ -23,7 +26,7 @@ - (id)initWithServer:(HTTPServer *)server documentRoot:(NSString *)documentRoot; - (id)initWithServer:(HTTPServer *)server documentRoot:(NSString *)documentRoot queue:(dispatch_queue_t)q; -@property (nonatomic, unsafe_unretained, readonly) HTTPServer *server; +@property (nonatomic, strong, readonly) HTTPServer *server; @property (nonatomic, strong, readonly) NSString *documentRoot; @property (nonatomic, readonly) dispatch_queue_t queue; diff --git a/WebDriverAgentLib/Vendor/RoutingHTTPServer/RoutingConnection.m b/WebDriverAgentLib/Vendor/RoutingHTTPServer/RoutingConnection.m index 3eee19267..99769cd61 100644 --- a/WebDriverAgentLib/Vendor/RoutingHTTPServer/RoutingConnection.m +++ b/WebDriverAgentLib/Vendor/RoutingHTTPServer/RoutingConnection.m @@ -8,7 +8,9 @@ #pragma clang diagnostic ignored "-Wundeclared-selector" @implementation RoutingConnection { - __unsafe_unretained RoutingHTTPServer *http; + // Weak (was unsafe_unretained): replies run on connection queues and must not dereference a + // torn-down server; messaging nil is harmless, a dangling pointer crashes in objc_msgSend. + __weak RoutingHTTPServer *http; NSDictionary *headers; } @@ -63,7 +65,8 @@ - (void)processBodyData:(NSData *)postDataChunk { RouteResponse *response = [http routeMethod:method withPath:path parameters:params request:request connection:self]; if (response != nil) { - headers = response.headers; + // Snapshot instead of aliasing the route response's live (mutable) dictionary. + headers = [response.headers copy]; return response.proxiedResponse; } From 3d6fcdf9505f47d2798790e20fed4f51097e9906 Mon Sep 17 00:00:00 2001 From: timo <44401485+Timo972@users.noreply.github.com> Date: Thu, 11 Jun 2026 01:43:32 +0200 Subject: [PATCH 10/11] feat(screencapture): repeat the last frame to hold the requested fps ReplayKit only delivers frames while the screen changes and VideoToolbox has no Android-style KEY_REPEAT_PREVIOUS_FRAME_AFTER mode, so streams stalled on static content. Each pipeline now retains its most recent (pool-owned) scaled frame and a timer on the pipeline queue re-encodes it whenever no live frame arrived within the frame interval, keeping the output cadence at the session's requested fps regardless of screen activity. In the direct-encode path the repeat copy goes through the pixel transfer session, since retaining ReplayKit's own buffers would stall its capture pool. Repeated frames of unchanged content encode to near-empty delta frames, and the constant flow also restores the periodic 2s IDR cadence for late-joining clients. The heartbeat gains a 'repeated' counter/rate. Measured on an iPhone XS with a near-static screen: accepted ~24/s + repeated ~7/s = encoded 30.7-33.3/s, encode latency ~12ms. Co-Authored-By: Claude Fable 5 --- .../FBExtBroadcastClient.m | 2 +- .../FBExtSessionPipeline.h | 4 +- .../FBExtSessionPipeline.m | 112 +++++++++++++++++- docs/broadcast-extension.md | 3 + 4 files changed, 117 insertions(+), 4 deletions(-) diff --git a/WebDriverAgentBroadcast/FBExtBroadcastClient.m b/WebDriverAgentBroadcast/FBExtBroadcastClient.m index c15db353e..0d8fd1195 100644 --- a/WebDriverAgentBroadcast/FBExtBroadcastClient.m +++ b/WebDriverAgentBroadcast/FBExtBroadcastClient.m @@ -192,7 +192,7 @@ - (void)sendHeartbeat NSDictionary *metrics = [pipeline metricsSnapshot]; NSMutableDictionary *entry = [metrics mutableCopy]; NSDictionary *previous = self.previousPipelineTotals[sessionId.stringValue]; - for (NSString *key in @[@"samplesIn", @"accepted", @"encoded"]) { + for (NSString *key in @[@"samplesIn", @"accepted", @"encoded", @"repeated"]) { double delta = metrics[key].doubleValue - [previous[key] doubleValue]; entry[[key stringByAppendingString:@"PerSec"]] = @(FBExtRatePerSec(delta, intervalSec)); } diff --git a/WebDriverAgentBroadcast/FBExtSessionPipeline.h b/WebDriverAgentBroadcast/FBExtSessionPipeline.h index b90a3c7a0..b7751c1eb 100644 --- a/WebDriverAgentBroadcast/FBExtSessionPipeline.h +++ b/WebDriverAgentBroadcast/FBExtSessionPipeline.h @@ -61,8 +61,8 @@ NS_ASSUME_NONNULL_BEGIN A point-in-time snapshot of this pipeline's counters, safe to call from any thread: samplesIn (frames offered by ReplayKit), accepted (passed the fps gate), encoded (encoder callbacks), droppedFpsGate, droppedReplaced (latched frame superseded before processing), - droppedPool (scaled-buffer pool exhausted), encodeLatencyMsLast/encodeLatencyMsAvg - (submit-to-encoder-callback time). + droppedPool (scaled-buffer pool exhausted), repeated (last frame re-encoded to fill a + delivery gap), encodeLatencyMsLast/encodeLatencyMsAvg (submit-to-encoder-callback time). */ - (NSDictionary *)metricsSnapshot; diff --git a/WebDriverAgentBroadcast/FBExtSessionPipeline.m b/WebDriverAgentBroadcast/FBExtSessionPipeline.m index 46702650f..3b5019f58 100644 --- a/WebDriverAgentBroadcast/FBExtSessionPipeline.m +++ b/WebDriverAgentBroadcast/FBExtSessionPipeline.m @@ -24,6 +24,8 @@ @interface FBExtSessionPipeline () { CMSampleBufferRef _pendingSampleBuffer; + /** The most recently encoded (pool-owned) frame, re-encoded to fill delivery gaps (queue-confined). */ + CVPixelBufferRef _repeatBuffer; } @property (nonatomic, weak) id sink; @@ -53,9 +55,15 @@ @interface FBExtSessionPipeline () { @property (atomic) uint64_t droppedFpsGateCount; @property (atomic) uint64_t droppedReplacedCount; @property (atomic) uint64_t droppedPoolCount; +@property (atomic) uint64_t repeatedCount; @property (atomic) uint64_t lastEncodeLatencyMs; @property (atomic) double avgEncodeLatencyMs; +// Frame repeater state (queue-confined except the timer handle). +@property (nonatomic, nullable) dispatch_source_t repeatTimer; +@property (nonatomic) uint64_t frameIntervalMs; +@property (nonatomic) uint64_t lastEncodeAtMs; + - (void)replacePendingSampleBuffer:(CMSampleBufferRef)sampleBuffer atTimeMs:(uint64_t)nowMs orientation:(uint8_t)orientation; @@ -155,10 +163,90 @@ - (nullable instancetype)initWithSessionId:(uint32_t)sessionId // Open every session with an IDR: WDA only switches a stream onto the broadcast source once // a key frame (with parameter sets) has arrived. [encoder requestKeyFrame]; + + // ReplayKit delivers nothing while the screen is static and VideoToolbox has no + // Android-style repeat-previous-frame mode, so the last frame is re-encoded manually to + // keep the output cadence at the requested fps. + _frameIntervalMs = fps > 0 ? (uint64_t)(1000 / fps) : 0; + if (_frameIntervalMs > 1) { + [self startRepeatTimer]; + } } return self; } +- (void)startRepeatTimer +{ + dispatch_source_t timer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, self.queue); + if (nil == timer) { + return; + } + uint64_t intervalNs = self.frameIntervalMs * NSEC_PER_MSEC; + dispatch_source_set_timer(timer, + dispatch_time(DISPATCH_TIME_NOW, (int64_t)intervalNs), + intervalNs, + intervalNs / 4); + __weak typeof(self) weakSelf = self; + dispatch_source_set_event_handler(timer, ^{ + [weakSelf repeatLastFrameIfIdle]; + }); + dispatch_resume(timer); + self.repeatTimer = timer; +} + +- (void)repeatLastFrameIfIdle +{ + // Runs on self.queue, serialized with live encodes. + if (!self.active || NULL == _repeatBuffer || 0 == self.lastEncodeAtMs) { + return; + } + uint64_t nowMs = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) / NSEC_PER_MSEC; + if (nowMs - self.lastEncodeAtMs < self.frameIntervalMs) { + return; + } + NSError *error; + if ([self.encoder encodePixelBuffer:_repeatBuffer presentationTimeMs:nowMs error:&error]) { + self.repeatedCount += 1; + self.lastEncodeAtMs = nowMs; + } else { + FBExtLogError("Session %u: cannot re-encode the repeat frame: %{public}@", + self.sessionId, error.description); + } +} + +// Runs on self.queue. Retains the given pool-owned buffer as the repeat source. +- (void)storeRepeatBuffer:(CVPixelBufferRef)buffer +{ + if (buffer == _repeatBuffer) { + return; + } + CVPixelBufferRef previous = _repeatBuffer; + _repeatBuffer = (CVPixelBufferRef)CFRetain(buffer); + if (NULL != previous) { + CVPixelBufferRelease(previous); + } +} + +// Runs on self.queue. Copies a non-pool source (direct-encode path) into a pool buffer for the +// repeater: ReplayKit recycles its sample buffers, so retaining one would stall its capture. +- (void)refreshRepeatBufferWithCopyOf:(CVPixelBufferRef)sourceBuffer +{ + CVPixelBufferRef copyBuffer = NULL; + NSDictionary *auxAttributes = @{(id)kCVPixelBufferPoolAllocationThresholdKey: @(POOL_ALLOCATION_THRESHOLD)}; + CVReturn poolStatus = CVPixelBufferPoolCreatePixelBufferWithAuxAttributes(kCFAllocatorDefault, + self.bufferPool, + (__bridge CFDictionaryRef)auxAttributes, + ©Buffer); + if (poolStatus != kCVReturnSuccess || NULL == copyBuffer) { + // Keep the previous (slightly stale) repeat buffer instead of growing the pool. + return; + } + if (noErr == VTPixelTransferSessionTransferImage(self.transferSession, sourceBuffer, copyBuffer)) { + [self storeRepeatBuffer:copyBuffer]; + } + CVPixelBufferRelease(copyBuffer); +} + - (void)submitSampleBuffer:(CMSampleBufferRef)sampleBuffer orientation:(uint8_t)orientation { self.samplesInCount += 1; @@ -289,6 +377,8 @@ - (void)processRetainedSampleBuffer:(CMSampleBufferRef)sampleBuffer && CVPixelBufferGetHeight(sourceBuffer) == self.height) { NSError *error; if ([self.encoder encodePixelBuffer:sourceBuffer presentationTimeMs:nowMs error:&error]) { + self.lastEncodeAtMs = nowMs; + [self refreshRepeatBufferWithCopyOf:sourceBuffer]; return; } self.directSourceEncodingDisabled = YES; @@ -317,7 +407,10 @@ - (void)processRetainedSampleBuffer:(CMSampleBufferRef)sampleBuffer } NSError *error; - if (![self.encoder encodePixelBuffer:scaledBuffer presentationTimeMs:nowMs error:&error]) { + if ([self.encoder encodePixelBuffer:scaledBuffer presentationTimeMs:nowMs error:&error]) { + self.lastEncodeAtMs = nowMs; + [self storeRepeatBuffer:scaledBuffer]; + } else { FBExtLogError("Session %u: cannot encode a frame: %{public}@", self.sessionId, error.description); } CVPixelBufferRelease(scaledBuffer); @@ -337,6 +430,7 @@ - (void)requestKeyFrame @"droppedFpsGate": @(self.droppedFpsGateCount), @"droppedReplaced": @(self.droppedReplacedCount), @"droppedPool": @(self.droppedPoolCount), + @"repeated": @(self.repeatedCount), @"encodeLatencyMsLast": @(self.lastEncodeLatencyMs), @"encodeLatencyMsAvg": @(round(self.avgEncodeLatencyMs * 10) / 10), }; @@ -348,7 +442,16 @@ - (void)teardown @synchronized (self) { [self clearPendingSampleBufferLocked]; } + dispatch_source_t timer = self.repeatTimer; + if (nil != timer) { + dispatch_source_cancel(timer); + self.repeatTimer = nil; + } dispatch_async(self.queue, ^{ + if (NULL != self->_repeatBuffer) { + CVPixelBufferRelease(self->_repeatBuffer); + self->_repeatBuffer = NULL; + } if (nil != self.encoder) { self.encoder.delegate = nil; [self.encoder stop]; @@ -376,6 +479,13 @@ - (void)dealloc @synchronized (self) { [self clearPendingSampleBufferLocked]; } + if (nil != _repeatTimer) { + dispatch_source_cancel(_repeatTimer); + } + if (NULL != _repeatBuffer) { + CVPixelBufferRelease(_repeatBuffer); + _repeatBuffer = NULL; + } if (nil != _encoder) { _encoder.delegate = nil; [_encoder stop]; diff --git a/docs/broadcast-extension.md b/docs/broadcast-extension.md index b8da67032..725eabf9b 100644 --- a/docs/broadcast-extension.md +++ b/docs/broadcast-extension.md @@ -50,6 +50,9 @@ Notes: - Frames are encoded in the native ReplayKit orientation (not rotated upright like the screenshot path). The current `orientation` (CGImagePropertyOrientation 1-8) is exposed via the status endpoint and heartbeat. +- ReplayKit only delivers frames while the screen changes; the extension re-encodes the last + frame to fill delivery gaps, so the stream holds the session's requested fps on static + screens too (`repeated` counter in the heartbeat). ## Environment variables From 3f8c9e41534f67659eaa5e2fe125c738ff2afb87 Mon Sep 17 00:00:00 2001 From: timo <44401485+Timo972@users.noreply.github.com> Date: Thu, 11 Jun 2026 09:00:38 +0200 Subject: [PATCH 11/11] fix(screencapture): regenerate appex entitlements when the embed script rewrites the bundle id When a device build overrides PRODUCT_BUNDLE_IDENTIFIER, the embed script rewrites the appex CFBundleIdentifier to .broadcast but re-signed with --preserve-metadata=entitlements, keeping an application-identifier minted for the pre-rewrite id. installd rejects an extension whose signed identity does not match its bundle id, so exactly the downstream-override case the rewrite exists for produced an uninstallable appex. Extract the entitlements, point application-identifier at . and re-sign with them; warn that the embedded provisioning profile must cover the new id. The no-rewrite path (and simulator ad-hoc signing without an application-identifier) keeps the previous behavior. Verified with a synthetic rewrite harness (host id changed, ad-hoc identity): the re-signed appex carries the corrected application-identifier and a valid signature; a real device build (no rewrite) still produces a valid deep signature. Co-Authored-By: Claude Fable 5 --- Scripts/embed-broadcast-extension.sh | 34 ++++++++++++++++++++++++++-- docs/broadcast-extension.md | 5 +++- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/Scripts/embed-broadcast-extension.sh b/Scripts/embed-broadcast-extension.sh index e2845eefe..a2559e471 100755 --- a/Scripts/embed-broadcast-extension.sh +++ b/Scripts/embed-broadcast-extension.sh @@ -45,8 +45,10 @@ cp -R "$APPEX_SRC" "$APPEX_DST" HOST_ID=$(/usr/libexec/PlistBuddy -c "Print :CFBundleIdentifier" "$RUNNER_APP/Info.plist") WANT_ID="${HOST_ID}.broadcast" CURRENT_ID=$(/usr/libexec/PlistBuddy -c "Print :CFBundleIdentifier" "$APPEX_DST/Info.plist") +ID_REWRITTEN=0 if [ "$CURRENT_ID" != "$WANT_ID" ]; then /usr/libexec/PlistBuddy -c "Set :CFBundleIdentifier $WANT_ID" "$APPEX_DST/Info.plist" + ID_REWRITTEN=1 fi # Re-codesign since we modified the bundle after Xcode signed it: the appex first (its bundle @@ -69,8 +71,36 @@ if [ -d "$RUNNER_APP/_CodeSignature" ]; then EXISTING_IDENT="-" fi if [ -n "$EXISTING_IDENT" ]; then - codesign --force --sign "$EXISTING_IDENT" \ - --preserve-metadata=entitlements "$APPEX_DST" + if [ "$ID_REWRITTEN" = "1" ]; then + # The bundle id changed, so the signed entitlements' application-identifier must be + # regenerated to match it - preserving the stale entitlements makes installd reject + # the extension (identity no longer matches the bundle id). Rewrite the identifier in + # the extracted entitlements and re-sign with them. + ENTITLEMENTS_PLIST=$(mktemp -t broadcast-entitlements).plist + if codesign -d --entitlements - --xml "$APPEX_DST" > "$ENTITLEMENTS_PLIST" 2>/dev/null \ + && [ -s "$ENTITLEMENTS_PLIST" ]; then + OLD_APP_ID=$(/usr/libexec/PlistBuddy -c "Print :application-identifier" "$ENTITLEMENTS_PLIST" 2>/dev/null || true) + TEAM_ID="${OLD_APP_ID%%.*}" + if [ -n "$TEAM_ID" ] && [ "$TEAM_ID" != "$OLD_APP_ID" ]; then + /usr/libexec/PlistBuddy -c "Set :application-identifier ${TEAM_ID}.${WANT_ID}" "$ENTITLEMENTS_PLIST" + codesign --force --sign "$EXISTING_IDENT" \ + --entitlements "$ENTITLEMENTS_PLIST" "$APPEX_DST" + else + # No application-identifier (e.g. simulator ad-hoc signing): nothing to fix up. + codesign --force --sign "$EXISTING_IDENT" \ + --preserve-metadata=entitlements "$APPEX_DST" + fi + else + codesign --force --sign "$EXISTING_IDENT" "$APPEX_DST" + fi + rm -f "$ENTITLEMENTS_PLIST" + echo "warning: appex bundle id was rewritten to $WANT_ID; the embedded provisioning" \ + "profile must cover that id (a wildcard development profile works), otherwise" \ + "re-sign with a matching profile - see docs/broadcast-extension.md" + else + codesign --force --sign "$EXISTING_IDENT" \ + --preserve-metadata=entitlements "$APPEX_DST" + fi codesign --force --sign "$EXISTING_IDENT" \ --preserve-metadata=identifier,entitlements "$RUNNER_APP" else diff --git a/docs/broadcast-extension.md b/docs/broadcast-extension.md index 725eabf9b..b2686fc66 100644 --- a/docs/broadcast-extension.md +++ b/docs/broadcast-extension.md @@ -89,7 +89,10 @@ Pipelines that re-sign `WebDriverAgentRunner-Runner.app` must keep the nested ap 2. The appex `CFBundleIdentifier` must remain `.broadcast` — installd rejects extensions whose bundle id is not prefixed by the host app's, or whose team differs. If the pipeline changes the runner's bundle id, patch the appex id accordingly (the embed - script shows how) and set `BROADCAST_EXT_BUNDLE_ID` if the suffix differs. + script shows how) and set `BROADCAST_EXT_BUNDLE_ID` if the suffix differs. The signed + entitlements' `application-identifier` must match the (new) bundle id — when the embed + script rewrites the id it regenerates that entitlement automatically; custom re-signers + must do the same. 3. Provisioning: the embedded profile must cover the appex bundle id; a wildcard development profile (`TEAM.*`) is the low-friction option. The extension needs no special entitlements (no app groups — IPC is loopback TCP).