diff --git a/Scripts/embed-broadcast-extension.sh b/Scripts/embed-broadcast-extension.sh new file mode 100755 index 000000000..a2559e471 --- /dev/null +++ b/Scripts/embed-broadcast-extension.sh @@ -0,0 +1,111 @@ +#!/bin/bash +# Embed the WebDriverAgentBroadcast ReplayKit upload extension into the +# wrapping XCTRunner host app. +# +# Apple's USES_XCTRUNNER auto-generates a Runner.app around UI-testing +# .xctest bundles after all target build phases have run, so an appex +# cannot reach Runner.app/PlugIns through a regular embed build phase. +# The extension target is built into BUILT_PRODUCTS_DIR via a target +# dependency of WebDriverAgentRunner; this scheme post-action copies it +# into Runner.app/PlugIns, fixes its bundle id to match the host app +# (extensions must be prefixed by the host's CFBundleIdentifier, which +# Xcode suffixes with '.xctrunner') and re-signs inner-first. +# +# Limitations: +# - Touches XCTRunner internals; may need updates across Xcode versions. +# - iOS only; the extension is not built for tvOS. +# - Cloud device farms that re-sign WDA must re-sign the nested appex +# with the same team first (or use codesign --deep); see +# docs/broadcast-extension.md. + +set -euo pipefail + +RUNNER_APP="${BUILT_PRODUCTS_DIR}/${PRODUCT_NAME}-Runner.app" +APPEX_NAME="WebDriverAgentBroadcast.appex" +APPEX_SRC="${BUILT_PRODUCTS_DIR}/${APPEX_NAME}" + +if [ ! -d "$RUNNER_APP" ]; then + echo "warning: ${PRODUCT_NAME}-Runner.app not found at $RUNNER_APP; skipping broadcast extension embed" + exit 0 +fi + +if [ ! -d "$APPEX_SRC" ]; then + echo "warning: $APPEX_NAME not found at $APPEX_SRC; skipping broadcast extension embed" + exit 0 +fi + +APPEX_DST="$RUNNER_APP/PlugIns/$APPEX_NAME" +rm -rf "$APPEX_DST" +mkdir -p "$RUNNER_APP/PlugIns" +cp -R "$APPEX_SRC" "$APPEX_DST" + +# Extensions must carry a bundle id prefixed by the host app's. The host id is only final at +# this point (Xcode appends '.xctrunner'; downstream tooling may override the prefix), so +# always derive the appex id from the embedded Runner.app instead of trusting build settings. +HOST_ID=$(/usr/libexec/PlistBuddy -c "Print :CFBundleIdentifier" "$RUNNER_APP/Info.plist") +WANT_ID="${HOST_ID}.broadcast" +CURRENT_ID=$(/usr/libexec/PlistBuddy -c "Print :CFBundleIdentifier" "$APPEX_DST/Info.plist") +ID_REWRITTEN=0 +if [ "$CURRENT_ID" != "$WANT_ID" ]; then + /usr/libexec/PlistBuddy -c "Set :CFBundleIdentifier $WANT_ID" "$APPEX_DST/Info.plist" + ID_REWRITTEN=1 +fi + +# Re-codesign since we modified the bundle after Xcode signed it: the appex first (its bundle +# id may just have changed, so do NOT preserve the identifier), then the app so its seal covers +# the new nested code. In a scheme post-action context Xcode's CODE_SIGN_* env vars are not +# exposed, so discover the existing signing identity from the already-signed bundle. +if [ -d "$RUNNER_APP/_CodeSignature" ]; then + # Capture the signature info once. Piping codesign straight into + # `awk ... exit` makes awk close the pipe early, killing codesign with + # SIGPIPE -- which `set -o pipefail` turns into a fatal error. That trips + # only when an Authority line exists, i.e. on every real-device build. + SIGN_INFO=$(codesign -dvv "$RUNNER_APP" 2>&1 || true) + EXISTING_IDENT="${EXPANDED_CODE_SIGN_IDENTITY:-}" + if [ -z "$EXISTING_IDENT" ]; then + EXISTING_IDENT=$(awk -F'=' '/^Authority/ {print $2; exit}' <<< "$SIGN_INFO") + fi + # Simulator builds are ad-hoc signed: there is no Authority line, but the + # bundle can still be re-signed ad-hoc with an identity of "-". + if [ -z "$EXISTING_IDENT" ] && grep -q '^Signature=adhoc' <<< "$SIGN_INFO"; then + EXISTING_IDENT="-" + fi + if [ -n "$EXISTING_IDENT" ]; then + if [ "$ID_REWRITTEN" = "1" ]; then + # The bundle id changed, so the signed entitlements' application-identifier must be + # regenerated to match it - preserving the stale entitlements makes installd reject + # the extension (identity no longer matches the bundle id). Rewrite the identifier in + # the extracted entitlements and re-sign with them. + ENTITLEMENTS_PLIST=$(mktemp -t broadcast-entitlements).plist + if codesign -d --entitlements - --xml "$APPEX_DST" > "$ENTITLEMENTS_PLIST" 2>/dev/null \ + && [ -s "$ENTITLEMENTS_PLIST" ]; then + OLD_APP_ID=$(/usr/libexec/PlistBuddy -c "Print :application-identifier" "$ENTITLEMENTS_PLIST" 2>/dev/null || true) + TEAM_ID="${OLD_APP_ID%%.*}" + if [ -n "$TEAM_ID" ] && [ "$TEAM_ID" != "$OLD_APP_ID" ]; then + /usr/libexec/PlistBuddy -c "Set :application-identifier ${TEAM_ID}.${WANT_ID}" "$ENTITLEMENTS_PLIST" + codesign --force --sign "$EXISTING_IDENT" \ + --entitlements "$ENTITLEMENTS_PLIST" "$APPEX_DST" + else + # No application-identifier (e.g. simulator ad-hoc signing): nothing to fix up. + codesign --force --sign "$EXISTING_IDENT" \ + --preserve-metadata=entitlements "$APPEX_DST" + fi + else + codesign --force --sign "$EXISTING_IDENT" "$APPEX_DST" + fi + rm -f "$ENTITLEMENTS_PLIST" + echo "warning: appex bundle id was rewritten to $WANT_ID; the embedded provisioning" \ + "profile must cover that id (a wildcard development profile works), otherwise" \ + "re-sign with a matching profile - see docs/broadcast-extension.md" + else + codesign --force --sign "$EXISTING_IDENT" \ + --preserve-metadata=entitlements "$APPEX_DST" + fi + codesign --force --sign "$EXISTING_IDENT" \ + --preserve-metadata=identifier,entitlements "$RUNNER_APP" + else + echo "warning: bundle is signed but no identity discovered; signature will be invalid" + fi +fi + +echo "embedded $APPEX_NAME into $RUNNER_APP (bundle id $WANT_ID)" diff --git a/WebDriverAgent.xcodeproj/project.pbxproj b/WebDriverAgent.xcodeproj/project.pbxproj index f0388b4db..76234df98 100644 --- a/WebDriverAgent.xcodeproj/project.pbxproj +++ b/WebDriverAgent.xcodeproj/project.pbxproj @@ -39,6 +39,28 @@ FBCAFE00000000000000001A /* FBPixelBufferConverterTests.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000000019 /* FBPixelBufferConverterTests.m */; }; FBCAFE00000000000000001C /* FBVideoEncoderTests.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE00000000000000001B /* FBVideoEncoderTests.m */; }; FBCAFE000000000000004002 /* FBVideoStreamSessionTests.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000004001 /* FBVideoStreamSessionTests.m */; }; + FBCAFE000000000000005103 /* FBBroadcastSampleHandler.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005102 /* FBBroadcastSampleHandler.m */; }; + FBCAFE000000000000005106 /* FBExtBroadcastClient.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005105 /* FBExtBroadcastClient.m */; }; + FBCAFE000000000000005109 /* FBExtSessionPipeline.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005108 /* FBExtSessionPipeline.m */; }; + FBCAFE000000000000005201 /* FBVideoEncoder.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000000010 /* FBVideoEncoder.m */; }; + FBCAFE000000000000005202 /* GCDAsyncSocket.m in Sources */ = {isa = PBXBuildFile; fileRef = 718226C82587443600661B83 /* GCDAsyncSocket.m */; }; + FBCAFE000000000000005203 /* FBBroadcastProtocol.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005301 /* FBBroadcastProtocol.m */; }; + FBCAFE000000000000005302 /* FBBroadcastProtocol.h in Headers */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005300 /* FBBroadcastProtocol.h */; }; + FBCAFE000000000000005303 /* FBBroadcastProtocol.h in Headers */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005300 /* FBBroadcastProtocol.h */; }; + FBCAFE000000000000005304 /* FBBroadcastProtocol.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005301 /* FBBroadcastProtocol.m */; }; + FBCAFE000000000000005305 /* FBBroadcastProtocol.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005301 /* FBBroadcastProtocol.m */; }; + FBCAFE000000000000005312 /* FBBroadcastControlServer.h in Headers */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005310 /* FBBroadcastControlServer.h */; }; + FBCAFE000000000000005313 /* FBBroadcastControlServer.h in Headers */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005310 /* FBBroadcastControlServer.h */; }; + FBCAFE000000000000005314 /* FBBroadcastControlServer.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005311 /* FBBroadcastControlServer.m */; }; + FBCAFE000000000000005315 /* FBBroadcastControlServer.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005311 /* FBBroadcastControlServer.m */; }; + FBCAFE000000000000005322 /* FBBroadcastManager.h in Headers */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005320 /* FBBroadcastManager.h */; }; + FBCAFE000000000000005323 /* FBBroadcastManager.h in Headers */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005320 /* FBBroadcastManager.h */; }; + FBCAFE000000000000005324 /* FBBroadcastManager.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005321 /* FBBroadcastManager.m */; }; + FBCAFE000000000000005325 /* FBBroadcastManager.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005321 /* FBBroadcastManager.m */; }; + FBCAFE000000000000005332 /* FBBroadcastPickerHost.h in Headers */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005330 /* FBBroadcastPickerHost.h */; }; + FBCAFE000000000000005333 /* FBBroadcastPickerHost.h in Headers */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005330 /* FBBroadcastPickerHost.h */; }; + FBCAFE000000000000005334 /* FBBroadcastPickerHost.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005331 /* FBBroadcastPickerHost.m */; }; + FBCAFE000000000000005335 /* FBBroadcastPickerHost.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005331 /* FBBroadcastPickerHost.m */; }; 0E0413382DF1E15100AF007C /* XCUIElement+FBMinMax.m in Sources */ = {isa = PBXBuildFile; fileRef = 0E0413372DF1E15100AF007C /* XCUIElement+FBMinMax.m */; }; 0E0413392DF1E15100AF007C /* XCUIElement+FBMinMax.m in Sources */ = {isa = PBXBuildFile; fileRef = 0E0413372DF1E15100AF007C /* XCUIElement+FBMinMax.m */; }; 0E04133B2DF1E15900AF007C /* XCUIElement+FBMinMax.h in Headers */ = {isa = PBXBuildFile; fileRef = 0E04133A2DF1E15900AF007C /* XCUIElement+FBMinMax.h */; }; @@ -944,6 +966,13 @@ remoteGlobalIDString = EE158A981CBD452B00A3E3F0; remoteInfo = WebDriverAgentLib; }; + FBCAFE000000000000005009 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 91F9DAE11B99DBC2001349B2 /* Project object */; + proxyType = 1; + remoteGlobalIDString = FBCAFE000000000000005001; + remoteInfo = WebDriverAgentBroadcast; + }; /* End PBXContainerItemProxy section */ /* Begin PBXCopyFilesBuildPhase section */ @@ -990,6 +1019,23 @@ FBCAFE000000000000000019 /* FBPixelBufferConverterTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBPixelBufferConverterTests.m; sourceTree = ""; }; FBCAFE00000000000000001B /* FBVideoEncoderTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBVideoEncoderTests.m; sourceTree = ""; }; FBCAFE000000000000004001 /* FBVideoStreamSessionTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBVideoStreamSessionTests.m; sourceTree = ""; }; + FBCAFE000000000000005002 /* WebDriverAgentBroadcast.appex */ = {isa = PBXFileReference; explicitFileType = "wrapper.app-extension"; includeInIndex = 0; path = WebDriverAgentBroadcast.appex; sourceTree = BUILT_PRODUCTS_DIR; }; + FBCAFE00000000000000500C /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + FBCAFE000000000000005101 /* FBBroadcastSampleHandler.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FBBroadcastSampleHandler.h; sourceTree = ""; }; + FBCAFE000000000000005102 /* FBBroadcastSampleHandler.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBBroadcastSampleHandler.m; sourceTree = ""; }; + FBCAFE000000000000005104 /* FBExtBroadcastClient.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FBExtBroadcastClient.h; sourceTree = ""; }; + FBCAFE000000000000005105 /* FBExtBroadcastClient.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBExtBroadcastClient.m; sourceTree = ""; }; + FBCAFE000000000000005107 /* FBExtSessionPipeline.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FBExtSessionPipeline.h; sourceTree = ""; }; + FBCAFE000000000000005108 /* FBExtSessionPipeline.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBExtSessionPipeline.m; sourceTree = ""; }; + FBCAFE00000000000000510A /* FBExtLogging.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FBExtLogging.h; sourceTree = ""; }; + FBCAFE000000000000005300 /* FBBroadcastProtocol.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FBBroadcastProtocol.h; sourceTree = ""; }; + FBCAFE000000000000005301 /* FBBroadcastProtocol.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBBroadcastProtocol.m; sourceTree = ""; }; + FBCAFE000000000000005310 /* FBBroadcastControlServer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FBBroadcastControlServer.h; sourceTree = ""; }; + FBCAFE000000000000005311 /* FBBroadcastControlServer.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBBroadcastControlServer.m; sourceTree = ""; }; + FBCAFE000000000000005320 /* FBBroadcastManager.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FBBroadcastManager.h; sourceTree = ""; }; + FBCAFE000000000000005321 /* FBBroadcastManager.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBBroadcastManager.m; sourceTree = ""; }; + FBCAFE000000000000005330 /* FBBroadcastPickerHost.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FBBroadcastPickerHost.h; sourceTree = ""; }; + FBCAFE000000000000005331 /* FBBroadcastPickerHost.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBBroadcastPickerHost.m; sourceTree = ""; }; 0E0413372DF1E15100AF007C /* XCUIElement+FBMinMax.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = "XCUIElement+FBMinMax.m"; sourceTree = ""; }; 0E04133A2DF1E15900AF007C /* XCUIElement+FBMinMax.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "XCUIElement+FBMinMax.h"; sourceTree = ""; }; 1357E295233D05240054BDB8 /* XCUIHitPointResult.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = XCUIHitPointResult.h; sourceTree = ""; }; @@ -1573,6 +1619,13 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + FBCAFE000000000000005007 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ @@ -1680,6 +1733,7 @@ EEC288F81BF0ED2500B4DC79 /* WebDriverAgentLib */, EE9B75F91CF7964100275851 /* WebDriverAgentTests */, EEF988341C486655005CA669 /* WebDriverAgentRunner */, + FBCAFE00000000000000500B /* WebDriverAgentBroadcast */, EE9AB8021CAEE182008C271F /* Scripts */, 91F9DAEA1B99DBC2001349B2 /* Products */, B6E83A410C45944B036B6B0F /* Frameworks */, @@ -1703,6 +1757,7 @@ 641EE2DA2240BBE300173FCB /* WebDriverAgentRunner_tvOS.xctest */, 641EE6F82240C5CA00173FCB /* WebDriverAgentLib_tvOS.framework */, 64B264F9228C50E0002A5025 /* UnitTests_tvOS.xctest */, + FBCAFE000000000000005002 /* WebDriverAgentBroadcast.appex */, ); name = Products; sourceTree = ""; @@ -2096,6 +2151,14 @@ FBCAFE000000000000000010 /* FBVideoEncoder.m */, FBCAFE000000000000000013 /* FBVideoStreamManager.h */, FBCAFE000000000000000016 /* FBVideoStreamManager.m */, + FBCAFE000000000000005300 /* FBBroadcastProtocol.h */, + FBCAFE000000000000005301 /* FBBroadcastProtocol.m */, + FBCAFE000000000000005310 /* FBBroadcastControlServer.h */, + FBCAFE000000000000005311 /* FBBroadcastControlServer.m */, + FBCAFE000000000000005320 /* FBBroadcastManager.h */, + FBCAFE000000000000005321 /* FBBroadcastManager.m */, + FBCAFE000000000000005330 /* FBBroadcastPickerHost.h */, + FBCAFE000000000000005331 /* FBBroadcastPickerHost.m */, FBCAFE000000000000001001 /* FBVideoStreamSession.h */, FBCAFE000000000000001004 /* FBVideoStreamSession.m */, ); @@ -2398,6 +2461,21 @@ path = XCTUITestRunner; sourceTree = SOURCE_ROOT; }; + FBCAFE00000000000000500B /* WebDriverAgentBroadcast */ = { + isa = PBXGroup; + children = ( + FBCAFE00000000000000500C /* Info.plist */, + FBCAFE000000000000005101 /* FBBroadcastSampleHandler.h */, + FBCAFE000000000000005102 /* FBBroadcastSampleHandler.m */, + FBCAFE000000000000005104 /* FBExtBroadcastClient.h */, + FBCAFE000000000000005105 /* FBExtBroadcastClient.m */, + FBCAFE000000000000005107 /* FBExtSessionPipeline.h */, + FBCAFE000000000000005108 /* FBExtSessionPipeline.m */, + FBCAFE00000000000000510A /* FBExtLogging.h */, + ); + path = WebDriverAgentBroadcast; + sourceTree = ""; + }; /* End PBXGroup section */ /* Begin PBXHeadersBuildPhase section */ @@ -2648,6 +2726,10 @@ FBCAFE000000000000000009 /* FBPixelBufferConverter.h in Headers */, FBCAFE00000000000000000F /* FBVideoEncoder.h in Headers */, FBCAFE000000000000000015 /* FBVideoStreamManager.h in Headers */, + FBCAFE000000000000005302 /* FBBroadcastProtocol.h in Headers */, + FBCAFE000000000000005312 /* FBBroadcastControlServer.h in Headers */, + FBCAFE000000000000005322 /* FBBroadcastManager.h in Headers */, + FBCAFE000000000000005332 /* FBBroadcastPickerHost.h in Headers */, FBCAFE000000000000001003 /* FBVideoStreamSession.h in Headers */, ); runOnlyForDeploymentPostprocessing = 0; @@ -2902,6 +2984,10 @@ FBCAFE000000000000000008 /* FBPixelBufferConverter.h in Headers */, FBCAFE00000000000000000E /* FBVideoEncoder.h in Headers */, FBCAFE000000000000000014 /* FBVideoStreamManager.h in Headers */, + FBCAFE000000000000005303 /* FBBroadcastProtocol.h in Headers */, + FBCAFE000000000000005313 /* FBBroadcastControlServer.h in Headers */, + FBCAFE000000000000005323 /* FBBroadcastManager.h in Headers */, + FBCAFE000000000000005333 /* FBBroadcastPickerHost.h in Headers */, FBCAFE000000000000001002 /* FBVideoStreamSession.h in Headers */, ); runOnlyForDeploymentPostprocessing = 0; @@ -3087,12 +3173,30 @@ ); dependencies = ( EE158B5C1CBD462500A3E3F0 /* PBXTargetDependency */, + FBCAFE00000000000000500A /* PBXTargetDependency */, ); name = WebDriverAgentRunner; productName = XCTUITestRunner; productReference = EEF9882A1C486603005CA669 /* WebDriverAgentRunner.xctest */; productType = "com.apple.product-type.bundle.ui-testing"; }; + FBCAFE000000000000005001 /* WebDriverAgentBroadcast */ = { + isa = PBXNativeTarget; + buildConfigurationList = FBCAFE000000000000005003 /* Build configuration list for PBXNativeTarget "WebDriverAgentBroadcast" */; + buildPhases = ( + FBCAFE000000000000005006 /* Sources */, + FBCAFE000000000000005007 /* Frameworks */, + FBCAFE000000000000005008 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = WebDriverAgentBroadcast; + productName = WebDriverAgentBroadcast; + productReference = FBCAFE000000000000005002 /* WebDriverAgentBroadcast.appex */; + productType = "com.apple.product-type.app-extension"; + }; /* End PBXNativeTarget section */ /* Begin PBXProject section */ @@ -3129,6 +3233,9 @@ EEF988291C486603005CA669 = { CreatedOnToolsVersion = 7.2; }; + FBCAFE000000000000005001 = { + CreatedOnToolsVersion = 15.0; + }; }; }; buildConfigurationList = 91F9DAE41B99DBC2001349B2 /* Build configuration list for PBXProject "WebDriverAgent" */; @@ -3147,6 +3254,7 @@ EE158A981CBD452B00A3E3F0 /* WebDriverAgentLib */, 641EE5D52240C5CA00173FCB /* WebDriverAgentLib_tvOS */, EEF988291C486603005CA669 /* WebDriverAgentRunner */, + FBCAFE000000000000005001 /* WebDriverAgentBroadcast */, 641EE2D92240BBE300173FCB /* WebDriverAgentRunner_tvOS */, EE836C011C0F118600D87246 /* UnitTests */, 64B264F8228C50E0002A5025 /* UnitTests_tvOS */, @@ -3231,6 +3339,13 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + FBCAFE000000000000005008 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXResourcesBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ @@ -3377,6 +3492,10 @@ FBCAFE00000000000000000C /* FBPixelBufferConverter.m in Sources */, FBCAFE000000000000000012 /* FBVideoEncoder.m in Sources */, FBCAFE000000000000000018 /* FBVideoStreamManager.m in Sources */, + FBCAFE000000000000005304 /* FBBroadcastProtocol.m in Sources */, + FBCAFE000000000000005314 /* FBBroadcastControlServer.m in Sources */, + FBCAFE000000000000005324 /* FBBroadcastManager.m in Sources */, + FBCAFE000000000000005334 /* FBBroadcastPickerHost.m in Sources */, FBCAFE000000000000001006 /* FBVideoStreamSession.m in Sources */, ); runOnlyForDeploymentPostprocessing = 0; @@ -3526,6 +3645,10 @@ FBCAFE00000000000000000B /* FBPixelBufferConverter.m in Sources */, FBCAFE000000000000000011 /* FBVideoEncoder.m in Sources */, FBCAFE000000000000000017 /* FBVideoStreamManager.m in Sources */, + FBCAFE000000000000005305 /* FBBroadcastProtocol.m in Sources */, + FBCAFE000000000000005315 /* FBBroadcastControlServer.m in Sources */, + FBCAFE000000000000005325 /* FBBroadcastManager.m in Sources */, + FBCAFE000000000000005335 /* FBBroadcastPickerHost.m in Sources */, FBCAFE000000000000001005 /* FBVideoStreamSession.m in Sources */, ); runOnlyForDeploymentPostprocessing = 0; @@ -3655,6 +3778,19 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + FBCAFE000000000000005006 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + FBCAFE000000000000005103 /* FBBroadcastSampleHandler.m in Sources */, + FBCAFE000000000000005106 /* FBExtBroadcastClient.m in Sources */, + FBCAFE000000000000005109 /* FBExtSessionPipeline.m in Sources */, + FBCAFE000000000000005201 /* FBVideoEncoder.m in Sources */, + FBCAFE000000000000005202 /* GCDAsyncSocket.m in Sources */, + FBCAFE000000000000005203 /* FBBroadcastProtocol.m in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXSourcesBuildPhase section */ /* Begin PBXTargetDependency section */ @@ -3708,6 +3844,11 @@ target = EE158A981CBD452B00A3E3F0 /* WebDriverAgentLib */; targetProxy = EE9B769E1CF79C0A00275851 /* PBXContainerItemProxy */; }; + FBCAFE00000000000000500A /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = FBCAFE000000000000005001 /* WebDriverAgentBroadcast */; + targetProxy = FBCAFE000000000000005009 /* PBXContainerItemProxy */; + }; /* End PBXTargetDependency section */ /* Begin PBXVariantGroup section */ @@ -4594,6 +4735,38 @@ }; name = Release; }; + FBCAFE000000000000005004 /* Debug */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = EEE5CABF1C80361500CBBDD9 /* IOSSettings.xcconfig */; + buildSettings = { + APPLICATION_EXTENSION_API_ONLY = YES; + DEBUG_INFORMATION_FORMAT = dwarf; + GCC_TREAT_WARNINGS_AS_ERRORS = NO; + INFOPLIST_FILE = WebDriverAgentBroadcast/Info.plist; + PRODUCT_BUNDLE_IDENTIFIER = com.facebook.WebDriverAgentRunner.xctrunner.broadcast; + PRODUCT_NAME = "$(TARGET_NAME)"; + SKIP_INSTALL = YES; + SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Debug; + }; + FBCAFE000000000000005005 /* Release */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = EEE5CABF1C80361500CBBDD9 /* IOSSettings.xcconfig */; + buildSettings = { + APPLICATION_EXTENSION_API_ONLY = YES; + GCC_TREAT_WARNINGS_AS_ERRORS = NO; + INFOPLIST_FILE = WebDriverAgentBroadcast/Info.plist; + ONLY_ACTIVE_ARCH = YES; + PRODUCT_BUNDLE_IDENTIFIER = com.facebook.WebDriverAgentRunner.xctrunner.broadcast; + PRODUCT_NAME = "$(TARGET_NAME)"; + SKIP_INSTALL = YES; + SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Release; + }; /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ @@ -4696,6 +4869,15 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; + FBCAFE000000000000005003 /* Build configuration list for PBXNativeTarget "WebDriverAgentBroadcast" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + FBCAFE000000000000005004 /* Debug */, + FBCAFE000000000000005005 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; /* End XCConfigurationList section */ }; rootObject = 91F9DAE11B99DBC2001349B2 /* Project object */; diff --git a/WebDriverAgent.xcodeproj/xcshareddata/xcschemes/WebDriverAgentRunner-nodebug.xcscheme b/WebDriverAgent.xcodeproj/xcshareddata/xcschemes/WebDriverAgentRunner-nodebug.xcscheme index 5ff961179..c6878787c 100644 --- a/WebDriverAgent.xcodeproj/xcshareddata/xcschemes/WebDriverAgentRunner-nodebug.xcscheme +++ b/WebDriverAgent.xcodeproj/xcshareddata/xcschemes/WebDriverAgentRunner-nodebug.xcscheme @@ -5,6 +5,24 @@ + + + + + + + + + + + version = "1.7"> @@ -22,6 +22,22 @@ + + + + + + + + + +NS_ASSUME_NONNULL_BEGIN + +/** + The broadcast upload extension's principal class: receives the system screen frames from + ReplayKit and forwards them to the per-session encode pipelines managed by + FBExtBroadcastClient. Audio samples are ignored (audio capture is a planned follow-up). + */ +@interface FBBroadcastSampleHandler : RPBroadcastSampleHandler + +@end + +NS_ASSUME_NONNULL_END diff --git a/WebDriverAgentBroadcast/FBBroadcastSampleHandler.m b/WebDriverAgentBroadcast/FBBroadcastSampleHandler.m new file mode 100644 index 000000000..604f5f956 --- /dev/null +++ b/WebDriverAgentBroadcast/FBBroadcastSampleHandler.m @@ -0,0 +1,138 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import "FBBroadcastSampleHandler.h" + +#import +#import + +#import "FBBroadcastProtocol.h" +#import "FBExtBroadcastClient.h" +#import "FBExtLogging.h" +#import "FBExtSessionPipeline.h" + +static NSString *const FBBroadcastSampleHandlerErrorDomain = @"com.facebook.WebDriverAgent.FBBroadcastSampleHandler"; + +@interface FBBroadcastSampleHandler () + +@property (nonatomic, nullable) FBExtBroadcastClient *client; + +@end + +@implementation FBBroadcastSampleHandler + +- (void)broadcastStartedWithSetupInfo:(nullable NSDictionary *)setupInfo +{ + // setupInfo is nil for broadcasts started from the system picker; nothing to read from it. + FBExtLogInfo("Broadcast started"); + self.client = [[FBExtBroadcastClient alloc] init]; + self.client.delegate = self; + [self.client start]; +} + +- (void)broadcastPaused +{ + FBExtLogInfo("Broadcast paused"); + self.client.paused = YES; + [self sendStatusEvent:@"paused" reason:nil]; +} + +- (void)broadcastResumed +{ + FBExtLogInfo("Broadcast resumed"); + self.client.paused = NO; + [self sendStatusEvent:@"resumed" reason:nil]; +} + +- (void)broadcastFinished +{ + FBExtLogInfo("Broadcast finished"); + [self sendStatusEvent:@"finishing" reason:@"broadcastFinished"]; + [self.client shutdown]; + self.client = nil; +} + +- (void)processSampleBuffer:(CMSampleBufferRef)sampleBuffer + withType:(RPSampleBufferType)sampleBufferType +{ + if (sampleBufferType != RPSampleBufferTypeVideo) { + // Audio capture is a planned follow-up. + return; + } + FBExtBroadcastClient *client = self.client; + if (nil == client) { + return; + } + + client.framesReceived += 1; + + CFTypeRef orientationAttachment = CMGetAttachment(sampleBuffer, + (__bridge CFStringRef)RPVideoSampleOrientationKey, + NULL); + if (NULL != orientationAttachment) { + client.currentOrientation = (uint8_t)[(__bridge NSNumber *)orientationAttachment unsignedIntValue]; + } + + CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + if (NULL != pixelBuffer && client.screenWidth == 0) { + client.screenWidth = CVPixelBufferGetWidth(pixelBuffer); + client.screenHeight = CVPixelBufferGetHeight(pixelBuffer); + } + + NSDictionary *pipelines = client.activePipelines; + for (FBExtSessionPipeline *pipeline in pipelines.allValues) { + [pipeline submitSampleBuffer:sampleBuffer orientation:client.currentOrientation]; + } +} + +- (void)sendStatusEvent:(NSString *)event reason:(nullable NSString *)reason +{ + NSMutableDictionary *payload = [NSMutableDictionary dictionaryWithObject:event + forKey:FBBroadcastKeyEvent]; + if (nil != reason) { + payload[FBBroadcastKeyReason] = reason; + } + NSData *message = FBBroadcastEncodeJSONMessage(FBBroadcastMessageTypeStatus, 0, payload); + if (nil != message) { + [self.client sendProtocolMessage:message isDroppable:NO]; + } +} + +- (void)finishBroadcast +{ + // The public API only offers finishBroadcastWithError:, which makes the system show an error + // alert. The private graceful variant avoids that; fall back to the public one when absent. + SEL gracefulSelector = NSSelectorFromString(@"finishBroadcastGracefully:"); + if ([self respondsToSelector:gracefulSelector]) { +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Warc-performSelector-leaks" + [self performSelector:gracefulSelector withObject:nil]; +#pragma clang diagnostic pop + return; + } + NSError *error = [NSError errorWithDomain:FBBroadcastSampleHandlerErrorDomain + code:1 + userInfo:@{NSLocalizedDescriptionKey: @"WebDriverAgent stopped the broadcast"}]; + [self finishBroadcastWithError:error]; +} + +#pragma mark - + +- (void)broadcastClientDidRequestStop:(FBExtBroadcastClient *)client +{ + [self sendStatusEvent:@"finishing" reason:@"stopRequested"]; + [self finishBroadcast]; +} + +- (void)broadcastClient:(FBExtBroadcastClient *)client didFailPermanently:(NSError *)error +{ + FBExtLogError("Finishing the broadcast: %{public}@", error.description); + [self finishBroadcastWithError:error]; +} + +@end diff --git a/WebDriverAgentBroadcast/FBExtBroadcastClient.h b/WebDriverAgentBroadcast/FBExtBroadcastClient.h new file mode 100644 index 000000000..bfab075c0 --- /dev/null +++ b/WebDriverAgentBroadcast/FBExtBroadcastClient.h @@ -0,0 +1,54 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import + +#import "FBExtSessionPipeline.h" + +NS_ASSUME_NONNULL_BEGIN + +@class FBExtBroadcastClient; + +@protocol FBExtBroadcastClientDelegate + +/** WDA asked the extension to finish the broadcast. */ +- (void)broadcastClientDidRequestStop:(FBExtBroadcastClient *)client; + +/** The connection to WDA could not be (re-)established; the broadcast should be finished. */ +- (void)broadcastClient:(FBExtBroadcastClient *)client didFailPermanently:(NSError *)error; + +@end + +/** + The extension's endpoint of the WDA control connection: connects to WDA's loopback control + port, answers session add/remove/keyframe requests by managing FBExtSessionPipeline instances, + sends the HELLO/heartbeat messages and pushes the pipelines' encoded output. + */ +@interface FBExtBroadcastClient : NSObject + +@property (nonatomic, weak) id delegate; + +/** Stats fed into the heartbeat; updated by the sample handler. */ +@property (atomic) BOOL paused; +@property (atomic) uint64_t framesReceived; +@property (atomic) uint8_t currentOrientation; +@property (atomic) size_t screenWidth; +@property (atomic) size_t screenHeight; + +/** A point-in-time snapshot of the active pipelines, safe to read from any thread. */ +@property (atomic, copy, readonly) NSDictionary *activePipelines; + +/** Starts connecting to the WDA control port (with retries). */ +- (void)start; + +/** Tears down all pipelines and closes the connection (no reconnect attempts). */ +- (void)shutdown; + +@end + +NS_ASSUME_NONNULL_END diff --git a/WebDriverAgentBroadcast/FBExtBroadcastClient.m b/WebDriverAgentBroadcast/FBExtBroadcastClient.m new file mode 100644 index 000000000..0d8fd1195 --- /dev/null +++ b/WebDriverAgentBroadcast/FBExtBroadcastClient.m @@ -0,0 +1,345 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import "FBExtBroadcastClient.h" + +#include + +#import "FBBroadcastProtocol.h" +#import "FBExtLogging.h" +#import "GCDAsyncSocket.h" + +static const long TAG_HEADER = 1; +static const long TAG_PAYLOAD = 2; + +static const NSUInteger CONNECT_ATTEMPTS = 3; +static const NSTimeInterval CONNECT_RETRY_DELAY = 1.0; +static const NSTimeInterval CONNECT_TIMEOUT = 5.0; +static const NSTimeInterval HEARTBEAT_INTERVAL = 2.0; +// Loopback writes should never back up this far; if they do, WDA is wedged - shed delta frames. +static const size_t MAX_OUTSTANDING_WRITE_BYTES = 4 * 1024 * 1024; + +static NSString *const FBExtClientErrorDomain = @"com.facebook.WebDriverAgent.FBExtBroadcastClient"; + +@interface FBExtBroadcastClient () + +@property (nonatomic) dispatch_queue_t queue; +@property (nonatomic, nullable) GCDAsyncSocket *socket; +@property (nonatomic, nullable) dispatch_source_t heartbeatTimer; +@property (nonatomic) NSMutableDictionary *pipelines; +@property (atomic, copy, readwrite) NSDictionary *activePipelines; +@property (nonatomic) NSUInteger remainingConnectAttempts; +@property (atomic) BOOL stopped; +@property (atomic) BOOL connected; +@property (nonatomic) size_t outstandingWriteBytes; +@property (nonatomic) FBBroadcastMessageHeader pendingHeader; +// Previous heartbeat totals used to derive per-second rates (queue-confined). +@property (nonatomic, nullable) NSDictionary *previousPipelineTotals; +@property (nonatomic) uint64_t previousFramesReceived; +@property (nonatomic) uint64_t lastHeartbeatAtMs; + +@end + +static double FBExtRatePerSec(double delta, double intervalSec) +{ + return round(delta / intervalSec * 10) / 10; +} + +@implementation FBExtBroadcastClient + +- (instancetype)init +{ + if ((self = [super init])) { + _queue = dispatch_queue_create("wda.broadcast.client", DISPATCH_QUEUE_SERIAL); + _pipelines = [NSMutableDictionary dictionary]; + _activePipelines = @{}; + _remainingConnectAttempts = CONNECT_ATTEMPTS; + } + return self; +} + +- (void)start +{ + dispatch_async(self.queue, ^{ + [self connect]; + }); +} + +- (void)connect +{ + if (self.stopped) { + return; + } + self.socket = [[GCDAsyncSocket alloc] initWithDelegate:self delegateQueue:self.queue]; + NSError *error; + if (![self.socket connectToHost:@"127.0.0.1" onPort:FBBroadcastDefaultControlPort + withTimeout:CONNECT_TIMEOUT error:&error]) { + FBExtLogError("Cannot start connecting to WDA: %{public}@", error.description); + [self scheduleReconnectOrGiveUp:error]; + } +} + +- (void)scheduleReconnectOrGiveUp:(nullable NSError *)error +{ + if (self.stopped) { + return; + } + if (self.remainingConnectAttempts == 0) { + NSError *permanentError = error + ?: [NSError errorWithDomain:FBExtClientErrorDomain + code:1 + userInfo:@{NSLocalizedDescriptionKey: @"Lost the connection to WebDriverAgent"}]; + [self.delegate broadcastClient:self didFailPermanently:permanentError]; + return; + } + self.remainingConnectAttempts -= 1; + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(CONNECT_RETRY_DELAY * NSEC_PER_SEC)), + self.queue, ^{ + [self connect]; + }); +} + +- (void)shutdown +{ + self.stopped = YES; + dispatch_async(self.queue, ^{ + [self stopHeartbeat]; + for (FBExtSessionPipeline *pipeline in self.pipelines.allValues) { + [pipeline teardown]; + } + [self.pipelines removeAllObjects]; + self.activePipelines = @{}; + self.socket.delegate = nil; + [self.socket disconnect]; + self.socket = nil; + }); +} + +#pragma mark - + +- (void)sendProtocolMessage:(NSData *)message isDroppable:(BOOL)droppable +{ + dispatch_async(self.queue, ^{ + if (!self.connected || nil == self.socket) { + return; + } + if (droppable && self.outstandingWriteBytes > MAX_OUTSTANDING_WRITE_BYTES) { + return; + } + self.outstandingWriteBytes += message.length; + [self.socket writeData:message withTimeout:-1 tag:(long)message.length]; + }); +} + +#pragma mark - Heartbeat + +- (void)startHeartbeat +{ + [self stopHeartbeat]; + dispatch_source_t timer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, self.queue); + if (nil == timer) { + return; + } + dispatch_source_set_timer(timer, + dispatch_time(DISPATCH_TIME_NOW, (int64_t)(HEARTBEAT_INTERVAL * NSEC_PER_SEC)), + (uint64_t)(HEARTBEAT_INTERVAL * NSEC_PER_SEC), + (uint64_t)(0.2 * NSEC_PER_SEC)); + __weak typeof(self) weakSelf = self; + dispatch_source_set_event_handler(timer, ^{ + [weakSelf sendHeartbeat]; + }); + dispatch_resume(timer); + self.heartbeatTimer = timer; +} + +- (void)stopHeartbeat +{ + dispatch_source_t timer = self.heartbeatTimer; + if (nil != timer) { + dispatch_source_cancel(timer); + self.heartbeatTimer = nil; + } +} + +- (void)sendHeartbeat +{ + uint64_t nowMs = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) / NSEC_PER_MSEC; + double intervalSec = self.lastHeartbeatAtMs > 0 + ? MAX(0.001, (double)(nowMs - self.lastHeartbeatAtMs) / 1000.0) + : HEARTBEAT_INTERVAL; + uint64_t framesReceived = self.framesReceived; + + NSMutableDictionary *payload = [NSMutableDictionary dictionaryWithDictionary:@{ + FBBroadcastKeyState: self.paused ? @"paused" : @"active", + FBBroadcastKeyFramesReceived: @(framesReceived), + FBBroadcastKeyOrientation: @(self.currentOrientation), + FBBroadcastKeyScreenWidth: @(self.screenWidth), + FBBroadcastKeyScreenHeight: @(self.screenHeight), + }]; + // ReplayKit delivery rate plus loopback backpressure, so a low consumer-side fps can be + // attributed to delivery, a pipeline stage (see the per-pipeline counters) or the socket. + payload[@"framesReceivedPerSec"] = @(FBExtRatePerSec((double)(framesReceived - self.previousFramesReceived), intervalSec)); + payload[@"socketOutstandingBytes"] = @(self.outstandingWriteBytes); + + NSMutableDictionary *pipelinesJson = [NSMutableDictionary dictionary]; + NSMutableDictionary *newTotals = [NSMutableDictionary dictionary]; + [self.pipelines enumerateKeysAndObjectsUsingBlock:^(NSNumber *sessionId, FBExtSessionPipeline *pipeline, BOOL *stop) { + NSDictionary *metrics = [pipeline metricsSnapshot]; + NSMutableDictionary *entry = [metrics mutableCopy]; + NSDictionary *previous = self.previousPipelineTotals[sessionId.stringValue]; + for (NSString *key in @[@"samplesIn", @"accepted", @"encoded", @"repeated"]) { + double delta = metrics[key].doubleValue - [previous[key] doubleValue]; + entry[[key stringByAppendingString:@"PerSec"]] = @(FBExtRatePerSec(delta, intervalSec)); + } + pipelinesJson[sessionId.stringValue] = entry; + newTotals[sessionId.stringValue] = metrics; + }]; + if (pipelinesJson.count > 0) { + payload[@"pipelines"] = pipelinesJson; + } + self.previousPipelineTotals = newTotals; + self.previousFramesReceived = framesReceived; + self.lastHeartbeatAtMs = nowMs; + + NSData *message = FBBroadcastEncodeJSONMessage(FBBroadcastMessageTypeHeartbeat, 0, payload); + if (nil != message) { + [self sendProtocolMessage:message isDroppable:NO]; + } +} + +#pragma mark - Incoming messages + +- (void)handleMessageWithHeader:(FBBroadcastMessageHeader)header payload:(NSData *)payload +{ + switch (header.type) { + case FBBroadcastMessageTypeSessionAdd: { + NSDictionary *configuration = FBBroadcastParseJSONPayload(payload); + if (nil == configuration) { + FBExtLogError("Session %u: SESSION_ADD payload is not a JSON object", header.sessionId); + return; + } + [self addSession:header.sessionId configuration:configuration]; + return; + } + case FBBroadcastMessageTypeSessionRemove: + [self removeSession:header.sessionId]; + return; + case FBBroadcastMessageTypeKeyframeRequest: + [self.pipelines[@(header.sessionId)] requestKeyFrame]; + return; + case FBBroadcastMessageTypeStopBroadcast: + FBExtLogInfo("WDA requested the broadcast to stop"); + self.stopped = YES; + [self.delegate broadcastClientDidRequestStop:self]; + return; + default: + FBExtLogError("Ignoring an unexpected message of type 0x%02x", header.type); + return; + } +} + +- (void)addSession:(uint32_t)sessionId configuration:(NSDictionary *)configuration +{ + [self removeSession:sessionId]; + NSError *error; + FBExtSessionPipeline *pipeline = [[FBExtSessionPipeline alloc] initWithSessionId:sessionId + configuration:configuration + sink:self + error:&error]; + if (nil == pipeline) { + FBExtLogError("Session %u: cannot create a pipeline: %{public}@", sessionId, error.description); + NSData *message = FBBroadcastEncodeJSONMessage(FBBroadcastMessageTypeSessionError, sessionId, @{ + FBBroadcastKeyMessage: error.localizedDescription ?: @"Cannot create the session pipeline", + }); + if (nil != message) { + [self sendProtocolMessage:message isDroppable:NO]; + } + return; + } + self.pipelines[@(sessionId)] = pipeline; + self.activePipelines = self.pipelines; + FBExtLogInfo("Session %u: pipeline created (%{public}@)", sessionId, configuration.description); +} + +- (void)removeSession:(uint32_t)sessionId +{ + FBExtSessionPipeline *pipeline = self.pipelines[@(sessionId)]; + if (nil == pipeline) { + return; + } + [pipeline teardown]; + [self.pipelines removeObjectForKey:@(sessionId)]; + self.activePipelines = self.pipelines; + FBExtLogInfo("Session %u: pipeline removed", sessionId); +} + +#pragma mark - + +- (void)socket:(GCDAsyncSocket *)sock didConnectToHost:(NSString *)host port:(uint16_t)port +{ + FBExtLogInfo("Connected to WDA at %{public}@:%d", host, port); + self.connected = YES; + self.remainingConnectAttempts = CONNECT_ATTEMPTS; + NSData *hello = FBBroadcastEncodeJSONMessage(FBBroadcastMessageTypeHello, 0, @{ + FBBroadcastKeyProtocolVersion: @(FBBroadcastProtocolVersion), + FBBroadcastKeyOsVersion: NSProcessInfo.processInfo.operatingSystemVersionString, + }); + if (nil != hello) { + [self sendProtocolMessage:hello isDroppable:NO]; + } + [self startHeartbeat]; + [sock readDataToLength:FBBroadcastHeaderLength withTimeout:-1 tag:TAG_HEADER]; +} + +- (void)socket:(GCDAsyncSocket *)sock didReadData:(NSData *)data withTag:(long)tag +{ + if (tag == TAG_HEADER) { + FBBroadcastMessageHeader header; + if (!FBBroadcastParseHeader(data, &header)) { + FBExtLogError("Malformed control message header; disconnecting"); + [sock disconnect]; + return; + } + self.pendingHeader = header; + if (header.payloadLength == 0) { + [self handleMessageWithHeader:header payload:NSData.data]; + [sock readDataToLength:FBBroadcastHeaderLength withTimeout:-1 tag:TAG_HEADER]; + } else { + [sock readDataToLength:header.payloadLength withTimeout:-1 tag:TAG_PAYLOAD]; + } + return; + } + [self handleMessageWithHeader:self.pendingHeader payload:data]; + [sock readDataToLength:FBBroadcastHeaderLength withTimeout:-1 tag:TAG_HEADER]; +} + +- (void)socket:(GCDAsyncSocket *)sock didWriteDataWithTag:(long)tag +{ + if (tag > 0 && self.outstandingWriteBytes >= (size_t)tag) { + self.outstandingWriteBytes -= (size_t)tag; + } else { + self.outstandingWriteBytes = 0; + } +} + +- (void)socketDidDisconnect:(GCDAsyncSocket *)sock withError:(nullable NSError *)error +{ + if (sock != self.socket) { + return; + } + self.connected = NO; + self.outstandingWriteBytes = 0; + [self stopHeartbeat]; + if (self.stopped) { + return; + } + FBExtLogError("Disconnected from WDA: %{public}@", error.description ?: @"closed"); + [self scheduleReconnectOrGiveUp:error]; +} + +@end diff --git a/WebDriverAgentBroadcast/FBExtLogging.h b/WebDriverAgentBroadcast/FBExtLogging.h new file mode 100644 index 000000000..f898407df --- /dev/null +++ b/WebDriverAgentBroadcast/FBExtLogging.h @@ -0,0 +1,27 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import + +/** + Minimal logging for the broadcast extension. FBLogger cannot be used here because it pulls in + FBConfiguration and, transitively, XCTest, which is forbidden in app extensions. + View with: log stream --predicate 'subsystem == "com.facebook.WebDriverAgent.broadcast"' + */ +static inline os_log_t FBExtLog(void) +{ + static os_log_t logger; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + logger = os_log_create("com.facebook.WebDriverAgent.broadcast", "extension"); + }); + return logger; +} + +#define FBExtLogInfo(...) os_log_info(FBExtLog(), __VA_ARGS__) +#define FBExtLogError(...) os_log_error(FBExtLog(), __VA_ARGS__) diff --git a/WebDriverAgentBroadcast/FBExtSessionPipeline.h b/WebDriverAgentBroadcast/FBExtSessionPipeline.h new file mode 100644 index 000000000..b7751c1eb --- /dev/null +++ b/WebDriverAgentBroadcast/FBExtSessionPipeline.h @@ -0,0 +1,74 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +/** Receives fully framed protocol messages produced by a pipeline. */ +@protocol FBExtMessageSink + +/** + @param message A complete wire message (header + payload) + @param droppable YES when the message may be dropped under backpressure (delta frames) + */ +- (void)sendProtocolMessage:(NSData *)message isDroppable:(BOOL)droppable; + +@end + +/** + One per WDA capture session: letterbox-scales incoming ReplayKit pixel buffers to the session's + dimensions and encodes them with the session's codec/bitrate/fps, emitting VIDEO_PARAMS and + VIDEO_FRAME protocol messages to the sink. + */ +@interface FBExtSessionPipeline : NSObject + +@property (nonatomic, readonly) uint32_t sessionId; + +/** + @param sessionId The WDA session identifier + @param configuration The SESSION_ADD JSON payload (width, height, codec, bitrate, fps) + @param sink The message sink (held weakly) + @param error Set when the encoder or scaler cannot be created + */ +- (nullable instancetype)initWithSessionId:(uint32_t)sessionId + configuration:(NSDictionary *)configuration + sink:(id)sink + error:(NSError **)error; + +- (instancetype)init NS_UNAVAILABLE; + +/** + Submits a ReplayKit video sample for this session. Called on the ReplayKit sample queue and + never blocks: the frame is dropped when the session is not yet due for a new frame (fps pacing) + or when the previous frame is still being scaled/submitted. + + @param sampleBuffer The video sample from ReplayKit + @param orientation The CGImagePropertyOrientation (1-8) reported for the sample + */ +- (void)submitSampleBuffer:(CMSampleBufferRef)sampleBuffer orientation:(uint8_t)orientation; + +/** Forces the next encoded frame to be a key frame. */ +- (void)requestKeyFrame; + +/** + A point-in-time snapshot of this pipeline's counters, safe to call from any thread: + samplesIn (frames offered by ReplayKit), accepted (passed the fps gate), encoded (encoder + callbacks), droppedFpsGate, droppedReplaced (latched frame superseded before processing), + droppedPool (scaled-buffer pool exhausted), repeated (last frame re-encoded to fill a + delivery gap), encodeLatencyMsLast/encodeLatencyMsAvg (submit-to-encoder-callback time). + */ +- (NSDictionary *)metricsSnapshot; + +/** Stops the encoder and releases the scaler and buffer pool. */ +- (void)teardown; + +@end + +NS_ASSUME_NONNULL_END diff --git a/WebDriverAgentBroadcast/FBExtSessionPipeline.m b/WebDriverAgentBroadcast/FBExtSessionPipeline.m new file mode 100644 index 000000000..3b5019f58 --- /dev/null +++ b/WebDriverAgentBroadcast/FBExtSessionPipeline.m @@ -0,0 +1,544 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import "FBExtSessionPipeline.h" + +#include + +#import +#import + +#import "FBBroadcastProtocol.h" +#import "FBExtLogging.h" +#import "FBVideoEncoder.h" + +static NSString *const FBExtPipelineErrorDomain = @"com.facebook.WebDriverAgent.FBExtSessionPipeline"; + +// Bounds the scaled-buffer pool so a stuck consumer cannot grow the extension past its ~50MB cap. +static const int POOL_ALLOCATION_THRESHOLD = 4; + +@interface FBExtSessionPipeline () { + CMSampleBufferRef _pendingSampleBuffer; + /** The most recently encoded (pool-owned) frame, re-encoded to fill delivery gaps (queue-confined). */ + CVPixelBufferRef _repeatBuffer; +} + +@property (nonatomic, weak) id sink; +@property (nonatomic) dispatch_queue_t queue; +@property (nonatomic, nullable) FBVideoEncoder *encoder; +@property (nonatomic) VTPixelTransferSessionRef transferSession; +@property (nonatomic) CVPixelBufferPoolRef bufferPool; +@property (nonatomic) NSUInteger width; +@property (nonatomic) NSUInteger height; +@property (nonatomic) NSUInteger fps; +@property (nonatomic) uint64_t lastSubmitTimeMs; +/** The next monotonic timestamp at which a frame is due (fps gate accumulator). */ +@property (nonatomic) uint64_t nextDueMs; +@property (nonatomic) BOOL inFlight; +@property (nonatomic) uint64_t pendingSubmitTimeMs; +@property (nonatomic) uint8_t pendingOrientation; +@property (atomic) BOOL active; +@property (atomic) uint8_t currentOrientation; +@property (nonatomic) BOOL directSourceEncodingDisabled; +@property (nonatomic, nullable, copy) NSData *lastSentParameterSets; + +// Lightweight diagnostics, exposed via metricsSnapshot/the heartbeat. Increments are not +// strictly atomic RMW, which is acceptable for metrics. +@property (atomic) uint64_t samplesInCount; +@property (atomic) uint64_t acceptedCount; +@property (atomic) uint64_t encodedCount; +@property (atomic) uint64_t droppedFpsGateCount; +@property (atomic) uint64_t droppedReplacedCount; +@property (atomic) uint64_t droppedPoolCount; +@property (atomic) uint64_t repeatedCount; +@property (atomic) uint64_t lastEncodeLatencyMs; +@property (atomic) double avgEncodeLatencyMs; + +// Frame repeater state (queue-confined except the timer handle). +@property (nonatomic, nullable) dispatch_source_t repeatTimer; +@property (nonatomic) uint64_t frameIntervalMs; +@property (nonatomic) uint64_t lastEncodeAtMs; + +- (void)replacePendingSampleBuffer:(CMSampleBufferRef)sampleBuffer + atTimeMs:(uint64_t)nowMs + orientation:(uint8_t)orientation; +- (nullable CMSampleBufferRef)copyPendingSampleBufferAtTimeMs:(uint64_t *)timeMs + orientation:(uint8_t *)orientation; +- (void)clearPendingSampleBufferLocked; + +@end + +@implementation FBExtSessionPipeline + +- (nullable instancetype)initWithSessionId:(uint32_t)sessionId + configuration:(NSDictionary *)configuration + sink:(id)sink + error:(NSError **)error +{ + if ((self = [super init])) { + _sessionId = sessionId; + _sink = sink; + _queue = dispatch_queue_create([NSString stringWithFormat:@"wda.broadcast.session.%u", sessionId].UTF8String, + DISPATCH_QUEUE_SERIAL); + + NSUInteger width = [configuration[FBBroadcastKeyWidth] unsignedIntegerValue]; + NSUInteger height = [configuration[FBBroadcastKeyHeight] unsignedIntegerValue]; + // Hardware encoders require even dimensions (same rule as the WDA-side converter). + width -= width % 2; + height -= height % 2; + NSUInteger bitrate = [configuration[FBBroadcastKeyBitrate] unsignedIntegerValue]; + NSUInteger fps = [configuration[FBBroadcastKeyFps] unsignedIntegerValue]; + NSString *codecName = [configuration[FBBroadcastKeyCodec] isKindOfClass:NSString.class] + ? (NSString *)configuration[FBBroadcastKeyCodec] + : @""; + FBVideoCodec codec = [FBBroadcastCodecH265 isEqualToString:codecName] + ? FBVideoCodecH265 + : FBVideoCodecH264; + if (width == 0 || height == 0) { + if (error) { + *error = [NSError errorWithDomain:FBExtPipelineErrorDomain + code:1 + userInfo:@{NSLocalizedDescriptionKey: @"Session configuration is missing positive 'width'/'height'"}]; + } + return nil; + } + _fps = fps; + _width = width; + _height = height; + + OSStatus status = VTPixelTransferSessionCreate(kCFAllocatorDefault, &_transferSession); + if (status != noErr || NULL == _transferSession) { + if (error) { + *error = [NSError errorWithDomain:FBExtPipelineErrorDomain + code:status + userInfo:@{NSLocalizedDescriptionKey: [NSString stringWithFormat:@"Cannot create a pixel transfer session (status %d)", (int)status]}]; + } + return nil; + } + // Letterbox to preserve the aspect ratio, matching the WDA-side screenshot converter. + VTSessionSetProperty(_transferSession, kVTPixelTransferPropertyKey_ScalingMode, kVTScalingMode_Letterbox); + VTSessionSetProperty(_transferSession, kVTPixelTransferPropertyKey_RealTime, kCFBooleanTrue); + + NSDictionary *pixelBufferAttributes = @{ + (id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange), + (id)kCVPixelBufferWidthKey: @(width), + (id)kCVPixelBufferHeightKey: @(height), + (id)kCVPixelBufferIOSurfacePropertiesKey: @{}, + }; + NSDictionary *poolAttributes = @{(id)kCVPixelBufferPoolMinimumBufferCountKey: @(POOL_ALLOCATION_THRESHOLD)}; + CVReturn poolStatus = CVPixelBufferPoolCreate(kCFAllocatorDefault, + (__bridge CFDictionaryRef)poolAttributes, + (__bridge CFDictionaryRef)pixelBufferAttributes, + &_bufferPool); + if (poolStatus != kCVReturnSuccess || NULL == _bufferPool) { + VTPixelTransferSessionInvalidate(_transferSession); + CFRelease(_transferSession); + _transferSession = NULL; + if (error) { + *error = [NSError errorWithDomain:FBExtPipelineErrorDomain + code:poolStatus + userInfo:@{NSLocalizedDescriptionKey: [NSString stringWithFormat:@"Cannot create a pixel buffer pool (status %d)", (int)poolStatus]}]; + } + return nil; + } + + FBVideoEncoder *encoder = [[FBVideoEncoder alloc] initWithCodec:codec + width:width + height:height + bitrate:bitrate > 0 ? bitrate : 6000000 + fps:fps > 0 ? fps : 30 + error:error]; + if (nil == encoder) { + [self releaseScalerResources]; + return nil; + } + encoder.delegate = self; + _encoder = encoder; + _active = YES; + // Open every session with an IDR: WDA only switches a stream onto the broadcast source once + // a key frame (with parameter sets) has arrived. + [encoder requestKeyFrame]; + + // ReplayKit delivers nothing while the screen is static and VideoToolbox has no + // Android-style repeat-previous-frame mode, so the last frame is re-encoded manually to + // keep the output cadence at the requested fps. + _frameIntervalMs = fps > 0 ? (uint64_t)(1000 / fps) : 0; + if (_frameIntervalMs > 1) { + [self startRepeatTimer]; + } + } + return self; +} + +- (void)startRepeatTimer +{ + dispatch_source_t timer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, self.queue); + if (nil == timer) { + return; + } + uint64_t intervalNs = self.frameIntervalMs * NSEC_PER_MSEC; + dispatch_source_set_timer(timer, + dispatch_time(DISPATCH_TIME_NOW, (int64_t)intervalNs), + intervalNs, + intervalNs / 4); + __weak typeof(self) weakSelf = self; + dispatch_source_set_event_handler(timer, ^{ + [weakSelf repeatLastFrameIfIdle]; + }); + dispatch_resume(timer); + self.repeatTimer = timer; +} + +- (void)repeatLastFrameIfIdle +{ + // Runs on self.queue, serialized with live encodes. + if (!self.active || NULL == _repeatBuffer || 0 == self.lastEncodeAtMs) { + return; + } + uint64_t nowMs = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) / NSEC_PER_MSEC; + if (nowMs - self.lastEncodeAtMs < self.frameIntervalMs) { + return; + } + NSError *error; + if ([self.encoder encodePixelBuffer:_repeatBuffer presentationTimeMs:nowMs error:&error]) { + self.repeatedCount += 1; + self.lastEncodeAtMs = nowMs; + } else { + FBExtLogError("Session %u: cannot re-encode the repeat frame: %{public}@", + self.sessionId, error.description); + } +} + +// Runs on self.queue. Retains the given pool-owned buffer as the repeat source. +- (void)storeRepeatBuffer:(CVPixelBufferRef)buffer +{ + if (buffer == _repeatBuffer) { + return; + } + CVPixelBufferRef previous = _repeatBuffer; + _repeatBuffer = (CVPixelBufferRef)CFRetain(buffer); + if (NULL != previous) { + CVPixelBufferRelease(previous); + } +} + +// Runs on self.queue. Copies a non-pool source (direct-encode path) into a pool buffer for the +// repeater: ReplayKit recycles its sample buffers, so retaining one would stall its capture. +- (void)refreshRepeatBufferWithCopyOf:(CVPixelBufferRef)sourceBuffer +{ + CVPixelBufferRef copyBuffer = NULL; + NSDictionary *auxAttributes = @{(id)kCVPixelBufferPoolAllocationThresholdKey: @(POOL_ALLOCATION_THRESHOLD)}; + CVReturn poolStatus = CVPixelBufferPoolCreatePixelBufferWithAuxAttributes(kCFAllocatorDefault, + self.bufferPool, + (__bridge CFDictionaryRef)auxAttributes, + ©Buffer); + if (poolStatus != kCVReturnSuccess || NULL == copyBuffer) { + // Keep the previous (slightly stale) repeat buffer instead of growing the pool. + return; + } + if (noErr == VTPixelTransferSessionTransferImage(self.transferSession, sourceBuffer, copyBuffer)) { + [self storeRepeatBuffer:copyBuffer]; + } + CVPixelBufferRelease(copyBuffer); +} + +- (void)submitSampleBuffer:(CMSampleBufferRef)sampleBuffer orientation:(uint8_t)orientation +{ + self.samplesInCount += 1; + if (!self.active) { + return; + } + + // Respect this session's framerate even though ReplayKit may deliver faster. The gate is a + // due-time accumulator rather than a minimum gap from the last accepted frame: gap-based + // pacing beats against jittery ~30Hz delivery (a frame arriving 1ms early is dropped and the + // next accepted gap doubles), which measured ~22fps out of a 42fps input. The accumulator + // admits exactly one frame per interval on average regardless of arrival jitter. + uint64_t nowMs = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) / NSEC_PER_MSEC; + uint64_t minIntervalMs = self.fps > 0 ? (uint64_t)(1000 / self.fps) : 0; + @synchronized (self) { + if (minIntervalMs > 1) { + if (nowMs + 1 < self.nextDueMs) { + self.droppedFpsGateCount += 1; + return; + } + // Re-anchor after stalls so accumulated due time does not admit a burst. + self.nextDueMs = (self.nextDueMs == 0 || nowMs > self.nextDueMs + minIntervalMs) + ? nowMs + minIntervalMs + : self.nextDueMs + minIntervalMs; + } + self.acceptedCount += 1; + + // Never block the ReplayKit sample queue. If a frame is already being scaled/submitted, + // retain only the newest due sample and replace any older pending sample. + if (self.inFlight) { + [self replacePendingSampleBuffer:sampleBuffer atTimeMs:nowMs orientation:orientation]; + return; + } + self.inFlight = YES; + self.lastSubmitTimeMs = nowMs; + } + + CFRetain(sampleBuffer); + dispatch_async(self.queue, ^{ + [self drainRetainedSampleBuffer:sampleBuffer atTimeMs:nowMs orientation:orientation]; + CFRelease(sampleBuffer); + }); +} + +- (void)replacePendingSampleBuffer:(CMSampleBufferRef)sampleBuffer + atTimeMs:(uint64_t)nowMs + orientation:(uint8_t)orientation +{ + CMSampleBufferRef retainedSampleBuffer = (CMSampleBufferRef)CFRetain(sampleBuffer); + CMSampleBufferRef previous = _pendingSampleBuffer; + _pendingSampleBuffer = retainedSampleBuffer; + self.pendingSubmitTimeMs = nowMs; + self.pendingOrientation = orientation; + if (NULL != previous) { + CFRelease(previous); + self.droppedReplacedCount += 1; + } +} + +- (nullable CMSampleBufferRef)copyPendingSampleBufferAtTimeMs:(uint64_t *)timeMs + orientation:(uint8_t *)orientation +{ + @synchronized (self) { + if (!self.active) { + [self clearPendingSampleBufferLocked]; + self.inFlight = NO; + return NULL; + } + CMSampleBufferRef pending = _pendingSampleBuffer; + if (NULL == pending) { + self.inFlight = NO; + return NULL; + } + _pendingSampleBuffer = NULL; + *timeMs = self.pendingSubmitTimeMs; + *orientation = self.pendingOrientation; + self.lastSubmitTimeMs = self.pendingSubmitTimeMs; + return pending; + } +} + +- (void)clearPendingSampleBufferLocked +{ + CMSampleBufferRef pending = _pendingSampleBuffer; + _pendingSampleBuffer = NULL; + if (NULL != pending) { + CFRelease(pending); + } +} + +- (void)drainRetainedSampleBuffer:(CMSampleBufferRef)sampleBuffer + atTimeMs:(uint64_t)nowMs + orientation:(uint8_t)orientation +{ + CMSampleBufferRef currentSampleBuffer = sampleBuffer; + uint64_t currentTimeMs = nowMs; + uint8_t currentOrientation = orientation; + BOOL shouldReleaseCurrentSampleBuffer = NO; + + while (NULL != currentSampleBuffer) { + [self processRetainedSampleBuffer:currentSampleBuffer + atTimeMs:currentTimeMs + orientation:currentOrientation]; + if (shouldReleaseCurrentSampleBuffer) { + CFRelease(currentSampleBuffer); + } + currentSampleBuffer = [self copyPendingSampleBufferAtTimeMs:¤tTimeMs + orientation:¤tOrientation]; + shouldReleaseCurrentSampleBuffer = (NULL != currentSampleBuffer); + } +} + +- (void)processRetainedSampleBuffer:(CMSampleBufferRef)sampleBuffer + atTimeMs:(uint64_t)nowMs + orientation:(uint8_t)orientation +{ + if (!self.active) { + return; + } + self.currentOrientation = orientation; + CVPixelBufferRef sourceBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + if (NULL == sourceBuffer) { + return; + } + + if (!self.directSourceEncodingDisabled + && CVPixelBufferGetWidth(sourceBuffer) == self.width + && CVPixelBufferGetHeight(sourceBuffer) == self.height) { + NSError *error; + if ([self.encoder encodePixelBuffer:sourceBuffer presentationTimeMs:nowMs error:&error]) { + self.lastEncodeAtMs = nowMs; + [self refreshRepeatBufferWithCopyOf:sourceBuffer]; + return; + } + self.directSourceEncodingDisabled = YES; + FBExtLogError("Session %u: direct source encode failed; falling back to pixel transfer: %{public}@", + self.sessionId, + error.description); + } + + CVPixelBufferRef scaledBuffer = NULL; + NSDictionary *auxAttributes = @{(id)kCVPixelBufferPoolAllocationThresholdKey: @(POOL_ALLOCATION_THRESHOLD)}; + CVReturn poolStatus = CVPixelBufferPoolCreatePixelBufferWithAuxAttributes(kCFAllocatorDefault, + self.bufferPool, + (__bridge CFDictionaryRef)auxAttributes, + &scaledBuffer); + if (poolStatus != kCVReturnSuccess || NULL == scaledBuffer) { + // The pool is exhausted (encoder still holds the buffers) - drop the frame instead of growing. + self.droppedPoolCount += 1; + return; + } + + OSStatus transferStatus = VTPixelTransferSessionTransferImage(self.transferSession, sourceBuffer, scaledBuffer); + if (transferStatus != noErr) { + FBExtLogError("Session %u: pixel transfer failed (status %d)", self.sessionId, (int)transferStatus); + CVPixelBufferRelease(scaledBuffer); + return; + } + + NSError *error; + if ([self.encoder encodePixelBuffer:scaledBuffer presentationTimeMs:nowMs error:&error]) { + self.lastEncodeAtMs = nowMs; + [self storeRepeatBuffer:scaledBuffer]; + } else { + FBExtLogError("Session %u: cannot encode a frame: %{public}@", self.sessionId, error.description); + } + CVPixelBufferRelease(scaledBuffer); +} + +- (void)requestKeyFrame +{ + [self.encoder requestKeyFrame]; +} + +- (NSDictionary *)metricsSnapshot +{ + return @{ + @"samplesIn": @(self.samplesInCount), + @"accepted": @(self.acceptedCount), + @"encoded": @(self.encodedCount), + @"droppedFpsGate": @(self.droppedFpsGateCount), + @"droppedReplaced": @(self.droppedReplacedCount), + @"droppedPool": @(self.droppedPoolCount), + @"repeated": @(self.repeatedCount), + @"encodeLatencyMsLast": @(self.lastEncodeLatencyMs), + @"encodeLatencyMsAvg": @(round(self.avgEncodeLatencyMs * 10) / 10), + }; +} + +- (void)teardown +{ + self.active = NO; + @synchronized (self) { + [self clearPendingSampleBufferLocked]; + } + dispatch_source_t timer = self.repeatTimer; + if (nil != timer) { + dispatch_source_cancel(timer); + self.repeatTimer = nil; + } + dispatch_async(self.queue, ^{ + if (NULL != self->_repeatBuffer) { + CVPixelBufferRelease(self->_repeatBuffer); + self->_repeatBuffer = NULL; + } + if (nil != self.encoder) { + self.encoder.delegate = nil; + [self.encoder stop]; + self.encoder = nil; + } + [self releaseScalerResources]; + }); +} + +- (void)releaseScalerResources +{ + if (NULL != self.transferSession) { + VTPixelTransferSessionInvalidate(self.transferSession); + CFRelease(self.transferSession); + self.transferSession = NULL; + } + if (NULL != self.bufferPool) { + CVPixelBufferPoolRelease(self.bufferPool); + self.bufferPool = NULL; + } +} + +- (void)dealloc +{ + @synchronized (self) { + [self clearPendingSampleBufferLocked]; + } + if (nil != _repeatTimer) { + dispatch_source_cancel(_repeatTimer); + } + if (NULL != _repeatBuffer) { + CVPixelBufferRelease(_repeatBuffer); + _repeatBuffer = NULL; + } + if (nil != _encoder) { + _encoder.delegate = nil; + [_encoder stop]; + } + [self releaseScalerResources]; +} + +#pragma mark - + +- (void)videoEncoder:(FBVideoEncoder *)encoder + didEncodeFrame:(NSData *)annexBPictureData + isKeyFrame:(BOOL)isKeyFrame + presentationTimeUs:(uint64_t)presentationTimeUs +{ + self.encodedCount += 1; + // The frame pts is the monotonic submit time, so callback-time minus pts is the + // scale+encode latency. + uint64_t nowMs = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) / NSEC_PER_MSEC; + uint64_t submitMs = presentationTimeUs / 1000; + if (nowMs >= submitMs) { + uint64_t latencyMs = nowMs - submitMs; + self.lastEncodeLatencyMs = latencyMs; + double avg = self.avgEncodeLatencyMs; + self.avgEncodeLatencyMs = avg <= 0 ? (double)latencyMs : avg * 0.9 + (double)latencyMs * 0.1; + } + + if (!self.active || annexBPictureData.length == 0) { + return; + } + id sink = self.sink; + if (nil == sink) { + return; + } + + // WDA needs the parameter sets before the first IDR that uses them. + if (isKeyFrame) { + NSData *parameterSets = encoder.parameterSetAnnexB; + NSData *lastSent = self.lastSentParameterSets; + if (parameterSets.length > 0 && (nil == lastSent || ![parameterSets isEqualToData:lastSent])) { + self.lastSentParameterSets = parameterSets; + [sink sendProtocolMessage:FBBroadcastEncodeMessage(FBBroadcastMessageTypeVideoParams, + self.sessionId, + parameterSets) + isDroppable:NO]; + } + } + + [sink sendProtocolMessage:FBBroadcastEncodeVideoFrameMessage(self.sessionId, + presentationTimeUs, + isKeyFrame, + self.currentOrientation, + annexBPictureData) + isDroppable:!isKeyFrame]; +} + +@end diff --git a/WebDriverAgentBroadcast/Info.plist b/WebDriverAgentBroadcast/Info.plist new file mode 100644 index 000000000..acc5d9fe0 --- /dev/null +++ b/WebDriverAgentBroadcast/Info.plist @@ -0,0 +1,35 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleDisplayName + WebDriverAgent Broadcast + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + XPC! + CFBundleShortVersionString + 1.0 + CFBundleSignature + ???? + CFBundleVersion + 1 + NSExtension + + NSExtensionPointIdentifier + com.apple.broadcast-services-upload + NSExtensionPrincipalClass + FBBroadcastSampleHandler + RPBroadcastProcessMode + RPBroadcastProcessModeSampleBuffer + + + diff --git a/WebDriverAgentLib/Commands/FBScreenCaptureCommands.m b/WebDriverAgentLib/Commands/FBScreenCaptureCommands.m index d4f79a0c9..fe984e5fe 100644 --- a/WebDriverAgentLib/Commands/FBScreenCaptureCommands.m +++ b/WebDriverAgentLib/Commands/FBScreenCaptureCommands.m @@ -8,12 +8,14 @@ #import "FBScreenCaptureCommands.h" +#import "FBBroadcastManager.h" #import "FBConfiguration.h" #import "FBRouteRequest.h" #import "FBVideoStreamManager.h" static const NSUInteger DEFAULT_CAPTURE_FPS = 30; static const NSUInteger DEFAULT_CAPTURE_BITRATE = 6000000; +static const CGFloat DEFAULT_CAPTURE_QUALITY = 0.8; @implementation FBScreenCaptureCommands @@ -23,6 +25,16 @@ + (NSArray *)routes { return @[ + // The broadcast routes must be registered before the '/:id' routes: RoutingHTTPServer + // matches routes in registration order, so 'GET /mobilerun/screencapture/broadcast' would + // otherwise be swallowed by 'GET /mobilerun/screencapture/:id'. + [[FBRoute POST:@"/mobilerun/screencapture/broadcast/start"] respondWithTarget:self action:@selector(handleStartBroadcast:)], + [[FBRoute POST:@"/mobilerun/screencapture/broadcast/stop"] respondWithTarget:self action:@selector(handleStopBroadcast:)], + [[FBRoute GET:@"/mobilerun/screencapture/broadcast"] respondWithTarget:self action:@selector(handleGetBroadcastStatus:)], + [[FBRoute POST:@"/mobilerun/screencapture/broadcast/start"].withoutSession respondWithTarget:self action:@selector(handleStartBroadcast:)], + [[FBRoute POST:@"/mobilerun/screencapture/broadcast/stop"].withoutSession respondWithTarget:self action:@selector(handleStopBroadcast:)], + [[FBRoute GET:@"/mobilerun/screencapture/broadcast"].withoutSession respondWithTarget:self action:@selector(handleGetBroadcastStatus:)], + [[FBRoute POST:@"/mobilerun/screencapture/start"] respondWithTarget:self action:@selector(handleStartScreenCapture:)], [[FBRoute POST:@"/mobilerun/screencapture/stop"] respondWithTarget:self action:@selector(handleStopAllScreenCapture:)], [[FBRoute GET:@"/mobilerun/screencapture"] respondWithTarget:self action:@selector(handleListScreenCapture:)], @@ -41,6 +53,59 @@ + (NSArray *)routes #pragma mark - Commands ++ (id)handleStartBroadcast:(FBRouteRequest *)request +{ + NSTimeInterval timeout = 30.0; + NSNumber *timeoutArg = request.arguments[@"timeout"]; + if ([timeoutArg isKindOfClass:NSNumber.class] && timeoutArg.doubleValue > 0) { + timeout = timeoutArg.doubleValue; + } + NSMutableArray *confirmButtonLabels = [NSMutableArray array]; + id labelsArg = request.arguments[@"confirmButtonLabels"]; + if ([labelsArg isKindOfClass:NSArray.class]) { + for (id label in (NSArray *)labelsArg) { + if ([label isKindOfClass:NSString.class] && [(NSString *)label length] > 0) { + [confirmButtonLabels addObject:label]; + } + } + } + NSNumber *restoreArg = request.arguments[@"restoreForegroundApp"]; + BOOL restoreForegroundApp = [restoreArg isKindOfClass:NSNumber.class] ? restoreArg.boolValue : YES; + + NSError *error; + if (![FBBroadcastManager.sharedInstance startBroadcastWithTimeout:timeout + confirmButtonLabels:confirmButtonLabels + restoreForegroundApp:restoreForegroundApp + error:&error]) { + if ([error.domain isEqualToString:FBBroadcastManagerErrorDomain]) { + switch (error.code) { + case FBBroadcastManagerErrorUnsupported: + return FBResponseWithStatus([FBCommandStatus unsupportedOperationErrorWithMessage:error.localizedDescription traceback:nil]); + case FBBroadcastManagerErrorTimeout: + return FBResponseWithStatus([FBCommandStatus timeoutErrorWithMessage:error.localizedDescription traceback:nil]); + default: + break; + } + } + return FBResponseWithUnknownError(error); + } + return FBResponseWithObject([FBBroadcastManager.sharedInstance statusDictionary]); +} + ++ (id)handleStopBroadcast:(FBRouteRequest *)request +{ + NSError *error; + if (![FBBroadcastManager.sharedInstance stopBroadcastWithError:&error]) { + return FBResponseWithStatus([FBCommandStatus timeoutErrorWithMessage:error.localizedDescription traceback:nil]); + } + return FBResponseWithObject([FBBroadcastManager.sharedInstance statusDictionary]); +} + ++ (id)handleGetBroadcastStatus:(FBRouteRequest *)request +{ + return FBResponseWithObject([FBBroadcastManager.sharedInstance statusDictionary]); +} + + (id)handleStartScreenCapture:(FBRouteRequest *)request { FBVideoCodec codec; @@ -69,6 +134,14 @@ + (NSArray *)routes configuration.height = (NSUInteger)(height - (height % 2)); NSNumber *bitrate = request.arguments[@"bitrate"]; configuration.bitrate = (nil != bitrate && bitrate.integerValue > 0) ? bitrate.unsignedIntegerValue : DEFAULT_CAPTURE_BITRATE; + id quality = request.arguments[@"quality"]; + if (nil == quality) { + configuration.quality = DEFAULT_CAPTURE_QUALITY; + } else if (![quality isKindOfClass:NSNumber.class] || ((NSNumber *)quality).doubleValue < 0.0 || ((NSNumber *)quality).doubleValue > 1.0) { + return FBResponseWithStatus([FBCommandStatus invalidArgumentErrorWithMessage:@"'quality' must be a number in the range 0.0..1.0" traceback:nil]); + } else { + configuration.quality = (CGFloat)((NSNumber *)quality).doubleValue; + } NSNumber *fps = request.arguments[@"fps"]; configuration.fps = (nil != fps && fps.integerValue > 0) ? fps.unsignedIntegerValue : DEFAULT_CAPTURE_FPS; NSNumber *port = request.arguments[@"port"]; diff --git a/WebDriverAgentLib/Routing/FBWebServer.m b/WebDriverAgentLib/Routing/FBWebServer.m index 9c37d2f5b..29bdaa6f6 100644 --- a/WebDriverAgentLib/Routing/FBWebServer.m +++ b/WebDriverAgentLib/Routing/FBWebServer.m @@ -11,6 +11,7 @@ #import "RoutingConnection.h" #import "RoutingHTTPServer.h" +#import "FBBroadcastManager.h" #import "FBCommandHandler.h" #import "FBErrorBuilder.h" #import "FBExceptionHandler.h" @@ -79,6 +80,8 @@ - (void)startServing self.exceptionHandler = [FBExceptionHandler new]; [self startHTTPServer]; [self initScreenshotsBroadcaster]; + // Listen permanently so broadcasts started from Control Center attach as well. + [FBBroadcastManager.sharedInstance startListening]; self.keepAlive = YES; NSRunLoop *runLoop = [NSRunLoop mainRunLoop]; @@ -179,6 +182,7 @@ - (void)stopServing { [FBSession.activeSession kill]; [FBVideoStreamManager.sharedInstance stopAllSessions]; + [FBBroadcastManager.sharedInstance stopListening]; [self stopScreenshotsBroadcaster]; if (self.server.isRunning) { [self.server stop:NO]; diff --git a/WebDriverAgentLib/Utilities/FBBroadcastControlServer.h b/WebDriverAgentLib/Utilities/FBBroadcastControlServer.h new file mode 100644 index 000000000..7d2635dc3 --- /dev/null +++ b/WebDriverAgentLib/Utilities/FBBroadcastControlServer.h @@ -0,0 +1,84 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import + +#import "FBBroadcastProtocol.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + Callbacks are invoked on the server's internal serial queue. + */ +@protocol FBBroadcastControlServerDelegate + +/** The extension connected and sent its HELLO message. */ +- (void)broadcastServerDidConnect:(NSDictionary *)helloInfo; + +/** A periodic heartbeat with the extension's stats. */ +- (void)broadcastServerDidReceiveHeartbeat:(NSDictionary *)heartbeat; + +/** A broadcast lifecycle event (paused/resumed/finishing). */ +- (void)broadcastServerDidReceiveStatus:(NSDictionary *)status; + +/** The extension could not serve the given session (e.g. its encoder failed). */ +- (void)broadcastServerDidReceiveSessionError:(NSString *)message forSession:(uint32_t)sessionId; + +/** Fresh Annex-B parameter sets for the given session. */ +- (void)broadcastServerDidReceiveParameterSets:(NSData *)parameterSets forSession:(uint32_t)sessionId; + +/** An encoded picture for the given session. */ +- (void)broadcastServerDidReceiveFrame:(NSData *)annexBPictureData + isKeyFrame:(BOOL)isKeyFrame + ptsUs:(uint64_t)ptsUs + orientation:(uint8_t)orientation + forSession:(uint32_t)sessionId; + +/** The extension disconnected or went silent (watchdog). */ +- (void)broadcastServerDidDisconnect; + +@end + +/** + The WDA endpoint of the broadcast-extension control connection: listens on the loopback + control port, accepts a single extension connection at a time and speaks the + FBBroadcastProtocol wire format in both directions. The existing FBTCPSocket cannot be used + here because its delegate API discards the received bytes. + */ +@interface FBBroadcastControlServer : NSObject + +@property (nonatomic, weak) id delegate; + +/** YES while an extension connection is established (HELLO received, watchdog content). */ +@property (atomic, readonly) BOOL isExtensionConnected; + +- (instancetype)initWithPort:(uint16_t)port; + +- (instancetype)init NS_UNAVAILABLE; + +/** Starts listening on 127.0.0.1:port. */ +- (BOOL)startWithError:(NSError **)error; + +/** Stops listening and drops any extension connection. */ +- (void)stop; + +/** Sends SESSION_ADD for the given session id with a JSON configuration payload. */ +- (void)sendSessionAdd:(uint32_t)sessionId configuration:(NSDictionary *)configuration; + +/** Sends SESSION_REMOVE for the given session id. */ +- (void)sendSessionRemove:(uint32_t)sessionId; + +/** Sends KEYFRAME_REQUEST for the given session id. */ +- (void)sendKeyframeRequest:(uint32_t)sessionId; + +/** Asks the extension to finish the broadcast. */ +- (void)sendStopBroadcast; + +@end + +NS_ASSUME_NONNULL_END diff --git a/WebDriverAgentLib/Utilities/FBBroadcastControlServer.m b/WebDriverAgentLib/Utilities/FBBroadcastControlServer.m new file mode 100644 index 000000000..95d6a6783 --- /dev/null +++ b/WebDriverAgentLib/Utilities/FBBroadcastControlServer.m @@ -0,0 +1,274 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import "FBBroadcastControlServer.h" + +#include + +#import "FBLogger.h" +#import "GCDAsyncSocket.h" + +static const long TAG_HEADER = 1; +static const long TAG_PAYLOAD = 2; + +// The extension heartbeats every 2s; treat the broadcast as dead after 6s of silence. +static const NSTimeInterval WATCHDOG_INTERVAL = 3.0; +static const NSTimeInterval STALENESS_TIMEOUT = 6.0; + +@interface FBBroadcastControlServer () + +@property (nonatomic, readonly) uint16_t port; +@property (nonatomic) dispatch_queue_t queue; +@property (nonatomic, nullable) GCDAsyncSocket *listenSocket; +@property (nonatomic, nullable) GCDAsyncSocket *extensionSocket; +@property (nonatomic, nullable) dispatch_source_t watchdogTimer; +@property (atomic, readwrite) BOOL isExtensionConnected; +@property (nonatomic) uint64_t lastMessageAtMs; +@property (nonatomic) FBBroadcastMessageHeader pendingHeader; + +@end + +@implementation FBBroadcastControlServer + +- (instancetype)initWithPort:(uint16_t)port +{ + if ((self = [super init])) { + _port = port; + _queue = dispatch_queue_create("wda.broadcast.control", DISPATCH_QUEUE_SERIAL); + } + return self; +} + +- (BOOL)startWithError:(NSError **)error +{ + self.listenSocket = [[GCDAsyncSocket alloc] initWithDelegate:self delegateQueue:self.queue]; + // Loopback only: the control port must never be reachable from the network. + if (![self.listenSocket acceptOnInterface:@"127.0.0.1" port:self.port error:error]) { + self.listenSocket = nil; + return NO; + } + [FBLogger logFmt:@"Broadcast control server is listening on 127.0.0.1:%d", self.port]; + return YES; +} + +- (void)stop +{ + dispatch_sync(self.queue, ^{ + [self stopWatchdog]; + [self dropExtensionSocketLocked]; + self.listenSocket.delegate = nil; + [self.listenSocket disconnect]; + self.listenSocket = nil; + }); +} + +#pragma mark - Outgoing messages + +- (void)sendMessage:(nullable NSData *)message +{ + if (nil == message) { + return; + } + dispatch_async(self.queue, ^{ + [self.extensionSocket writeData:(NSData *)message withTimeout:-1 tag:0]; + }); +} + +- (void)sendSessionAdd:(uint32_t)sessionId configuration:(NSDictionary *)configuration +{ + [self sendMessage:FBBroadcastEncodeJSONMessage(FBBroadcastMessageTypeSessionAdd, sessionId, configuration)]; +} + +- (void)sendSessionRemove:(uint32_t)sessionId +{ + [self sendMessage:FBBroadcastEncodeMessage(FBBroadcastMessageTypeSessionRemove, sessionId, nil)]; +} + +- (void)sendKeyframeRequest:(uint32_t)sessionId +{ + [self sendMessage:FBBroadcastEncodeMessage(FBBroadcastMessageTypeKeyframeRequest, sessionId, nil)]; +} + +- (void)sendStopBroadcast +{ + [self sendMessage:FBBroadcastEncodeMessage(FBBroadcastMessageTypeStopBroadcast, 0, nil)]; +} + +#pragma mark - Watchdog + +- (void)startWatchdog +{ + [self stopWatchdog]; + dispatch_source_t timer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, self.queue); + if (nil == timer) { + return; + } + dispatch_source_set_timer(timer, + dispatch_time(DISPATCH_TIME_NOW, (int64_t)(WATCHDOG_INTERVAL * NSEC_PER_SEC)), + (uint64_t)(WATCHDOG_INTERVAL * NSEC_PER_SEC), + (uint64_t)(0.5 * NSEC_PER_SEC)); + __weak typeof(self) weakSelf = self; + dispatch_source_set_event_handler(timer, ^{ + [weakSelf checkStaleness]; + }); + dispatch_resume(timer); + self.watchdogTimer = timer; +} + +- (void)stopWatchdog +{ + dispatch_source_t timer = self.watchdogTimer; + if (nil != timer) { + dispatch_source_cancel(timer); + self.watchdogTimer = nil; + } +} + +- (void)checkStaleness +{ + if (nil == self.extensionSocket) { + return; + } + uint64_t nowMs = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) / NSEC_PER_MSEC; + if (nowMs - self.lastMessageAtMs > (uint64_t)(STALENESS_TIMEOUT * 1000)) { + [FBLogger log:@"The broadcast extension went silent; dropping the connection"]; + [self dropExtensionSocketLocked]; + [self.delegate broadcastServerDidDisconnect]; + } +} + +- (void)dropExtensionSocketLocked +{ + if (nil == self.extensionSocket) { + return; + } + GCDAsyncSocket *socket = self.extensionSocket; + self.extensionSocket = nil; + self.isExtensionConnected = NO; + socket.delegate = nil; + [socket disconnect]; + [self stopWatchdog]; +} + +#pragma mark - Incoming messages + +- (void)handleMessageWithHeader:(FBBroadcastMessageHeader)header payload:(NSData *)payload +{ + id delegate = self.delegate; + switch (header.type) { + case FBBroadcastMessageTypeHello: { + NSDictionary *helloInfo = FBBroadcastParseJSONPayload(payload) ?: @{}; + self.isExtensionConnected = YES; + [delegate broadcastServerDidConnect:helloInfo]; + return; + } + case FBBroadcastMessageTypeHeartbeat: { + NSDictionary *heartbeat = FBBroadcastParseJSONPayload(payload); + if (nil != heartbeat) { + [delegate broadcastServerDidReceiveHeartbeat:heartbeat]; + } + return; + } + case FBBroadcastMessageTypeStatus: { + NSDictionary *status = FBBroadcastParseJSONPayload(payload); + if (nil != status) { + [delegate broadcastServerDidReceiveStatus:status]; + } + return; + } + case FBBroadcastMessageTypeSessionError: { + NSDictionary *info = FBBroadcastParseJSONPayload(payload); + NSString *message = [info[FBBroadcastKeyMessage] isKindOfClass:NSString.class] + ? (NSString *)info[FBBroadcastKeyMessage] + : @"The extension reported a session failure"; + [delegate broadcastServerDidReceiveSessionError:message forSession:header.sessionId]; + return; + } + case FBBroadcastMessageTypeVideoParams: + [delegate broadcastServerDidReceiveParameterSets:payload forSession:header.sessionId]; + return; + case FBBroadcastMessageTypeVideoFrame: { + uint64_t ptsUs = 0; + BOOL isKeyFrame = NO; + uint8_t orientation = 0; + NSData *annexB = nil; + if (!FBBroadcastParseVideoFramePayload(payload, &ptsUs, &isKeyFrame, &orientation, &annexB)) { + [FBLogger logFmt:@"Session %u: malformed VIDEO_FRAME payload", header.sessionId]; + return; + } + [delegate broadcastServerDidReceiveFrame:(NSData *)annexB + isKeyFrame:isKeyFrame + ptsUs:ptsUs + orientation:orientation + forSession:header.sessionId]; + return; + } + default: + [FBLogger logFmt:@"Ignoring an unexpected broadcast control message of type 0x%02x", header.type]; + return; + } +} + +#pragma mark - + +- (void)socket:(GCDAsyncSocket *)sock didAcceptNewSocket:(GCDAsyncSocket *)newSocket +{ + // A reconnecting extension supersedes the previous connection. + if (nil != self.extensionSocket) { + [FBLogger log:@"A new broadcast extension connection replaces the previous one"]; + [self dropExtensionSocketLocked]; + [self.delegate broadcastServerDidDisconnect]; + } + [FBLogger logFmt:@"The broadcast extension connected from %@:%d", newSocket.connectedHost, newSocket.connectedPort]; + self.extensionSocket = newSocket; + self.lastMessageAtMs = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) / NSEC_PER_MSEC; + [self startWatchdog]; + [newSocket readDataToLength:FBBroadcastHeaderLength withTimeout:-1 tag:TAG_HEADER]; +} + +- (void)socket:(GCDAsyncSocket *)sock didReadData:(NSData *)data withTag:(long)tag +{ + if (sock != self.extensionSocket) { + return; + } + self.lastMessageAtMs = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) / NSEC_PER_MSEC; + + if (tag == TAG_HEADER) { + FBBroadcastMessageHeader header; + if (!FBBroadcastParseHeader(data, &header)) { + [FBLogger log:@"Malformed broadcast control message header; dropping the connection"]; + [self dropExtensionSocketLocked]; + [self.delegate broadcastServerDidDisconnect]; + return; + } + self.pendingHeader = header; + if (header.payloadLength == 0) { + [self handleMessageWithHeader:header payload:NSData.data]; + [sock readDataToLength:FBBroadcastHeaderLength withTimeout:-1 tag:TAG_HEADER]; + } else { + [sock readDataToLength:header.payloadLength withTimeout:-1 tag:TAG_PAYLOAD]; + } + return; + } + [self handleMessageWithHeader:self.pendingHeader payload:data]; + [sock readDataToLength:FBBroadcastHeaderLength withTimeout:-1 tag:TAG_HEADER]; +} + +- (void)socketDidDisconnect:(GCDAsyncSocket *)sock withError:(nullable NSError *)error +{ + if (sock != self.extensionSocket) { + return; + } + [FBLogger logFmt:@"The broadcast extension disconnected: %@", error.description ?: @"closed"]; + self.extensionSocket = nil; + self.isExtensionConnected = NO; + [self stopWatchdog]; + [self.delegate broadcastServerDidDisconnect]; +} + +@end diff --git a/WebDriverAgentLib/Utilities/FBBroadcastManager.h b/WebDriverAgentLib/Utilities/FBBroadcastManager.h new file mode 100644 index 000000000..99e65733c --- /dev/null +++ b/WebDriverAgentLib/Utilities/FBBroadcastManager.h @@ -0,0 +1,86 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import + +#import "FBVideoStreamSession.h" + +NS_ASSUME_NONNULL_BEGIN + +extern NSErrorDomain const FBBroadcastManagerErrorDomain; + +typedef NS_ERROR_ENUM(FBBroadcastManagerErrorDomain, FBBroadcastManagerError) { + /** ReplayKit broadcasts are not available in this environment (Simulator/tvOS). */ + FBBroadcastManagerErrorUnsupported = 1, + /** The broadcast did not reach the connected state within the allotted time. */ + FBBroadcastManagerErrorTimeout = 2, + /** The system broadcast picker could not be driven. */ + FBBroadcastManagerErrorPicker = 3, +}; + +/** + Coordinates the ReplayKit broadcast extension: owns the loopback control server the extension + connects to, drives the system broadcast picker UI to start a broadcast, and routes the + extension's pre-encoded frames into the matching FBVideoStreamSession instances. + + The control server listens permanently (started from FBWebServer), so broadcasts started + manually from Control Center attach exactly like ones started via the HTTP endpoint. + */ +@interface FBBroadcastManager : NSObject + ++ (instancetype)sharedInstance; + +/** YES while the broadcast extension is connected to the control server. */ +@property (nonatomic, readonly) BOOL isExtensionConnected; + +/** Starts the loopback control server. Safe to call multiple times. */ +- (void)startListening; + +/** Stops the control server and drops the extension connection. */ +- (void)stopListening; + +/** @return A dictionary describing the broadcast state for the status endpoint. */ +- (NSDictionary *)statusDictionary; + +/** + Starts a system broadcast targeting the bundled extension by foregrounding the runner app, + triggering the broadcast picker and confirming the system sheet via UI automation, then waits + for the extension to connect. Must be called on the main thread. Idempotent while connected. + + @param timeout The overall time budget in seconds for the broadcast to reach the connected state + @param confirmButtonLabels Labels to look for on the system confirmation sheet + @param restoreForegroundApp YES to re-activate the previously active application afterwards + @param error If there is an error, upon return contains an NSError describing the problem + @return NO in case of a failure + */ +- (BOOL)startBroadcastWithTimeout:(NSTimeInterval)timeout + confirmButtonLabels:(NSArray *)confirmButtonLabels + restoreForegroundApp:(BOOL)restoreForegroundApp + error:(NSError **)error; + +/** + Asks the extension to finish the broadcast and waits for it to disconnect. + Idempotent when no broadcast is running. + + @param error If there is an error, upon return contains an NSError describing the problem + @return NO in case of a failure + */ +- (BOOL)stopBroadcastWithError:(NSError **)error; + +/** Notifies the manager that a capture session started (sends SESSION_ADD when connected). */ +- (void)notifySessionAdded:(FBVideoStreamSession *)session; + +/** Notifies the manager that a capture session stopped (sends SESSION_REMOVE when connected). */ +- (void)notifySessionRemoved:(NSUInteger)identifier; + +/** Forwards a key frame request for the given session to the extension. */ +- (void)requestKeyFrameForSession:(NSUInteger)identifier; + +@end + +NS_ASSUME_NONNULL_END diff --git a/WebDriverAgentLib/Utilities/FBBroadcastManager.m b/WebDriverAgentLib/Utilities/FBBroadcastManager.m new file mode 100644 index 000000000..47e304995 --- /dev/null +++ b/WebDriverAgentLib/Utilities/FBBroadcastManager.m @@ -0,0 +1,485 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import "FBBroadcastManager.h" + +#include +#import + +#import "FBBroadcastControlServer.h" +#import "FBBroadcastPickerHost.h" +#import "FBBroadcastProtocol.h" +#import "FBConfiguration.h" +#import "FBLogger.h" +#import "FBRunLoopSpinner.h" +#import "FBScreen.h" +#import "FBUnattachedAppLauncher.h" +#import "FBVideoStreamManager.h" +#import "XCUIApplication+FBTouchAction.h" +#import "XCUIApplication.h" +#import "XCUIApplication+FBHelpers.h" + +NSErrorDomain const FBBroadcastManagerErrorDomain = @"com.facebook.WebDriverAgent.FBBroadcastManager"; + +#if !TARGET_OS_SIMULATOR && !TARGET_OS_TV +static const NSTimeInterval FOREGROUND_TIMEOUT = 5.0; +static const NSTimeInterval CONFIRM_BUTTON_TIMEOUT = 10.0; +// The picker press is dropped silently by the system when it fires before the scene is fully +// active, so it is re-fired periodically until the confirmation sheet shows up. +static const uint64_t PICKER_RETRIGGER_INTERVAL_MS = 2000; + +static uint64_t FBBroadcastNowMs(void) +{ + return clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) / NSEC_PER_MSEC; +} +#endif +static const NSTimeInterval STOP_TIMEOUT = 5.0; + +@interface FBBroadcastManager () + +@property (nonatomic, nullable) FBBroadcastControlServer *controlServer; +@property (atomic, nullable, copy) NSDictionary *helloInfo; +@property (atomic, nullable, copy) NSDictionary *lastHeartbeat; +@property (atomic, nullable) NSDate *connectedAt; +@property (atomic, nullable) NSDate *lastHeartbeatAt; +@property (atomic) BOOL paused; +/** YES while a start dance is driving the system UI (used to serialize concurrent starts). */ +@property (atomic) BOOL startInProgress; + +#if !TARGET_OS_SIMULATOR && !TARGET_OS_TV +- (BOOL)performBroadcastStartWithTimeout:(NSTimeInterval)timeout + confirmButtonLabels:(NSArray *)confirmButtonLabels + restoreForegroundApp:(BOOL)restoreForegroundApp + error:(NSError **)error; +#endif + +@end + +@implementation FBBroadcastManager + ++ (instancetype)sharedInstance +{ + static FBBroadcastManager *instance; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + instance = [[self alloc] init]; + }); + return instance; +} + +- (BOOL)isExtensionConnected +{ + return self.controlServer.isExtensionConnected; +} + +#pragma mark - Control server lifecycle + +- (void)startListening +{ + if (nil != self.controlServer) { + return; + } + uint16_t port = (uint16_t)FBConfiguration.broadcastControlPort; + FBBroadcastControlServer *server = [[FBBroadcastControlServer alloc] initWithPort:port]; + server.delegate = self; + NSError *error; + if (![server startWithError:&error]) { + [FBLogger logFmt:@"Cannot start the broadcast control server on port %d: %@", port, error.description]; + return; + } + self.controlServer = server; +} + +- (void)stopListening +{ + [self.controlServer stop]; + self.controlServer = nil; + [self resetConnectionState]; +} + +- (void)resetConnectionState +{ + self.helloInfo = nil; + self.lastHeartbeat = nil; + self.connectedAt = nil; + self.lastHeartbeatAt = nil; + self.paused = NO; +} + +#pragma mark - Status + +- (NSDictionary *)statusDictionary +{ + NSString *state = @"idle"; + if (self.isExtensionConnected) { + state = self.paused ? @"paused" : @"connected"; + } + NSDictionary *heartbeat = self.lastHeartbeat; + return @{ + @"state": state, + @"controlPort": @(FBConfiguration.broadcastControlPort), + @"preferredExtension": FBConfiguration.broadcastExtensionBundleId, + @"connectedAt": self.connectedAt ? @((uint64_t)(self.connectedAt.timeIntervalSince1970 * 1000)) : NSNull.null, + @"lastHeartbeatAt": self.lastHeartbeatAt ? @((uint64_t)(self.lastHeartbeatAt.timeIntervalSince1970 * 1000)) : NSNull.null, + @"hello": self.helloInfo ?: NSNull.null, + @"heartbeat": heartbeat ?: NSNull.null, + @"sessions": [FBVideoStreamManager.sharedInstance activeSessionsInfo], + }; +} + +#pragma mark - Broadcast start/stop + +- (BOOL)startBroadcastWithTimeout:(NSTimeInterval)timeout + confirmButtonLabels:(NSArray *)confirmButtonLabels + restoreForegroundApp:(BOOL)restoreForegroundApp + error:(NSError **)error +{ +#if TARGET_OS_SIMULATOR || TARGET_OS_TV + if (error) { + *error = [NSError errorWithDomain:FBBroadcastManagerErrorDomain + code:FBBroadcastManagerErrorUnsupported + userInfo:@{NSLocalizedDescriptionKey: @"ReplayKit broadcasts are only supported on physical iOS devices"}]; + } + return NO; +#else + if (self.isExtensionConnected) { + return YES; + } + if (nil == self.controlServer) { + [self startListening]; + } + + // Serialize concurrent starts: the dance below spins the main run loop, so another + // /broadcast/start request can be dispatched re-entrantly while the first one is still driving + // the system UI. Starting a second broadcast while one is launching makes iOS kill both, so + // followers just await the leader's outcome. + if (self.startInProgress) { + [FBLogger log:@"broadcast/start: another start attempt is already in progress; awaiting its outcome"]; + [[[[FBRunLoopSpinner new] timeout:(timeout > 0 ? timeout : 30.0)] interval:0.3] spinUntilTrue:^BOOL{ + return self.isExtensionConnected || !self.startInProgress; + }]; + if (self.isExtensionConnected) { + return YES; + } + if (error) { + *error = [NSError errorWithDomain:FBBroadcastManagerErrorDomain + code:FBBroadcastManagerErrorTimeout + userInfo:@{NSLocalizedDescriptionKey: @"A concurrent broadcast start attempt finished without the extension connecting"}]; + } + return NO; + } + + self.startInProgress = YES; + @try { + return [self performBroadcastStartWithTimeout:timeout + confirmButtonLabels:confirmButtonLabels + restoreForegroundApp:restoreForegroundApp + error:error]; + } @finally { + self.startInProgress = NO; + } +#endif +} + +#if !TARGET_OS_SIMULATOR && !TARGET_OS_TV +- (BOOL)performBroadcastStartWithTimeout:(NSTimeInterval)timeout + confirmButtonLabels:(NSArray *)confirmButtonLabels + restoreForegroundApp:(BOOL)restoreForegroundApp + error:(NSError **)error +{ + // The screen may already be captured by a live broadcast even though the extension is not + // connected (it crashed, or it is between TCP reconnect attempts). Driving the picker on top + // of a live broadcast makes iOS kill both, so wait for the extension instead. + if (UIScreen.mainScreen.isCaptured) { + [FBLogger log:@"broadcast/start: the screen is already being captured; waiting for the extension to connect instead of starting another broadcast"]; + [[[[FBRunLoopSpinner new] timeout:5.0] interval:0.2] spinUntilTrue:^BOOL{ + return self.isExtensionConnected || !UIScreen.mainScreen.isCaptured; + }]; + if (self.isExtensionConnected) { + return YES; + } + if (UIScreen.mainScreen.isCaptured) { + if (error) { + *error = [NSError errorWithDomain:FBBroadcastManagerErrorDomain + code:FBBroadcastManagerErrorTimeout + userInfo:@{NSLocalizedDescriptionKey: @"The screen is already being captured (an active broadcast or recording), but the WebDriverAgent broadcast extension did not connect. Stop the existing capture (e.g. via the status bar pill) and retry"}]; + } + return NO; + } + // The capture ended while waiting; fall through and start a fresh broadcast. + } + + uint64_t startedMs = FBBroadcastNowMs(); + XCUIApplication *runner = [[XCUIApplication alloc] initWithBundleIdentifier:(NSString *)NSBundle.mainBundle.bundleIdentifier]; + XCUIApplication *previousApp = nil; + BOOL runnerIsActive = UIApplication.sharedApplication.applicationState == UIApplicationStateActive; + // When the runner is already frontmost there is neither an app to restore nor a need for the + // (slow) active-app lookup. + if (restoreForegroundApp && !runnerIsActive) { + XCUIApplication *active = XCUIApplication.fb_activeApplication; + if (nil != active && ![active.bundleID isEqualToString:runner.bundleID]) { + previousApp = active; + } + [FBLogger logFmt:@"broadcast/start: active-app lookup finished after %llums", FBBroadcastNowMs() - startedMs]; + } + + // The picker can only present from a foreground app, so bring the runner up first. The + // LaunchServices route is used instead of XCUIApplication.activate because activating the + // runner from inside itself blocks on a self-quiescence wait that can only ever time out: + // the waiting thread is the very main thread whose idleness is being awaited. + if (!runnerIsActive) { + BOOL launched = [FBUnattachedAppLauncher launchAppWithBundleId:(NSString *)NSBundle.mainBundle.bundleIdentifier]; + BOOL foregrounded = launched && [[[[FBRunLoopSpinner new] timeout:2.0] interval:0.05] spinUntilTrue:^BOOL{ + return UIApplication.sharedApplication.applicationState == UIApplicationStateActive; + }]; + if (!foregrounded) { + // Reliable but slow fallback: XCTest's activation waits out its quiescence timeout. + [FBLogger log:@"broadcast/start: LaunchServices foregrounding failed; falling back to XCUIApplication.activate"]; + [runner activate]; + foregrounded = [[[[FBRunLoopSpinner new] timeout:FOREGROUND_TIMEOUT] interval:0.05] spinUntilTrue:^BOOL{ + return UIApplication.sharedApplication.applicationState == UIApplicationStateActive; + }]; + } + if (!foregrounded) { + if (error) { + *error = [NSError errorWithDomain:FBBroadcastManagerErrorDomain + code:FBBroadcastManagerErrorTimeout + userInfo:@{NSLocalizedDescriptionKey: @"The runner app could not be brought to the foreground to present the broadcast picker"}]; + } + return NO; + } + } + [FBLogger logFmt:@"broadcast/start: runner foreground after %llums", FBBroadcastNowMs() - startedMs]; + + NSError *pickerError; + if (![FBBroadcastPickerHost triggerPickerWithPreferredExtension:FBConfiguration.broadcastExtensionBundleId + error:&pickerError]) { + if (error) { + *error = [NSError errorWithDomain:FBBroadcastManagerErrorDomain + code:FBBroadcastManagerErrorPicker + userInfo:@{NSLocalizedDescriptionKey: pickerError.localizedDescription ?: @"Cannot trigger the broadcast picker"}]; + } + return NO; + } + [FBLogger logFmt:@"broadcast/start: picker triggered after %llums", FBBroadcastNowMs() - startedMs]; + + // The confirmation sheet is hosted by different processes depending on the iOS version, so + // look for the confirm button in both the system app and the runner itself. + NSArray *labels = confirmButtonLabels.count > 0 ? confirmButtonLabels : @[@"Start Broadcast"]; + NSArray *candidateApps = @[XCUIApplication.fb_systemApplication, runner]; + __block BOOL confirmButtonFound = NO; + __block CGRect confirmFrame = CGRectZero; + __block uint64_t lastTriggerMs = FBBroadcastNowMs(); + [[[[FBRunLoopSpinner new] timeout:CONFIRM_BUTTON_TIMEOUT] interval:0.25] spinUntilTrue:^BOOL{ + for (XCUIApplication *app in candidateApps) { + for (NSString *label in labels) { + XCUIElement *candidate = app.buttons[label]; + if (candidate.exists) { + confirmFrame = candidate.frame; + confirmButtonFound = YES; + return YES; + } + } + XCUIElement *prefixMatch = [app.buttons matchingPredicate:[NSPredicate predicateWithFormat:@"label BEGINSWITH[c] 'Start'"]].firstMatch; + if (prefixMatch.exists) { + confirmFrame = prefixMatch.frame; + confirmButtonFound = YES; + return YES; + } + } + if (FBBroadcastNowMs() - lastTriggerMs >= PICKER_RETRIGGER_INTERVAL_MS) { + lastTriggerMs = FBBroadcastNowMs(); + [FBLogger log:@"broadcast/start: confirmation sheet not visible yet; re-triggering the picker"]; + [FBBroadcastPickerHost triggerPickerWithPreferredExtension:FBConfiguration.broadcastExtensionBundleId + error:nil]; + } + return NO; + }]; + if (!confirmButtonFound || CGRectIsEmpty(confirmFrame)) { + [FBBroadcastPickerHost dismiss]; + if (error) { + *error = [NSError errorWithDomain:FBBroadcastManagerErrorDomain + code:FBBroadcastManagerErrorTimeout + userInfo:@{NSLocalizedDescriptionKey: [NSString stringWithFormat:@"The broadcast confirmation sheet did not show a button labeled %@ within %.0fs. Pass 'confirmButtonLabels' if the device language is not English", [labels componentsJoinedByString:@"/"], CONFIRM_BUTTON_TIMEOUT]}]; + } + return NO; + } + // Tap via WDA's own event synthesis instead of XCUIElement.tap: a missed XCUIElement tap + // (e.g. the sheet dismissed in between) records an XCTest failure that tears down the whole + // test session, whereas a missed synthesized tap is harmless and surfaces as a connect timeout. + CGFloat scale = (CGFloat)[FBScreen scale]; + CGPoint center = CGPointMake(CGRectGetMidX(confirmFrame) * scale, CGRectGetMidY(confirmFrame) * scale); + NSArray *tapActions = @[ + @{@"type": @"pointerDown", @"x": @(center.x), @"y": @(center.y)}, + @{@"type": @"pause", @"duration": @60}, + @{@"type": @"pointerUp", @"x": @(center.x), @"y": @(center.y)}, + ]; + NSError *tapError; + if (![runner fb_performMobilerunActions:tapActions scale:scale error:&tapError]) { + [FBBroadcastPickerHost dismiss]; + if (error) { + *error = [NSError errorWithDomain:FBBroadcastManagerErrorDomain + code:FBBroadcastManagerErrorPicker + userInfo:@{NSLocalizedDescriptionKey: [NSString stringWithFormat:@"Cannot tap the broadcast confirmation button: %@", tapError.localizedDescription]}]; + } + return NO; + } + [FBLogger logFmt:@"broadcast/start: confirmation tapped after %llums", FBBroadcastNowMs() - startedMs]; + + // Cover the system's 3-2-1 countdown plus the extension's connect/HELLO round trip. + BOOL connected = [[[[FBRunLoopSpinner new] timeout:(timeout > 0 ? timeout : 30.0)] interval:0.3] spinUntilTrue:^BOOL{ + return self.isExtensionConnected; + }]; + [FBLogger logFmt:@"broadcast/start: %@ after %llums", connected ? @"extension connected" : @"extension connect timeout", FBBroadcastNowMs() - startedMs]; + [FBBroadcastPickerHost dismiss]; + if (!connected) { + if (error) { + *error = [NSError errorWithDomain:FBBroadcastManagerErrorDomain + code:FBBroadcastManagerErrorTimeout + userInfo:@{NSLocalizedDescriptionKey: @"The broadcast was confirmed but the extension did not connect in time. Check that the extension is embedded and signed correctly (see docs/broadcast-extension.md)"}]; + } + return NO; + } + + if (nil != previousApp) { + [previousApp activate]; + } + return YES; +} +#endif + +- (BOOL)stopBroadcastWithError:(NSError **)error +{ + if (!self.isExtensionConnected) { + return YES; + } + [self.controlServer sendStopBroadcast]; + BOOL stopped = [[[[FBRunLoopSpinner new] timeout:STOP_TIMEOUT] interval:0.2] spinUntilTrue:^BOOL{ + return !self.isExtensionConnected; + }]; + if (!stopped) { + if (error) { + *error = [NSError errorWithDomain:FBBroadcastManagerErrorDomain + code:FBBroadcastManagerErrorTimeout + userInfo:@{NSLocalizedDescriptionKey: @"The broadcast extension did not finish the broadcast in time"}]; + } + return NO; + } + return YES; +} + +#pragma mark - Session bridging + ++ (NSDictionary *)sessionAddPayloadForConfiguration:(FBScreenCaptureConfiguration *)configuration +{ + return @{ + FBBroadcastKeyWidth: @(configuration.width), + FBBroadcastKeyHeight: @(configuration.height), + FBBroadcastKeyCodec: configuration.codec == FBVideoCodecH265 ? FBBroadcastCodecH265 : FBBroadcastCodecH264, + FBBroadcastKeyBitrate: @(configuration.bitrate), + FBBroadcastKeyFps: @(configuration.fps), + }; +} + +- (void)notifySessionAdded:(FBVideoStreamSession *)session +{ + if (!self.isExtensionConnected) { + return; + } + [self.controlServer sendSessionAdd:(uint32_t)session.identifier + configuration:[self.class sessionAddPayloadForConfiguration:session.configuration]]; +} + +- (void)notifySessionRemoved:(NSUInteger)identifier +{ + if (!self.isExtensionConnected) { + return; + } + [self.controlServer sendSessionRemove:(uint32_t)identifier]; +} + +- (void)requestKeyFrameForSession:(NSUInteger)identifier +{ + [self.controlServer sendKeyframeRequest:(uint32_t)identifier]; +} + +#pragma mark - + +- (void)broadcastServerDidConnect:(NSDictionary *)helloInfo +{ + [FBLogger logFmt:@"The broadcast extension connected: %@", helloInfo]; + self.helloInfo = helloInfo; + self.connectedAt = NSDate.date; + self.paused = NO; + // Attach every live capture session to the broadcast source. + for (FBVideoStreamSession *session in [FBVideoStreamManager.sharedInstance activeSessions]) { + [self.controlServer sendSessionAdd:(uint32_t)session.identifier + configuration:[self.class sessionAddPayloadForConfiguration:session.configuration]]; + } +} + +- (void)broadcastServerDidReceiveHeartbeat:(NSDictionary *)heartbeat +{ + self.lastHeartbeat = heartbeat; + self.lastHeartbeatAt = NSDate.date; + self.paused = [@"paused" isEqualToString:(NSString *)(heartbeat[FBBroadcastKeyState] ?: @"")]; +} + +- (void)broadcastServerDidReceiveStatus:(NSDictionary *)status +{ + NSString *event = status[FBBroadcastKeyEvent]; + [FBLogger logFmt:@"Broadcast status event: %@", status]; + if ([@"paused" isEqualToString:event]) { + self.paused = YES; + } else if ([@"resumed" isEqualToString:event]) { + self.paused = NO; + } +} + +- (void)broadcastServerDidReceiveSessionError:(NSString *)message forSession:(uint32_t)sessionId +{ + [FBLogger logFmt:@"The broadcast extension cannot serve session %u: %@", sessionId, message]; + FBVideoStreamSession *session = [FBVideoStreamManager.sharedInstance sessionWithIdentifier:sessionId]; + [session detachBroadcastSourceAndForceKeyFrame]; +} + +- (void)broadcastServerDidReceiveParameterSets:(NSData *)parameterSets forSession:(uint32_t)sessionId +{ + FBVideoStreamSession *session = [FBVideoStreamManager.sharedInstance sessionWithIdentifier:sessionId]; + if (nil == session) { + [self.controlServer sendSessionRemove:sessionId]; + return; + } + [session ingestBroadcastParameterSets:parameterSets]; +} + +- (void)broadcastServerDidReceiveFrame:(NSData *)annexBPictureData + isKeyFrame:(BOOL)isKeyFrame + ptsUs:(uint64_t)ptsUs + orientation:(uint8_t)orientation + forSession:(uint32_t)sessionId +{ + FBVideoStreamSession *session = [FBVideoStreamManager.sharedInstance sessionWithIdentifier:sessionId]; + if (nil == session) { + // The session is gone (stale extension pipeline); ask the extension to drop it. + [self.controlServer sendSessionRemove:sessionId]; + return; + } + [session ingestBroadcastFrame:annexBPictureData isKeyFrame:isKeyFrame]; +} + +- (void)broadcastServerDidDisconnect +{ + [FBLogger log:@"The broadcast extension disconnected; reverting sessions to the screenshot source"]; + [self resetConnectionState]; + for (FBVideoStreamSession *session in [FBVideoStreamManager.sharedInstance activeSessions]) { + [session detachBroadcastSourceAndForceKeyFrame]; + } +} + +@end diff --git a/WebDriverAgentLib/Utilities/FBBroadcastPickerHost.h b/WebDriverAgentLib/Utilities/FBBroadcastPickerHost.h new file mode 100644 index 000000000..837bbddc9 --- /dev/null +++ b/WebDriverAgentLib/Utilities/FBBroadcastPickerHost.h @@ -0,0 +1,35 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import + +NS_ASSUME_NONNULL_BEGIN + +/** + Hosts an (effectively invisible) RPSystemBroadcastPickerView inside the runner app and + triggers its button programmatically, which makes the system present the broadcast + confirmation sheet. Main thread only. + */ +@interface FBBroadcastPickerHost : NSObject + +/** + Creates the hosting window if needed and taps the picker's internal button. + + @param preferredExtension The bundle identifier of the broadcast upload extension to preselect + @param error Set when the picker cannot be hosted or its button cannot be found + @return NO in case of a failure + */ ++ (BOOL)triggerPickerWithPreferredExtension:(NSString *)preferredExtension + error:(NSError **)error; + +/** Hides and releases the hosting window. */ ++ (void)dismiss; + +@end + +NS_ASSUME_NONNULL_END diff --git a/WebDriverAgentLib/Utilities/FBBroadcastPickerHost.m b/WebDriverAgentLib/Utilities/FBBroadcastPickerHost.m new file mode 100644 index 000000000..26766eaa6 --- /dev/null +++ b/WebDriverAgentLib/Utilities/FBBroadcastPickerHost.m @@ -0,0 +1,129 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import "FBBroadcastPickerHost.h" + +#import +#if !TARGET_OS_TV +#import +#endif + +#import "FBLogger.h" + +static NSString *const FBBroadcastPickerHostErrorDomain = @"com.facebook.WebDriverAgent.FBBroadcastPickerHost"; + +#if !TARGET_OS_TV +static UIWindow *pickerWindow = nil; +#endif + +@implementation FBBroadcastPickerHost + +#if TARGET_OS_TV + ++ (BOOL)triggerPickerWithPreferredExtension:(NSString *)preferredExtension + error:(NSError **)error +{ + if (error) { + *error = [NSError errorWithDomain:FBBroadcastPickerHostErrorDomain + code:1 + userInfo:@{NSLocalizedDescriptionKey: @"The ReplayKit broadcast picker is not available on tvOS"}]; + } + return NO; +} + ++ (void)dismiss +{ +} + +#else + ++ (BOOL)triggerPickerWithPreferredExtension:(NSString *)preferredExtension + error:(NSError **)error +{ + NSAssert(NSThread.isMainThread, @"The broadcast picker must be triggered on the main thread"); + + if (nil == pickerWindow) { + // The picker view must live in a visible window for the system sheet to present, but the + // window can be tiny and nearly transparent so it never interferes with the runner's UI. + UIWindowScene *foregroundScene = nil; + for (UIScene *scene in UIApplication.sharedApplication.connectedScenes) { + if ([scene isKindOfClass:UIWindowScene.class] + && scene.activationState == UISceneActivationStateForegroundActive) { + foregroundScene = (UIWindowScene *)scene; + break; + } + } + UIWindow *window = nil == foregroundScene + ? [[UIWindow alloc] initWithFrame:CGRectMake(0, 0, 2, 2)] + : [[UIWindow alloc] initWithWindowScene:foregroundScene]; + window.frame = CGRectMake(0, 0, 2, 2); + window.windowLevel = UIWindowLevelNormal; + window.alpha = 0.02; + window.rootViewController = [[UIViewController alloc] init]; + + RPSystemBroadcastPickerView *pickerView = [[RPSystemBroadcastPickerView alloc] initWithFrame:CGRectMake(0, 0, 2, 2)]; + pickerView.preferredExtension = preferredExtension; + pickerView.showsMicrophoneButton = NO; + [window.rootViewController.view addSubview:pickerView]; + + pickerWindow = window; + } + [pickerWindow makeKeyAndVisible]; + + RPSystemBroadcastPickerView *pickerView = nil; + for (UIView *subview in pickerWindow.rootViewController.view.subviews) { + if ([subview isKindOfClass:RPSystemBroadcastPickerView.class]) { + pickerView = (RPSystemBroadcastPickerView *)subview; + break; + } + } + pickerView.preferredExtension = preferredExtension; + + UIButton *button = [self findButtonInView:pickerView]; + if (nil == button) { + if (error) { + *error = [NSError errorWithDomain:FBBroadcastPickerHostErrorDomain + code:2 + userInfo:@{NSLocalizedDescriptionKey: @"Cannot locate the button inside RPSystemBroadcastPickerView. The system view layout may have changed in this iOS version"}]; + } + return NO; + } + [FBLogger log:@"Triggering the system broadcast picker"]; + [button sendActionsForControlEvents:UIControlEventTouchUpInside]; + return YES; +} + ++ (nullable UIButton *)findButtonInView:(nullable UIView *)view +{ + if (nil == view) { + return nil; + } + if ([view isKindOfClass:UIButton.class]) { + return (UIButton *)view; + } + for (UIView *subview in view.subviews) { + UIButton *button = [self findButtonInView:subview]; + if (nil != button) { + return button; + } + } + return nil; +} + ++ (void)dismiss +{ + if (nil == pickerWindow) { + return; + } + pickerWindow.hidden = YES; + pickerWindow = nil; +} + +#endif + +@end diff --git a/WebDriverAgentLib/Utilities/FBBroadcastProtocol.h b/WebDriverAgentLib/Utilities/FBBroadcastProtocol.h new file mode 100644 index 000000000..ebd2d98f8 --- /dev/null +++ b/WebDriverAgentLib/Utilities/FBBroadcastProtocol.h @@ -0,0 +1,147 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import + +NS_ASSUME_NONNULL_BEGIN + +/** + Wire protocol shared between WebDriverAgent and the ReplayKit broadcast upload extension. + + This file is compiled into both WebDriverAgentLib and the WebDriverAgentBroadcast extension + target, so it must only depend on Foundation. + + Transport: a single TCP connection on loopback. The extension is the client, WDA is the + server. Every message is a fixed 16-byte big-endian header followed by a payload: + + offset 0 uint32 magic = 'WDAB' + offset 4 uint8 protocolVersion + offset 5 uint8 messageType + offset 6 uint16 reserved (0) + offset 8 uint32 sessionId (0 when not applicable) + offset 12 uint32 payloadLength + offset 16 ... payload + + Control payloads are UTF-8 JSON; the high-rate VIDEO_FRAME payload is fixed binary + (see FBBroadcastEncodeVideoFramePayload). + */ + +/** 'WDAB' */ +extern const uint32_t FBBroadcastProtocolMagic; +extern const uint8_t FBBroadcastProtocolVersion; +/** The fixed message header length in bytes. */ +extern const NSUInteger FBBroadcastHeaderLength; +/** The loopback TCP port WDA listens on for the extension connection. */ +extern const uint16_t FBBroadcastDefaultControlPort; + +typedef NS_ENUM(uint8_t, FBBroadcastMessageType) { + // WDA -> extension + /** JSON {width, height, codec, bitrate, fps}; sessionId in the header. */ + FBBroadcastMessageTypeSessionAdd = 0x01, + /** Empty payload; sessionId in the header. */ + FBBroadcastMessageTypeSessionRemove = 0x02, + /** Empty payload; sessionId in the header. */ + FBBroadcastMessageTypeKeyframeRequest = 0x03, + /** Empty payload. Asks the extension to finish the broadcast. */ + FBBroadcastMessageTypeStopBroadcast = 0x04, + + // extension -> WDA + /** JSON {protocolVersion, osVersion}. Sent once right after connecting. */ + FBBroadcastMessageTypeHello = 0x81, + /** JSON {state, framesReceived, orientation, screenWidth, screenHeight}. Sent every 2s. */ + FBBroadcastMessageTypeHeartbeat = 0x82, + /** Raw Annex-B parameter sets (VPS/SPS/PPS) for sessionId. Sent before the IDR that uses them. */ + FBBroadcastMessageTypeVideoParams = 0x83, + /** Binary: [8B ptsUs BE][1B flags][Annex-B VCL NAL units]; sessionId in the header. */ + FBBroadcastMessageTypeVideoFrame = 0x84, + /** JSON {event: "paused"|"resumed"|"finishing", reason}. */ + FBBroadcastMessageTypeStatus = 0x85, + /** JSON {message} for sessionId (e.g. the per-session encoder could not be created). */ + FBBroadcastMessageTypeSessionError = 0x86, +}; + +/** A parsed message header. */ +typedef struct { + uint8_t version; + uint8_t type; + uint32_t sessionId; + uint32_t payloadLength; +} FBBroadcastMessageHeader; + +/** VIDEO_FRAME flags bit 0: the frame is a key (IDR) frame. */ +extern const uint8_t FBBroadcastFrameFlagKeyFrame; +/** VIDEO_FRAME flags bits 1-3: CGImagePropertyOrientation (1-8) of the captured frame. */ +extern const uint8_t FBBroadcastFrameOrientationShift; +extern const uint8_t FBBroadcastFrameOrientationMask; + +/** JSON keys for SESSION_ADD / HELLO / HEARTBEAT / STATUS payloads. */ +extern NSString *const FBBroadcastKeyWidth; +extern NSString *const FBBroadcastKeyHeight; +extern NSString *const FBBroadcastKeyCodec; +extern NSString *const FBBroadcastKeyBitrate; +extern NSString *const FBBroadcastKeyFps; +extern NSString *const FBBroadcastKeyProtocolVersion; +extern NSString *const FBBroadcastKeyOsVersion; +extern NSString *const FBBroadcastKeyState; +extern NSString *const FBBroadcastKeyFramesReceived; +extern NSString *const FBBroadcastKeyOrientation; +extern NSString *const FBBroadcastKeyScreenWidth; +extern NSString *const FBBroadcastKeyScreenHeight; +extern NSString *const FBBroadcastKeyEvent; +extern NSString *const FBBroadcastKeyReason; +extern NSString *const FBBroadcastKeyMessage; + +/** Codec string values used in SESSION_ADD, matching the HTTP API. */ +extern NSString *const FBBroadcastCodecH264; +extern NSString *const FBBroadcastCodecH265; + +/** Builds a complete wire message (header + payload). */ +NSData *FBBroadcastEncodeMessage(FBBroadcastMessageType type, + uint32_t sessionId, + NSData *_Nullable payload); + +/** Builds a complete wire message with a JSON payload. Returns nil if serialization fails. */ +NSData *_Nullable FBBroadcastEncodeJSONMessage(FBBroadcastMessageType type, + uint32_t sessionId, + NSDictionary *payload); + +/** + Parses and validates a 16-byte header. + + @return NO when the data is too short, the magic does not match or the version is unsupported. + */ +BOOL FBBroadcastParseHeader(NSData *headerData, FBBroadcastMessageHeader *outHeader); + +/** Deserializes a JSON control payload. Returns nil when the payload is not a JSON object. */ +NSDictionary *_Nullable FBBroadcastParseJSONPayload(NSData *payload); + +/** Builds a VIDEO_FRAME payload: [8B ptsUs BE][1B flags][annexB]. */ +NSData *FBBroadcastEncodeVideoFramePayload(uint64_t ptsUs, + BOOL isKeyFrame, + uint8_t orientation, + NSData *annexBPictureData); + +/** Builds a complete VIDEO_FRAME wire message without first materializing a separate payload. */ +NSData *FBBroadcastEncodeVideoFrameMessage(uint32_t sessionId, + uint64_t ptsUs, + BOOL isKeyFrame, + uint8_t orientation, + NSData *annexBPictureData); + +/** + Parses a VIDEO_FRAME payload. + + @return NO when the payload is shorter than its fixed prefix. + */ +BOOL FBBroadcastParseVideoFramePayload(NSData *payload, + uint64_t *outPtsUs, + BOOL *outIsKeyFrame, + uint8_t *outOrientation, + NSData *_Nullable __autoreleasing *_Nonnull outAnnexB); + +NS_ASSUME_NONNULL_END diff --git a/WebDriverAgentLib/Utilities/FBBroadcastProtocol.m b/WebDriverAgentLib/Utilities/FBBroadcastProtocol.m new file mode 100644 index 000000000..1f32a4842 --- /dev/null +++ b/WebDriverAgentLib/Utilities/FBBroadcastProtocol.m @@ -0,0 +1,179 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import "FBBroadcastProtocol.h" + +const uint32_t FBBroadcastProtocolMagic = 0x57444142; // 'WDAB' +const uint8_t FBBroadcastProtocolVersion = 1; +const NSUInteger FBBroadcastHeaderLength = 16; +const uint16_t FBBroadcastDefaultControlPort = 9300; + +const uint8_t FBBroadcastFrameFlagKeyFrame = 1 << 0; +const uint8_t FBBroadcastFrameOrientationShift = 1; +const uint8_t FBBroadcastFrameOrientationMask = 0x07; + +NSString *const FBBroadcastKeyWidth = @"width"; +NSString *const FBBroadcastKeyHeight = @"height"; +NSString *const FBBroadcastKeyCodec = @"codec"; +NSString *const FBBroadcastKeyBitrate = @"bitrate"; +NSString *const FBBroadcastKeyFps = @"fps"; +NSString *const FBBroadcastKeyProtocolVersion = @"protocolVersion"; +NSString *const FBBroadcastKeyOsVersion = @"osVersion"; +NSString *const FBBroadcastKeyState = @"state"; +NSString *const FBBroadcastKeyFramesReceived = @"framesReceived"; +NSString *const FBBroadcastKeyOrientation = @"orientation"; +NSString *const FBBroadcastKeyScreenWidth = @"screenWidth"; +NSString *const FBBroadcastKeyScreenHeight = @"screenHeight"; +NSString *const FBBroadcastKeyEvent = @"event"; +NSString *const FBBroadcastKeyReason = @"reason"; +NSString *const FBBroadcastKeyMessage = @"message"; + +NSString *const FBBroadcastCodecH264 = @"h264"; +NSString *const FBBroadcastCodecH265 = @"h265"; + +NSData *FBBroadcastEncodeMessage(FBBroadcastMessageType type, + uint32_t sessionId, + NSData *_Nullable payload) +{ + NSUInteger payloadLength = payload.length; + NSMutableData *message = [NSMutableData dataWithCapacity:FBBroadcastHeaderLength + payloadLength]; + + uint32_t bigMagic = CFSwapInt32HostToBig(FBBroadcastProtocolMagic); + [message appendBytes:&bigMagic length:sizeof(bigMagic)]; + uint8_t version = FBBroadcastProtocolVersion; + [message appendBytes:&version length:sizeof(version)]; + uint8_t messageType = (uint8_t)type; + [message appendBytes:&messageType length:sizeof(messageType)]; + uint16_t reserved = 0; + [message appendBytes:&reserved length:sizeof(reserved)]; + uint32_t bigSessionId = CFSwapInt32HostToBig(sessionId); + [message appendBytes:&bigSessionId length:sizeof(bigSessionId)]; + uint32_t bigPayloadLength = CFSwapInt32HostToBig((uint32_t)payloadLength); + [message appendBytes:&bigPayloadLength length:sizeof(bigPayloadLength)]; + + if (payloadLength > 0) { + [message appendData:(NSData *)payload]; + } + return message; +} + +NSData *_Nullable FBBroadcastEncodeJSONMessage(FBBroadcastMessageType type, + uint32_t sessionId, + NSDictionary *payload) +{ + NSData *json = [NSJSONSerialization dataWithJSONObject:payload options:(NSJSONWritingOptions)0 error:NULL]; + if (nil == json) { + return nil; + } + return FBBroadcastEncodeMessage(type, sessionId, json); +} + +BOOL FBBroadcastParseHeader(NSData *headerData, FBBroadcastMessageHeader *outHeader) +{ + if (headerData.length < FBBroadcastHeaderLength) { + return NO; + } + const uint8_t *bytes = (const uint8_t *)headerData.bytes; + + uint32_t magic; + memcpy(&magic, bytes, sizeof(magic)); + if (CFSwapInt32BigToHost(magic) != FBBroadcastProtocolMagic) { + return NO; + } + uint8_t version = bytes[4]; + if (version != FBBroadcastProtocolVersion) { + return NO; + } + + outHeader->version = version; + outHeader->type = bytes[5]; + uint32_t sessionId; + memcpy(&sessionId, bytes + 8, sizeof(sessionId)); + outHeader->sessionId = CFSwapInt32BigToHost(sessionId); + uint32_t payloadLength; + memcpy(&payloadLength, bytes + 12, sizeof(payloadLength)); + outHeader->payloadLength = CFSwapInt32BigToHost(payloadLength); + return YES; +} + +NSDictionary *_Nullable FBBroadcastParseJSONPayload(NSData *payload) +{ + if (payload.length == 0) { + return nil; + } + id parsed = [NSJSONSerialization JSONObjectWithData:payload options:(NSJSONReadingOptions)0 error:NULL]; + return [parsed isKindOfClass:NSDictionary.class] ? parsed : nil; +} + +NSData *FBBroadcastEncodeVideoFramePayload(uint64_t ptsUs, + BOOL isKeyFrame, + uint8_t orientation, + NSData *annexBPictureData) +{ + NSMutableData *payload = [NSMutableData dataWithCapacity:sizeof(uint64_t) + sizeof(uint8_t) + annexBPictureData.length]; + uint64_t bigPts = CFSwapInt64HostToBig(ptsUs); + [payload appendBytes:&bigPts length:sizeof(bigPts)]; + uint8_t flags = (isKeyFrame ? FBBroadcastFrameFlagKeyFrame : 0) + | (uint8_t)((orientation & FBBroadcastFrameOrientationMask) << FBBroadcastFrameOrientationShift); + [payload appendBytes:&flags length:sizeof(flags)]; + [payload appendData:annexBPictureData]; + return payload; +} + +NSData *FBBroadcastEncodeVideoFrameMessage(uint32_t sessionId, + uint64_t ptsUs, + BOOL isKeyFrame, + uint8_t orientation, + NSData *annexBPictureData) +{ + static const NSUInteger prefixLength = sizeof(uint64_t) + sizeof(uint8_t); + NSUInteger payloadLength = prefixLength + annexBPictureData.length; + NSMutableData *message = [NSMutableData dataWithCapacity:FBBroadcastHeaderLength + payloadLength]; + + uint32_t bigMagic = CFSwapInt32HostToBig(FBBroadcastProtocolMagic); + [message appendBytes:&bigMagic length:sizeof(bigMagic)]; + uint8_t version = FBBroadcastProtocolVersion; + [message appendBytes:&version length:sizeof(version)]; + uint8_t messageType = (uint8_t)FBBroadcastMessageTypeVideoFrame; + [message appendBytes:&messageType length:sizeof(messageType)]; + uint16_t reserved = 0; + [message appendBytes:&reserved length:sizeof(reserved)]; + uint32_t bigSessionId = CFSwapInt32HostToBig(sessionId); + [message appendBytes:&bigSessionId length:sizeof(bigSessionId)]; + uint32_t bigPayloadLength = CFSwapInt32HostToBig((uint32_t)payloadLength); + [message appendBytes:&bigPayloadLength length:sizeof(bigPayloadLength)]; + + uint64_t bigPts = CFSwapInt64HostToBig(ptsUs); + [message appendBytes:&bigPts length:sizeof(bigPts)]; + uint8_t flags = (isKeyFrame ? FBBroadcastFrameFlagKeyFrame : 0) + | (uint8_t)((orientation & FBBroadcastFrameOrientationMask) << FBBroadcastFrameOrientationShift); + [message appendBytes:&flags length:sizeof(flags)]; + [message appendData:annexBPictureData]; + return message; +} + +BOOL FBBroadcastParseVideoFramePayload(NSData *payload, + uint64_t *outPtsUs, + BOOL *outIsKeyFrame, + uint8_t *outOrientation, + NSData *_Nullable __autoreleasing *_Nonnull outAnnexB) +{ + static const NSUInteger prefixLength = sizeof(uint64_t) + sizeof(uint8_t); + if (payload.length < prefixLength) { + return NO; + } + const uint8_t *bytes = (const uint8_t *)payload.bytes; + uint64_t bigPts; + memcpy(&bigPts, bytes, sizeof(bigPts)); + *outPtsUs = CFSwapInt64BigToHost(bigPts); + uint8_t flags = bytes[sizeof(uint64_t)]; + *outIsKeyFrame = (flags & FBBroadcastFrameFlagKeyFrame) != 0; + *outOrientation = (flags >> FBBroadcastFrameOrientationShift) & FBBroadcastFrameOrientationMask; + *outAnnexB = [payload subdataWithRange:NSMakeRange(prefixLength, payload.length - prefixLength)]; + return YES; +} diff --git a/WebDriverAgentLib/Utilities/FBConfiguration.h b/WebDriverAgentLib/Utilities/FBConfiguration.h index 25704d93d..4a2647c45 100644 --- a/WebDriverAgentLib/Utilities/FBConfiguration.h +++ b/WebDriverAgentLib/Utilities/FBConfiguration.h @@ -139,6 +139,20 @@ extern NSString *const FBSnapshotMaxDepthKey; */ + (NSInteger)screenCaptureServerPort; +/** + The loopback TCP port the ReplayKit broadcast extension connects to. The default value is 9300. + It can be overridden via the BROADCAST_CONTROL_PORT environment variable (the extension itself + always uses the compile-time default, so overriding requires rebuilding the extension too). + */ ++ (NSInteger)broadcastControlPort; + +/** + The bundle identifier of the embedded ReplayKit broadcast upload extension. Defaults to the + runner app's bundle identifier with a '.broadcast' suffix; can be overridden via the + BROADCAST_EXT_BUNDLE_ID environment variable. + */ ++ (NSString *)broadcastExtensionBundleId; + /** The scaling factor for frames of the mjpeg stream. The default (and maximum) value is 100, which does not perform any scaling. The minimum value must be greater than zero. diff --git a/WebDriverAgentLib/Utilities/FBConfiguration.m b/WebDriverAgentLib/Utilities/FBConfiguration.m index 1e57095af..56704f7ee 100644 --- a/WebDriverAgentLib/Utilities/FBConfiguration.m +++ b/WebDriverAgentLib/Utilities/FBConfiguration.m @@ -17,6 +17,7 @@ #import #include "TargetConditionals.h" +#import "FBBroadcastProtocol.h" #import "FBXCodeCompatibility.h" #import "XCAXClient_iOS+FBSnapshotReqParams.h" #import "XCTestPrivateSymbols.h" @@ -175,6 +176,25 @@ + (NSInteger)screenCaptureServerPort return DefaultScreenCaptureServerPort; } ++ (NSInteger)broadcastControlPort +{ + if (NSProcessInfo.processInfo.environment[@"BROADCAST_CONTROL_PORT"] && + [NSProcessInfo.processInfo.environment[@"BROADCAST_CONTROL_PORT"] length] > 0) { + return [NSProcessInfo.processInfo.environment[@"BROADCAST_CONTROL_PORT"] integerValue]; + } + + return FBBroadcastDefaultControlPort; +} + ++ (NSString *)broadcastExtensionBundleId +{ + NSString *fromEnv = NSProcessInfo.processInfo.environment[@"BROADCAST_EXT_BUNDLE_ID"]; + if (fromEnv.length > 0) { + return fromEnv; + } + return [NSBundle.mainBundle.bundleIdentifier stringByAppendingString:@".broadcast"]; +} + + (CGFloat)mjpegScalingFactor { return FBMjpegScalingFactor; diff --git a/WebDriverAgentLib/Utilities/FBPixelBufferConverter.m b/WebDriverAgentLib/Utilities/FBPixelBufferConverter.m index f948ac508..a8251ac03 100644 --- a/WebDriverAgentLib/Utilities/FBPixelBufferConverter.m +++ b/WebDriverAgentLib/Utilities/FBPixelBufferConverter.m @@ -11,6 +11,7 @@ #import NSErrorDomain const FBPixelBufferConverterErrorDomain = @"com.facebook.WebDriverAgent.FBPixelBufferConverter"; +static CFStringRef const FBPixelBufferBitmapContextAttachmentKey = CFSTR("com.facebook.WebDriverAgent.FBPixelBufferConverter.BitmapContext"); static size_t FBEvenDimension(size_t value) { @@ -23,6 +24,9 @@ static size_t FBEvenDimension(size_t value) @interface FBPixelBufferConverter () @property (nonatomic) CVPixelBufferPoolRef pixelBufferPool; @property (nonatomic) CGColorSpaceRef colorSpace; + +- (nullable CGContextRef)copyBitmapContextForPixelBuffer:(CVPixelBufferRef)pixelBuffer + error:(NSError **)error CF_RETURNS_RETAINED; @end @implementation FBPixelBufferConverter @@ -107,29 +111,14 @@ - (nullable CVPixelBufferRef)copyPixelBufferFromCGImage:(CGImageRef)image } CVPixelBufferLockBaseAddress(pixelBuffer, 0); - void *baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer); - CGContextRef context = CGBitmapContextCreate(baseAddress, - self.width, - self.height, - 8, - CVPixelBufferGetBytesPerRow(pixelBuffer), - self.colorSpace, - kCGImageAlphaNoneSkipFirst | kCGBitmapByteOrder32Little); + CGContextRef context = [self copyBitmapContextForPixelBuffer:pixelBuffer + error:error]; if (NULL == context) { CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); CVPixelBufferRelease(pixelBuffer); - if (error) { - *error = [NSError errorWithDomain:FBPixelBufferConverterErrorDomain - code:5 - userInfo:@{NSLocalizedDescriptionKey: @"Cannot create a bitmap context for the pixel buffer"}]; - } return NULL; } - // Fill the whole frame with black to produce the letterbox/pillarbox padding. - CGContextSetRGBFillColor(context, 0.0, 0.0, 0.0, 1.0); - CGContextFillRect(context, CGRectMake(0, 0, self.width, self.height)); - // Scale to fit while preserving the aspect ratio and center the result. CGFloat imageWidth = (CGFloat)CGImageGetWidth(image); CGFloat imageHeight = (CGFloat)CGImageGetHeight(image); @@ -139,6 +128,16 @@ - (nullable CVPixelBufferRef)copyPixelBufferFromCGImage:(CGImageRef)image CGFloat targetHeight = imageHeight * scale; CGFloat originX = ((CGFloat)self.width - targetWidth) / 2.0; CGFloat originY = ((CGFloat)self.height - targetHeight) / 2.0; + BOOL needsPadding = originX > 0.5 + || originY > 0.5 + || targetWidth < (CGFloat)self.width - 0.5 + || targetHeight < (CGFloat)self.height - 0.5; + if (needsPadding) { + // Fill only when the frame needs letterbox/pillarbox padding; a full-frame draw overwrites + // every pixel and does not need the clear. + CGContextSetRGBFillColor(context, 0.0, 0.0, 0.0, 1.0); + CGContextFillRect(context, CGRectMake(0, 0, self.width, self.height)); + } CGContextDrawImage(context, CGRectMake(originX, originY, targetWidth, targetHeight), image); } @@ -147,6 +146,40 @@ - (nullable CVPixelBufferRef)copyPixelBufferFromCGImage:(CGImageRef)image return pixelBuffer; } +- (nullable CGContextRef)copyBitmapContextForPixelBuffer:(CVPixelBufferRef)pixelBuffer + error:(NSError **)error +{ + CGContextRef cachedContext = (CGContextRef)CVBufferCopyAttachment(pixelBuffer, + FBPixelBufferBitmapContextAttachmentKey, + NULL); + if (NULL != cachedContext) { + return cachedContext; + } + + void *baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer); + CGContextRef context = CGBitmapContextCreate(baseAddress, + self.width, + self.height, + 8, + CVPixelBufferGetBytesPerRow(pixelBuffer), + self.colorSpace, + kCGImageAlphaNoneSkipFirst | kCGBitmapByteOrder32Little); + if (NULL == context) { + if (error) { + *error = [NSError errorWithDomain:FBPixelBufferConverterErrorDomain + code:5 + userInfo:@{NSLocalizedDescriptionKey: @"Cannot create a bitmap context for the pixel buffer"}]; + } + return NULL; + } + + CVBufferSetAttachment(pixelBuffer, + FBPixelBufferBitmapContextAttachmentKey, + context, + kCVAttachmentMode_ShouldNotPropagate); + return context; +} + - (void)dealloc { if (NULL != _pixelBufferPool) { diff --git a/WebDriverAgentLib/Utilities/FBVideoEncoder.m b/WebDriverAgentLib/Utilities/FBVideoEncoder.m index 18b3c13c4..4230ec9a7 100644 --- a/WebDriverAgentLib/Utilities/FBVideoEncoder.m +++ b/WebDriverAgentLib/Utilities/FBVideoEncoder.m @@ -88,6 +88,8 @@ - (void)configureSession { VTSessionSetProperty(self.session, kVTCompressionPropertyKey_RealTime, kCFBooleanTrue); VTSessionSetProperty(self.session, kVTCompressionPropertyKey_AllowFrameReordering, kCFBooleanFalse); + CFNumberRef maxFrameDelayCount = (__bridge CFNumberRef)@(1); + VTSessionSetProperty(self.session, kVTCompressionPropertyKey_MaxFrameDelayCount, maxFrameDelayCount); CFStringRef profileLevel = (self.codec == FBVideoCodecH265) ? kVTProfileLevel_HEVC_Main_AutoLevel @@ -254,27 +256,33 @@ - (nullable NSData *)annexBPictureDataFromSampleBuffer:(CMSampleBufferRef)sample return nil; } - NSMutableData *avccData = [NSMutableData dataWithLength:totalLength]; - OSStatus status = CMBlockBufferCopyDataBytes(blockBuffer, 0, totalLength, avccData.mutableBytes); - if (status != kCMBlockBufferNoErr) { - return nil; - } - - const uint8_t *bytes = (const uint8_t *)avccData.bytes; size_t headerLength = self.nalUnitHeaderLength == 0 ? 4 : self.nalUnitHeaderLength; - NSMutableData *annexBData = [NSMutableData data]; + uint8_t headerBytes[8]; + NSMutableData *annexBData = [NSMutableData dataWithCapacity:totalLength + sizeof(FBAnnexBStartCode)]; size_t offset = 0; while (offset + headerLength <= totalLength) { + OSStatus status = CMBlockBufferCopyDataBytes(blockBuffer, offset, headerLength, headerBytes); + if (status != kCMBlockBufferNoErr) { + return nil; + } uint64_t nalLength = 0; for (size_t i = 0; i < headerLength; i++) { - nalLength = (nalLength << 8) | bytes[offset + i]; + nalLength = (nalLength << 8) | headerBytes[i]; } offset += headerLength; if (nalLength == 0 || offset + nalLength > totalLength) { break; } [annexBData appendBytes:FBAnnexBStartCode length:sizeof(FBAnnexBStartCode)]; - [annexBData appendBytes:bytes + offset length:(NSUInteger)nalLength]; + NSUInteger previousLength = annexBData.length; + [annexBData setLength:previousLength + (NSUInteger)nalLength]; + status = CMBlockBufferCopyDataBytes(blockBuffer, + offset, + (size_t)nalLength, + (uint8_t *)annexBData.mutableBytes + previousLength); + if (status != kCMBlockBufferNoErr) { + return nil; + } offset += nalLength; } return annexBData.length > 0 ? annexBData : nil; diff --git a/WebDriverAgentLib/Utilities/FBVideoStreamManager.h b/WebDriverAgentLib/Utilities/FBVideoStreamManager.h index a2f6064dc..e9e34bcd8 100644 --- a/WebDriverAgentLib/Utilities/FBVideoStreamManager.h +++ b/WebDriverAgentLib/Utilities/FBVideoStreamManager.h @@ -50,6 +50,9 @@ NS_ASSUME_NONNULL_BEGIN /** @return The session with the given id, or nil. */ - (nullable FBVideoStreamSession *)sessionWithIdentifier:(NSUInteger)identifier; +/** @return A snapshot of all active session objects. */ +- (NSArray *)activeSessions; + /** @return An array of dictionaries describing all active sessions, ordered by id. */ - (NSArray *)activeSessionsInfo; diff --git a/WebDriverAgentLib/Utilities/FBVideoStreamManager.m b/WebDriverAgentLib/Utilities/FBVideoStreamManager.m index 9dc31b38b..172fafa2b 100644 --- a/WebDriverAgentLib/Utilities/FBVideoStreamManager.m +++ b/WebDriverAgentLib/Utilities/FBVideoStreamManager.m @@ -12,6 +12,7 @@ #import @import UniformTypeIdentifiers; +#import "FBBroadcastManager.h" #import "FBConfiguration.h" #import "FBImageUtils.h" #import "FBLogger.h" @@ -27,7 +28,6 @@ static const NSTimeInterval FRAME_TIMEOUT = 1.0; static const NSTimeInterval FAILURE_BACKOFF_MIN = 1.0; static const NSTimeInterval FAILURE_BACKOFF_MAX = 10.0; -static const CGFloat CAPTURE_COMPRESSION_QUALITY = 0.8; static const char *QUEUE_NAME = "Screen Capture Encoder Queue"; @interface FBVideoStreamManager () @@ -134,6 +134,12 @@ - (nullable FBVideoStreamSession *)startSessionWithConfiguration:(FBScreenCaptur [weakSelf captureFrameWithGeneration:generation]; }); } + // Attach the session to the broadcast extension (if connected): the session keeps serving + // locally encoded screenshot frames until the extension's first key frame arrives. + session.onBroadcastKeyFrameNeeded = ^(NSUInteger sessionIdentifier) { + [FBBroadcastManager.sharedInstance requestKeyFrameForSession:sessionIdentifier]; + }; + [FBBroadcastManager.sharedInstance notifySessionAdded:session]; [FBLogger logFmt:@"Started screen capture session %@ (%@ %@x%@) on port %@", @(identifier), session.toDictionary[@"codec"], @(configuration.width), @(configuration.height), @(configuration.port)]; return session; @@ -193,6 +199,7 @@ - (BOOL)stopSessionWithIdentifier:(NSUInteger)identifier } } [session stop]; + [FBBroadcastManager.sharedInstance notifySessionRemoved:identifier]; [FBLogger logFmt:@"Stopped screen capture session %@", @(identifier)]; return YES; } @@ -240,6 +247,14 @@ - (void)stopAllSessions } for (FBVideoStreamSession *session in snapshot) { [session stop]; + [FBBroadcastManager.sharedInstance notifySessionRemoved:session.identifier]; + } +} + +- (NSArray *)activeSessions +{ + @synchronized (self.sessions) { + return self.sessions.allValues; } } @@ -283,7 +298,9 @@ - (void)captureFrameWithGeneration:(NSUInteger)generation BOOL anyClients = NO; for (FBVideoStreamSession *session in snapshot) { loopFps = MAX(loopFps, session.configuration.fps); - if ([session hasClients]) { + // Sessions fed by the broadcast extension do not need (expensive) XCTest screenshots; the + // loop keeps ticking cheaply so it picks the screenshot capture back up if they detach. + if ([session requiresLocalFrames]) { anyClients = YES; } } @@ -299,8 +316,15 @@ - (void)captureFrameWithGeneration:(NSUInteger)generation } NSError *error; + CGFloat captureQuality = 1.0; + for (FBVideoStreamSession *session in snapshot) { + if ([session requiresLocalFrames]) { + captureQuality = MIN(captureQuality, session.configuration.quality); + } + } + NSData *screenshotData = [FBScreenshot takeInOriginalResolutionWithScreenID:self.mainScreenID - compressionQuality:CAPTURE_COMPRESSION_QUALITY + compressionQuality:captureQuality uti:UTTypeJPEG timeout:FRAME_TIMEOUT error:&error]; diff --git a/WebDriverAgentLib/Utilities/FBVideoStreamSession.h b/WebDriverAgentLib/Utilities/FBVideoStreamSession.h index 350211c0d..e656cdec3 100644 --- a/WebDriverAgentLib/Utilities/FBVideoStreamSession.h +++ b/WebDriverAgentLib/Utilities/FBVideoStreamSession.h @@ -21,6 +21,14 @@ typedef NS_ENUM(NSUInteger, FBVideoFraming) { FBVideoFramingScrcpy, }; +/** The origin of the encoded frames a session is currently serving. */ +typedef NS_ENUM(NSUInteger, FBVideoStreamSource) { + /** Frames are captured via XCTest screenshots and encoded locally (default). */ + FBVideoStreamSourceScreenshot, + /** Pre-encoded frames are received from the ReplayKit broadcast extension. */ + FBVideoStreamSourceBroadcast, +}; + /** Describes a single screen-capture streaming request. */ @interface FBScreenCaptureConfiguration : NSObject @@ -32,6 +40,8 @@ typedef NS_ENUM(NSUInteger, FBVideoFraming) { @property (nonatomic) NSUInteger height; /** The target average bitrate in bits per second. */ @property (nonatomic) NSUInteger bitrate; +/** JPEG compression quality used when capturing XCTest screenshot frames before video encoding. */ +@property (nonatomic) CGFloat quality; /** The capture/encode framerate in frames per second. */ @property (nonatomic) NSUInteger fps; /** The framing of the broadcast byte stream (raw Annex-B by default). */ @@ -53,6 +63,13 @@ typedef NS_ENUM(NSUInteger, FBVideoFraming) { @property (nonatomic, readonly) NSUInteger identifier; /** The configuration this session was started with. */ @property (nonatomic, readonly) FBScreenCaptureConfiguration *configuration; +/** The origin of the frames this session currently serves. */ +@property (atomic, readonly) FBVideoStreamSource activeSource; +/** + Invoked (with the session identifier) when a key frame is needed while the session serves + broadcast frames, so the request can be forwarded to the extension's encoder. + */ +@property (nonatomic, nullable, copy) void (^onBroadcastKeyFrameNeeded)(NSUInteger sessionIdentifier); - (instancetype)initWithIdentifier:(NSUInteger)identifier configuration:(FBScreenCaptureConfiguration *)configuration; @@ -86,6 +103,29 @@ typedef NS_ENUM(NSUInteger, FBVideoFraming) { */ - (void)maybeEncodeCGImage:(CGImageRef)image atTimeMs:(uint64_t)nowMs; +/** + YES while this session needs frames from the shared screenshot capture loop, i.e. it is active, + has at least one client and is not being fed by the broadcast extension. + */ +- (BOOL)requiresLocalFrames; + +/** Stores fresh Annex-B parameter sets received from the broadcast extension. */ +- (void)ingestBroadcastParameterSets:(NSData *)parameterSets; + +/** + Broadcasts a pre-encoded picture received from the broadcast extension. While the session is + still on the screenshot source it switches to the broadcast source on the first key frame that + has parameter sets available (delta frames before that are dropped, so clients always resync + at an IDR). + */ +- (void)ingestBroadcastFrame:(NSData *)annexBPictureData isKeyFrame:(BOOL)isKeyFrame; + +/** + Reverts the session to the local screenshot source (e.g. because the broadcast stopped) and + forces the local encoder to open with a key frame so clients can resync without reconnecting. + */ +- (void)detachBroadcastSourceAndForceKeyFrame; + /** @return A dictionary describing this session. */ - (NSDictionary *)toDictionary; diff --git a/WebDriverAgentLib/Utilities/FBVideoStreamSession.m b/WebDriverAgentLib/Utilities/FBVideoStreamSession.m index 807ce7e61..00908b9f2 100644 --- a/WebDriverAgentLib/Utilities/FBVideoStreamSession.m +++ b/WebDriverAgentLib/Utilities/FBVideoStreamSession.m @@ -25,8 +25,18 @@ static const uint64_t FBScrcpyFlagConfig = (uint64_t)1 << 63; static const uint64_t FBScrcpyFlagKeyFrame = (uint64_t)1 << 62; static const uint64_t FBScrcpyPtsMask = ~((uint64_t)3 << 62); +static const CGFloat FBDefaultScreenCaptureQuality = 0.8; @implementation FBScreenCaptureConfiguration + +- (instancetype)init +{ + if ((self = [super init])) { + _quality = FBDefaultScreenCaptureQuality; + } + return self; +} + @end @@ -41,6 +51,9 @@ @interface FBVideoStreamSession () /** The parameter sets most recently broadcast as a scrcpy config packet (for change detection). */ @property (nonatomic, nullable, copy) NSData *lastSentParameterSets; @property (atomic, getter=isActive) BOOL active; +@property (atomic, readwrite) FBVideoStreamSource activeSource; +/** The parameter sets most recently received from the broadcast extension. */ +@property (atomic, nullable, copy) NSData *broadcastParameterSets; @end @@ -55,6 +68,7 @@ - (instancetype)initWithIdentifier:(NSUInteger)identifier _configuration = configuration; _listeningClients = [NSMutableArray array]; _active = NO; + _activeSource = FBVideoStreamSourceScreenshot; } return self; } @@ -117,13 +131,30 @@ - (BOOL)hasClients - (void)requestKeyFrame { + // While the broadcast extension feeds this session the local encoder is idle, so the request + // must reach the extension's encoder instead. + if (self.activeSource == FBVideoStreamSourceBroadcast) { + void (^onKeyFrameNeeded)(NSUInteger) = self.onBroadcastKeyFrameNeeded; + if (nil != onKeyFrameNeeded) { + onKeyFrameNeeded(self.identifier); + return; + } + } @synchronized (self) { [self.encoder requestKeyFrame]; } } +- (BOOL)requiresLocalFrames +{ + return self.isActive && self.activeSource == FBVideoStreamSourceScreenshot && [self hasClients]; +} + - (void)maybeEncodeCGImage:(CGImageRef)image atTimeMs:(uint64_t)nowMs { + if (self.activeSource == FBVideoStreamSourceBroadcast) { + return; + } @synchronized (self) { if (!self.isActive || nil == self.encoder || ![self hasClients]) { return; @@ -159,12 +190,88 @@ - (uint64_t)nextPresentationTimeMs return candidate; } +#pragma mark - Broadcast (ReplayKit) source + +- (void)ingestBroadcastParameterSets:(NSData *)parameterSets +{ + if (parameterSets.length > 0) { + self.broadcastParameterSets = parameterSets; + } +} + +- (void)ingestBroadcastFrame:(NSData *)annexBPictureData isKeyFrame:(BOOL)isKeyFrame +{ + if (!self.isActive || annexBPictureData.length == 0) { + return; + } + uint64_t presentationTimeUs; + @synchronized (self) { + if (self.activeSource == FBVideoStreamSourceScreenshot) { + // Only take over at an IDR with parameter sets available, so connected clients can + // resync at the source switch without reconnecting. + if (!isKeyFrame || nil == self.broadcastParameterSets) { + return; + } + self.activeSource = FBVideoStreamSourceBroadcast; + // Force a fresh scrcpy config packet for the new elementary stream. + self.lastSentParameterSets = nil; + [FBLogger logFmt:@"Screen capture session %@: switched to the ReplayKit broadcast source", @(self.identifier)]; + } + // Re-stamp with the session's own monotonic clock so one time base survives source switches; + // the extension's timestamp is intentionally ignored. + presentationTimeUs = [self nextPresentationTimeMs] * 1000; + } + [self emitEncodedPicture:annexBPictureData + isKeyFrame:isKeyFrame + presentationTimeUs:presentationTimeUs + parameterSets:self.broadcastParameterSets]; +} + +- (void)detachBroadcastSourceAndForceKeyFrame +{ + @synchronized (self) { + if (self.activeSource != FBVideoStreamSourceBroadcast) { + return; + } + self.activeSource = FBVideoStreamSourceScreenshot; + self.broadcastParameterSets = nil; + // Force a fresh config packet and an IDR from the local encoder so clients resync. + self.lastSentParameterSets = nil; + [self.encoder requestKeyFrame]; + } + [FBLogger logFmt:@"Screen capture session %@: reverted to the screenshot source", @(self.identifier)]; +} + +/** The parameter sets of whichever source currently feeds this session. */ +- (nullable NSData *)currentParameterSets +{ + return self.activeSource == FBVideoStreamSourceBroadcast + ? self.broadcastParameterSets + : self.encoder.parameterSetAnnexB; +} + #pragma mark - - (void)videoEncoder:(FBVideoEncoder *)encoder didEncodeFrame:(NSData *)annexBPictureData isKeyFrame:(BOOL)isKeyFrame presentationTimeUs:(uint64_t)presentationTimeUs +{ + // While the broadcast source is active the local encoder's (at most one in-flight) output is + // discarded; clients resync at the broadcast IDR that triggered the switch. + if (self.activeSource == FBVideoStreamSourceBroadcast) { + return; + } + [self emitEncodedPicture:annexBPictureData + isKeyFrame:isKeyFrame + presentationTimeUs:presentationTimeUs + parameterSets:encoder.parameterSetAnnexB]; +} + +- (void)emitEncodedPicture:(NSData *)annexBPictureData + isKeyFrame:(BOOL)isKeyFrame + presentationTimeUs:(uint64_t)presentationTimeUs + parameterSets:(nullable NSData *)parameterSets { if (annexBPictureData.length == 0) { return; @@ -175,7 +282,6 @@ - (void)videoEncoder:(FBVideoEncoder *)encoder // packet must carry picture data only. Emit a separate config packet whenever the parameter // sets change. if (isKeyFrame) { - NSData *parameterSets = encoder.parameterSetAnnexB; if (parameterSets.length > 0 && ![parameterSets isEqualToData:self.lastSentParameterSets]) { self.lastSentParameterSets = parameterSets; [self broadcastData:[self.class scrcpyPacketWithPayload:parameterSets @@ -192,7 +298,6 @@ - (void)videoEncoder:(FBVideoEncoder *)encoder // Annex-B mode: prepend the parameter sets to key frames so each IDR is independently decodable. if (isKeyFrame) { - NSData *parameterSets = encoder.parameterSetAnnexB; if (parameterSets.length > 0) { NSMutableData *keyFrame = [NSMutableData dataWithCapacity:parameterSets.length + annexBPictureData.length]; [keyFrame appendData:parameterSets]; @@ -249,7 +354,7 @@ - (void)didClientConnect:(GCDAsyncSocket *)newClient } // Hand the latest parameter sets to the new client and force a key frame so it can start // decoding immediately. In scrcpy mode the parameter sets are wrapped as a config packet. - NSData *parameterSets = self.encoder.parameterSetAnnexB; + NSData *parameterSets = [self currentParameterSets]; if (parameterSets.length > 0) { NSData *payload = self.configuration.framing == FBVideoFramingScrcpy ? [self.class scrcpyPacketWithPayload:parameterSets flags:FBScrcpyFlagConfig presentationTimeUs:0] @@ -291,8 +396,10 @@ - (NSDictionary *)toDictionary @"height": @(self.configuration.height), @"fps": @(self.configuration.fps), @"bitrate": @(self.configuration.bitrate), + @"quality": @(self.configuration.quality), @"port": @(self.configuration.port), @"clients": @(clientCount), + @"source": self.activeSource == FBVideoStreamSourceBroadcast ? @"replaykit" : @"screenshot", }; } diff --git a/WebDriverAgentLib/Vendor/CocoaHTTPServer/HTTPConnection.h b/WebDriverAgentLib/Vendor/CocoaHTTPServer/HTTPConnection.h index e1868532f..f2d7b396a 100644 --- a/WebDriverAgentLib/Vendor/CocoaHTTPServer/HTTPConnection.h +++ b/WebDriverAgentLib/Vendor/CocoaHTTPServer/HTTPConnection.h @@ -15,7 +15,10 @@ @interface HTTPConfig : NSObject { - HTTPServer __unsafe_unretained *server; + // Strong on purpose: connections reply on their own queues, so an unretained server is a + // use-after-free once the server is torn down (e.g. via /wda/shutdown) while replies are in + // flight. The server->connections->config->server cycle is broken when connections die. + HTTPServer __strong *server; NSString __strong *documentRoot; dispatch_queue_t queue; } @@ -23,7 +26,7 @@ - (id)initWithServer:(HTTPServer *)server documentRoot:(NSString *)documentRoot; - (id)initWithServer:(HTTPServer *)server documentRoot:(NSString *)documentRoot queue:(dispatch_queue_t)q; -@property (nonatomic, unsafe_unretained, readonly) HTTPServer *server; +@property (nonatomic, strong, readonly) HTTPServer *server; @property (nonatomic, strong, readonly) NSString *documentRoot; @property (nonatomic, readonly) dispatch_queue_t queue; diff --git a/WebDriverAgentLib/Vendor/RoutingHTTPServer/RoutingConnection.m b/WebDriverAgentLib/Vendor/RoutingHTTPServer/RoutingConnection.m index 3eee19267..99769cd61 100644 --- a/WebDriverAgentLib/Vendor/RoutingHTTPServer/RoutingConnection.m +++ b/WebDriverAgentLib/Vendor/RoutingHTTPServer/RoutingConnection.m @@ -8,7 +8,9 @@ #pragma clang diagnostic ignored "-Wundeclared-selector" @implementation RoutingConnection { - __unsafe_unretained RoutingHTTPServer *http; + // Weak (was unsafe_unretained): replies run on connection queues and must not dereference a + // torn-down server; messaging nil is harmless, a dangling pointer crashes in objc_msgSend. + __weak RoutingHTTPServer *http; NSDictionary *headers; } @@ -63,7 +65,8 @@ - (void)processBodyData:(NSData *)postDataChunk { RouteResponse *response = [http routeMethod:method withPath:path parameters:params request:request connection:self]; if (response != nil) { - headers = response.headers; + // Snapshot instead of aliasing the route response's live (mutable) dictionary. + headers = [response.headers copy]; return response.proxiedResponse; } diff --git a/docs/broadcast-extension.md b/docs/broadcast-extension.md new file mode 100644 index 000000000..b2686fc66 --- /dev/null +++ b/docs/broadcast-extension.md @@ -0,0 +1,119 @@ +# ReplayKit Broadcast Extension + +WebDriverAgent embeds a ReplayKit **broadcast upload extension** +(`WebDriverAgentBroadcast.appex`) into the generated +`WebDriverAgentRunner-Runner.app`. When a broadcast is running, the extension receives the +system's screen frames as pixel buffers (up to 60 fps, no polling), encodes them with one +hardware H.264/H.265 encoder per capture session and ships the elementary stream to WDA over +a loopback TCP connection. `/mobilerun/screencapture` sessions then serve those frames instead +of the XCTest screenshot loop, raising the achievable frame rate from ~10-20 fps to 30-60 fps. + +The extension only works on **physical iOS devices** (ReplayKit broadcasts are unavailable on +the Simulator and tvOS). Without a running broadcast every capture session transparently uses +the legacy screenshot pipeline; each session reports its current origin via the `source` field +(`"replaykit"` or `"screenshot"`). + +## HTTP endpoints + +| Endpoint | Method | Description | +|---|---|---| +| `/mobilerun/screencapture/broadcast/start` | POST | Starts a system broadcast targeting the bundled extension. Foregrounds the runner app, triggers `RPSystemBroadcastPickerView` and confirms the system sheet via UI automation, then waits for the extension to connect. Idempotent while connected. | +| `/mobilerun/screencapture/broadcast` | GET | Broadcast status: `state` (`idle`/`connected`/`paused`), control port, extension id, last heartbeat (frames received, orientation, screen size) and the capture sessions with their active `source`. | +| `/mobilerun/screencapture/broadcast/stop` | POST | Asks the extension to finish the broadcast. Live sessions fall back to the screenshot source with a forced key frame; clients do not need to reconnect. | + +`broadcast/start` body (all optional): + +```json +{ + "timeout": 30, + "confirmButtonLabels": ["Start Broadcast"], + "restoreForegroundApp": true +} +``` + +- `timeout` — seconds to wait for the extension to connect (covers the system's 3-2-1 countdown). +- `confirmButtonLabels` — labels to look for on the system confirmation sheet. Pass the + localized label when the device language is not English (a button starting with "Start" is + used as fallback). +- `restoreForegroundApp` — re-activate the previously active app after the broadcast starts + (the start dance briefly foregrounds the runner app, ~2-3 s). + +Notes: + +- `/mobilerun/screencapture/start` never starts a broadcast by itself. Sessions started while + the broadcast is connected attach to it automatically; sessions started before it pick it up + on the extension's first key frame. +- Broadcasts started manually from Control Center attach exactly the same way (WDA listens + permanently). +- If the broadcast stops (endpoint, status-bar pill, extension crash), sessions revert to the + screenshot source within ~6 s (heartbeat staleness) without dropping client connections. +- Frames are encoded in the native ReplayKit orientation (not rotated upright like the + screenshot path). The current `orientation` (CGImagePropertyOrientation 1-8) is exposed via + the status endpoint and heartbeat. +- ReplayKit only delivers frames while the screen changes; the extension re-encodes the last + frame to fill delivery gaps, so the stream holds the session's requested fps on static + screens too (`repeated` counter in the heartbeat). + +## Environment variables + +| Variable | Default | Description | +|---|---|---| +| `BROADCAST_CONTROL_PORT` | `9300` | Loopback port WDA listens on for the extension. The extension always connects to the compile-time default, so overriding this requires rebuilding the extension with a matching `FBBroadcastDefaultControlPort`. | +| `BROADCAST_EXT_BUNDLE_ID` | `.broadcast` | Bundle id preselected in the broadcast picker. Only needed if a re-signing pipeline renames the appex. | + +## Build integration + +- The `WebDriverAgentBroadcast` target is a dependency of `WebDriverAgentRunner`, so any + scheme-based build (`xcodebuild build-for-testing -scheme WebDriverAgentRunner`, Fastlane, + the npm bundle scripts) builds the appex into `BUILT_PRODUCTS_DIR`. +- A scheme **build post-action** (`Scripts/embed-broadcast-extension.sh`) copies the appex + into `Runner.app/PlugIns/`, rewrites its `CFBundleIdentifier` to + `.broadcast` (the host id gets the `.xctrunner` suffix and may + be overridden by downstream tooling, so it is derived at embed time) and re-signs + inner-first. Post-actions run for scheme-based command-line builds too — the same mechanism + that embeds the runner icon today. +- Builds that bypass the scheme (`xcodebuild -target ...`) must run the embed step manually: + + ```bash + BUILT_PRODUCTS_DIR= PRODUCT_NAME=WebDriverAgentRunner \ + Scripts/embed-broadcast-extension.sh + ``` + +## Re-signing (device farms / prebuilt WDA) + +Pipelines that re-sign `WebDriverAgentRunner-Runner.app` must keep the nested appex valid: + +1. Sign `PlugIns/WebDriverAgentBroadcast.appex` first with the **same team/identity** as the + app (or use `codesign --deep` on the app, which handles nesting). Signing the outer app does + not invalidate an already-valid nested signature. +2. The appex `CFBundleIdentifier` must remain `.broadcast` — installd + rejects extensions whose bundle id is not prefixed by the host app's, or whose team differs. + If the pipeline changes the runner's bundle id, patch the appex id accordingly (the embed + script shows how) and set `BROADCAST_EXT_BUNDLE_ID` if the suffix differs. The signed + entitlements' `application-identifier` must match the (new) bundle id — when the embed + script rewrites the id it regenerates that entitlement automatically; custom re-signers + must do the same. +3. Provisioning: the embedded profile must cover the appex bundle id; a wildcard development + profile (`TEAM.*`) is the low-friction option. The extension needs no special entitlements + (no app groups — IPC is loopback TCP). + +## Architecture + +``` +WebDriverAgentRunner-Runner.app +├── PlugIns/WebDriverAgentRunner.xctest WDA +│ FBBroadcastManager ─ FBBroadcastControlServer (127.0.0.1:9300) +│ FBVideoStreamManager → FBVideoStreamSession (TCP fan-out :9200+, +│ screenshot-encode or broadcast-passthrough per session) +└── PlugIns/WebDriverAgentBroadcast.appex extension + FBBroadcastSampleHandler (RPBroadcastSampleHandler) + → FBExtSessionPipeline × N (VTPixelTransfer letterbox scale + + FBVideoEncoder per session, 420v end to end) + → FBExtBroadcastClient ──TCP──► WDA control port +``` + +The wire protocol (16-byte framed messages, JSON control payloads, binary video frames) is +defined in `WebDriverAgentLib/Utilities/FBBroadcastProtocol.h`, which is compiled into both +the framework and the extension. + +Audio capture (Opus) is a planned follow-up; audio sample buffers are currently ignored. diff --git a/docs/mobilerun-screencapture.md b/docs/mobilerun-screencapture.md index f5ac961bf..0c80b320b 100644 --- a/docs/mobilerun-screencapture.md +++ b/docs/mobilerun-screencapture.md @@ -42,6 +42,7 @@ not a WDA automation session is active. | `codec` | string | no | `"h264"` | `h264`/`avc` or `h265`/`hevc`. | | `framing` | string | no | `"annexb"` | `annexb`/`annex-b`/`raw`, or `scrcpy`/`packet`/`packetized`. See [Wire formats](#wire-formats). | | `bitrate` | int | no | `6000000` | Target average bits/sec. | +| `quality` | float | no | `0.8` | JPEG quality (`0.0`–`1.0`) used for XCTest screenshot capture before local H.264/H.265 encoding. Lower values can reduce screenshot capture/decode cost. Does not affect ReplayKit/broadcast-source frames. | | `fps` | int | no | `30` | Capture/encode frame rate. | | `port` | int | no | auto | `0` or omitted → auto-assign from **9200** (env `SCREEN_CAPTURE_SERVER_PORT` overrides the base), scanning forward up to 64 ports. An explicit port (1–65535) is tried once and surfaces a bind failure. | @@ -58,6 +59,7 @@ Returned by `start` and `GET …/:id`; also the array items of the list response "height": 1334, "fps": 30, "bitrate": 6000000, + "quality": 0.8, "port": 9200, "clients": 0 } @@ -100,7 +102,7 @@ Start (raw, default), against `http://:8100`: ```bash curl -s -X POST http://localhost:8100/mobilerun/screencapture/start \ -H 'Content-Type: application/json' \ - -d '{"width":750,"height":1334,"codec":"h264","fps":30}' + -d '{"width":750,"height":1334,"codec":"h264","fps":30,"quality":0.5}' # → { "id":1, "framing":"annexb", "port":9200, ... } ``` @@ -132,5 +134,8 @@ curl -s -X POST http://localhost:8100/mobilerun/screencapture/1/stop than buffered (1 s write timeout), and `TCP_NODELAY` is on for low latency. - A late joiner won't decode until the next key frame — hit the `/keyframe` endpoint (connecting already forces one) if you need an immediate resync. +- If multiple screenshot-source sessions request different `quality` values, WDA captures the + shared local screenshot frame at the lowest requested quality and fans it out to all local + encoders. - For `scrcpy` framing you must parse the 12-byte header yourself (or reuse `ReadFrame` from `h264reader.go`); you can't pipe it straight into ffmpeg the way you can with `annexb`. diff --git a/package.json b/package.json index cbea980b8..b7ce78b6f 100644 --- a/package.json +++ b/package.json @@ -89,10 +89,12 @@ "build/lib", "Scripts/build.sh", "Scripts/embed-runner-icon.sh", + "Scripts/embed-broadcast-extension.sh", "Scripts/*.mjs", "Configurations", "PrivateHeaders", "WebDriverAgent.xcodeproj", + "WebDriverAgentBroadcast", "WebDriverAgentLib", "WebDriverAgentRunner", "WebDriverAgentTests",