From 03831a9d9b20adea7810483259b44c5fde10e919 Mon Sep 17 00:00:00 2001 From: Liang Ma Date: Mon, 25 Nov 2019 23:23:54 -0800 Subject: [PATCH 01/45] set python version to PY2 --- BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/BUILD b/BUILD index bc06659..b528fe8 100644 --- a/BUILD +++ b/BUILD @@ -27,4 +27,5 @@ par_binary( ':simulator', ], data = glob(['test_runner/TestProject/**']), + python_version = 'PY2', ) From f6ebebef618dafcb14d4644142fdcb653f45e783 Mon Sep 17 00:00:00 2001 From: Jeff Verkoeyen Date: Sat, 28 Dec 2019 22:40:01 -0500 Subject: [PATCH 02/45] Fix test failures with Xcode 11 on pre-iOS 12.2 devices running Swift code. --- shared/xcode_info_util.py | 10 ++++++++++ test_runner/logic_test_util.py | 10 ++++++++++ test_runner/xctestrun.py | 27 +++++++++++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/shared/xcode_info_util.py b/shared/xcode_info_util.py index 134d73d..0521716 100644 --- a/shared/xcode_info_util.py +++ b/shared/xcode_info_util.py @@ -17,6 +17,7 @@ import os import subprocess +from shared import ios_constants _xcode_version_number = None @@ -26,6 +27,15 @@ def GetXcodeDeveloperPath(): return subprocess.check_output(('xcode-select', '-p')).strip() +def GetSwift5FallbackLibsDir(): + relativePath = "Toolchains/XcodeDefault.xctoolchain/usr/lib/swift-5.0" + swiftLibsDir = os.path.join(GetXcodeDeveloperPath(), relativePath) + swiftLibPlatformDir = os.path.join(swiftLibsDir, ios_constants.SDK.IPHONESIMULATOR) + if os.path.exists(swiftLibPlatformDir): + return swiftLibPlatformDir + return None + + def GetXcodeVersionNumber(): """Gets the Xcode version number. diff --git a/test_runner/logic_test_util.py b/test_runner/logic_test_util.py index 9021d8f..b2c7de3 100644 --- a/test_runner/logic_test_util.py +++ b/test_runner/logic_test_util.py @@ -48,6 +48,16 @@ def RunLogicTestOnSim( for key in env_vars: simctl_env_vars[_SIMCTL_ENV_VAR_PREFIX + key] = env_vars[key] simctl_env_vars['NSUnbufferedIO'] = 'YES' + + # Xcode 11+'s Swift dylibs are configured in a way that does not allow them to load the correct + # libswiftFoundation.dylib file from libXCTestSwiftSupport.dylib. This bug only affects tests that + # run on simulators running iOS 12.1 or lower. To fix this bug, we provide an explicit fallback to + # the correct Swift dylibs that have been packaged with Xcode. + # See https://github.com/bazelbuild/rules_apple/issues/684 for context. + swift5FallbackLibsDir = xcode_info_util.GetSwift5FallbackLibsDir() + if swift5FallbackLibsDir: + simctl_env_vars[_SIMCTL_ENV_VAR_PREFIX + "DYLD_FALLBACK_LIBRARY_PATH"] = swift5FallbackLibsDir + command = [ 'xcrun', 'simctl', 'spawn', '-s', sim_id, xcode_info_util.GetXctestToolPath(ios_constants.SDK.IPHONESIMULATOR)] diff --git a/test_runner/xctestrun.py b/test_runner/xctestrun.py index 3d3cf55..c46ad5b 100644 --- a/test_runner/xctestrun.py +++ b/test_runner/xctestrun.py @@ -475,6 +475,15 @@ def _GenerateTestRootForXcuitest(self): developer=developer_path), 'DYLD_LIBRARY_PATH': '__TESTROOT__:%s/usr/lib' % developer_path } + # Xcode 11+'s Swift dylibs are configured in a way that does not allow them to load the correct + # libswiftFoundation.dylib file from libXCTestSwiftSupport.dylib. This bug only affects tests that + # run on simulators running iOS 12.1 or lower. To fix this bug, we provide an explicit fallback to + # the correct Swift dylibs that have been packaged with Xcode. + # See https://github.com/bazelbuild/rules_apple/issues/684 for context. + swift5FallbackLibsDir = xcode_info_util.GetSwift5FallbackLibsDir() + if swift5FallbackLibsDir: + test_envs["DYLD_FALLBACK_LIBRARY_PATH"] = swift5FallbackLibsDir + self._xctestrun_dict = { 'IsUITestBundle': True, 'SystemAttachmentLifetime': 'keepNever', @@ -613,6 +622,15 @@ def _GenerateTestRootForXctest(self): 'DYLD_INSERT_LIBRARIES': dyld_insert_libs, 'DYLD_LIBRARY_PATH': '__TESTROOT__:%s/usr/lib:' % developer_path } + # Xcode 11+'s Swift dylibs are configured in a way that does not allow them to load the correct + # libswiftFoundation.dylib file from libXCTestSwiftSupport.dylib. This bug only affects tests that + # run on simulators running iOS 12.1 or lower. To fix this bug, we provide an explicit fallback to + # the correct Swift dylibs that have been packaged with Xcode. + # See https://github.com/bazelbuild/rules_apple/issues/684 for context. + swift5FallbackLibsDir = xcode_info_util.GetSwift5FallbackLibsDir() + if swift5FallbackLibsDir: + test_envs["DYLD_FALLBACK_LIBRARY_PATH"] = swift5FallbackLibsDir + self._xctestrun_dict = { 'TestHostPath': self._app_under_test_dir, 'TestBundlePath': self._test_bundle_dir, @@ -633,6 +651,15 @@ def _GenerateTestRootForLogicTest(self): 'DYLD_FRAMEWORK_PATH': dyld_framework_path, 'DYLD_LIBRARY_PATH': dyld_framework_path } + # Xcode 11+'s Swift dylibs are configured in a way that does not allow them to load the correct + # libswiftFoundation.dylib file from libXCTestSwiftSupport.dylib. This bug only affects tests that + # run on simulators running iOS 12.1 or lower. To fix this bug, we provide an explicit fallback to + # the correct Swift dylibs that have been packaged with Xcode. + # See https://github.com/bazelbuild/rules_apple/issues/684 for context. + swift5FallbackLibsDir = xcode_info_util.GetSwift5FallbackLibsDir() + if swift5FallbackLibsDir: + test_envs["DYLD_FALLBACK_LIBRARY_PATH"] = swift5FallbackLibsDir + self._xctestrun_dict = { 'TestBundlePath': self._test_bundle_dir, 'TestHostPath': xcode_info_util.GetXctestToolPath(self._sdk), From 091f9c01c3fd7db20a0ee39b4097549b82c4beb7 Mon Sep 17 00:00:00 2001 From: Jeff Verkoeyen Date: Wed, 1 Jan 2020 16:18:29 -0500 Subject: [PATCH 03/45] Docs. --- shared/xcode_info_util.py | 6 ++++++ test_runner/logic_test_util.py | 9 ++++----- test_runner/xctestrun.py | 30 +++++++++++++++--------------- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/shared/xcode_info_util.py b/shared/xcode_info_util.py index 0521716..d20db58 100644 --- a/shared/xcode_info_util.py +++ b/shared/xcode_info_util.py @@ -27,6 +27,12 @@ def GetXcodeDeveloperPath(): return subprocess.check_output(('xcode-select', '-p')).strip() +# Xcode 11+'s Swift dylibs are configured in a way that does not allow them to load the correct +# libswiftFoundation.dylib file from libXCTestSwiftSupport.dylib. This bug only affects tests that +# run on simulators running iOS 12.1 or lower. To fix this bug, we need to provide explicit +# fallbacks to the correct Swift dylibs that have been packaged with Xcode. This method returns the +# path to that fallback directory. +# See https://github.com/bazelbuild/rules_apple/issues/684 for context. def GetSwift5FallbackLibsDir(): relativePath = "Toolchains/XcodeDefault.xctoolchain/usr/lib/swift-5.0" swiftLibsDir = os.path.join(GetXcodeDeveloperPath(), relativePath) diff --git a/test_runner/logic_test_util.py b/test_runner/logic_test_util.py index b2c7de3..60fac3a 100644 --- a/test_runner/logic_test_util.py +++ b/test_runner/logic_test_util.py @@ -49,11 +49,10 @@ def RunLogicTestOnSim( simctl_env_vars[_SIMCTL_ENV_VAR_PREFIX + key] = env_vars[key] simctl_env_vars['NSUnbufferedIO'] = 'YES' - # Xcode 11+'s Swift dylibs are configured in a way that does not allow them to load the correct - # libswiftFoundation.dylib file from libXCTestSwiftSupport.dylib. This bug only affects tests that - # run on simulators running iOS 12.1 or lower. To fix this bug, we provide an explicit fallback to - # the correct Swift dylibs that have been packaged with Xcode. - # See https://github.com/bazelbuild/rules_apple/issues/684 for context. + # Fixes failures for unit test targets that depend on Swift libraries when running with Xcode 11 + # on pre-iOS 12.2 simulators. + # Example failure message this resolves: "The bundle couldn’t be loaded because it is damaged + # or missing necessary resources." swift5FallbackLibsDir = xcode_info_util.GetSwift5FallbackLibsDir() if swift5FallbackLibsDir: simctl_env_vars[_SIMCTL_ENV_VAR_PREFIX + "DYLD_FALLBACK_LIBRARY_PATH"] = swift5FallbackLibsDir diff --git a/test_runner/xctestrun.py b/test_runner/xctestrun.py index c46ad5b..8435b0d 100644 --- a/test_runner/xctestrun.py +++ b/test_runner/xctestrun.py @@ -475,11 +475,11 @@ def _GenerateTestRootForXcuitest(self): developer=developer_path), 'DYLD_LIBRARY_PATH': '__TESTROOT__:%s/usr/lib' % developer_path } - # Xcode 11+'s Swift dylibs are configured in a way that does not allow them to load the correct - # libswiftFoundation.dylib file from libXCTestSwiftSupport.dylib. This bug only affects tests that - # run on simulators running iOS 12.1 or lower. To fix this bug, we provide an explicit fallback to - # the correct Swift dylibs that have been packaged with Xcode. - # See https://github.com/bazelbuild/rules_apple/issues/684 for context. + + # Fixes failures for UI test targets that depend on Swift libraries when running with Xcode 11 + # on pre-iOS 12.2 simulators. + # Example failure message this resolves: "The bundle couldn’t be loaded because it is damaged + # or missing necessary resources." swift5FallbackLibsDir = xcode_info_util.GetSwift5FallbackLibsDir() if swift5FallbackLibsDir: test_envs["DYLD_FALLBACK_LIBRARY_PATH"] = swift5FallbackLibsDir @@ -622,11 +622,11 @@ def _GenerateTestRootForXctest(self): 'DYLD_INSERT_LIBRARIES': dyld_insert_libs, 'DYLD_LIBRARY_PATH': '__TESTROOT__:%s/usr/lib:' % developer_path } - # Xcode 11+'s Swift dylibs are configured in a way that does not allow them to load the correct - # libswiftFoundation.dylib file from libXCTestSwiftSupport.dylib. This bug only affects tests that - # run on simulators running iOS 12.1 or lower. To fix this bug, we provide an explicit fallback to - # the correct Swift dylibs that have been packaged with Xcode. - # See https://github.com/bazelbuild/rules_apple/issues/684 for context. + + # Fixes failures for test targets that depend on Swift libraries when running with Xcode 11 + # on pre-iOS 12.2 simulators. + # Example failure message this resolves: "The bundle couldn’t be loaded because it is damaged + # or missing necessary resources." swift5FallbackLibsDir = xcode_info_util.GetSwift5FallbackLibsDir() if swift5FallbackLibsDir: test_envs["DYLD_FALLBACK_LIBRARY_PATH"] = swift5FallbackLibsDir @@ -651,11 +651,11 @@ def _GenerateTestRootForLogicTest(self): 'DYLD_FRAMEWORK_PATH': dyld_framework_path, 'DYLD_LIBRARY_PATH': dyld_framework_path } - # Xcode 11+'s Swift dylibs are configured in a way that does not allow them to load the correct - # libswiftFoundation.dylib file from libXCTestSwiftSupport.dylib. This bug only affects tests that - # run on simulators running iOS 12.1 or lower. To fix this bug, we provide an explicit fallback to - # the correct Swift dylibs that have been packaged with Xcode. - # See https://github.com/bazelbuild/rules_apple/issues/684 for context. + + # Fixes failures for unit test targets that depend on Swift libraries when running with Xcode 11 + # on pre-iOS 12.2 simulators. + # Example failure message this resolves: "The bundle couldn’t be loaded because it is damaged + # or missing necessary resources." swift5FallbackLibsDir = xcode_info_util.GetSwift5FallbackLibsDir() if swift5FallbackLibsDir: test_envs["DYLD_FALLBACK_LIBRARY_PATH"] = swift5FallbackLibsDir From d2b9ed91fbe1fdf3d13a32010f5d67b29f100080 Mon Sep 17 00:00:00 2001 From: Weiming Dai Date: Sun, 9 Feb 2020 17:14:19 -0800 Subject: [PATCH 04/45] Refactor the code structure Revert the code structure change in commit aa0e121d3d9c7b24443a253c0fe6575d4d2fa930 --- README.md | 8 ++++++++ BUILD => xctestrunner/BUILD | 0 __init__.py => xctestrunner/__init__.py | 0 {shared => xctestrunner/shared}/__init__.py | 0 {shared => xctestrunner/shared}/bundle_util.py | 0 {shared => xctestrunner/shared}/ios_constants.py | 0 {shared => xctestrunner/shared}/ios_errors.py | 0 {shared => xctestrunner/shared}/plist_util.py | 0 {shared => xctestrunner/shared}/provisioning_profile.py | 0 {shared => xctestrunner/shared}/xcode_info_util.py | 2 +- .../simulator_control}/__init__.py | 0 .../simulator_control}/simtype_profile.py | 0 .../simulator_control}/simulator_util.py | 0 .../TestProject/TestProject.xcodeproj/project.pbxproj | 0 .../xcshareddata/xcschemes/TestProjectXctest.xcscheme | 0 .../xcshareddata/xcschemes/TestProjectXcuitest.xcscheme | 0 {test_runner => xctestrunner/test_runner}/__init__.py | 0 .../test_runner}/dummy_project.py | 0 .../test_runner}/ios_test_runner.py | 0 .../test_runner}/logic_test_util.py | 0 .../test_runner}/runner_exit_codes.py | 0 .../test_runner}/test_summaries_util.py | 0 .../test_runner}/xcodebuild_test_executor.py | 0 .../test_runner}/xctest_session.py | 0 {test_runner => xctestrunner/test_runner}/xctestrun.py | 0 25 files changed, 9 insertions(+), 1 deletion(-) rename BUILD => xctestrunner/BUILD (100%) rename __init__.py => xctestrunner/__init__.py (100%) rename {shared => xctestrunner/shared}/__init__.py (100%) rename {shared => xctestrunner/shared}/bundle_util.py (100%) rename {shared => xctestrunner/shared}/ios_constants.py (100%) rename {shared => xctestrunner/shared}/ios_errors.py (100%) rename {shared => xctestrunner/shared}/plist_util.py (100%) rename {shared => xctestrunner/shared}/provisioning_profile.py (100%) rename {shared => xctestrunner/shared}/xcode_info_util.py (98%) rename {simulator_control => xctestrunner/simulator_control}/__init__.py (100%) rename {simulator_control => xctestrunner/simulator_control}/simtype_profile.py (100%) rename {simulator_control => xctestrunner/simulator_control}/simulator_util.py (100%) rename {test_runner => xctestrunner/test_runner}/TestProject/TestProject.xcodeproj/project.pbxproj (100%) rename {test_runner => xctestrunner/test_runner}/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXctest.xcscheme (100%) rename {test_runner => xctestrunner/test_runner}/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXcuitest.xcscheme (100%) rename {test_runner => xctestrunner/test_runner}/__init__.py (100%) rename {test_runner => xctestrunner/test_runner}/dummy_project.py (100%) rename {test_runner => xctestrunner/test_runner}/ios_test_runner.py (100%) rename {test_runner => xctestrunner/test_runner}/logic_test_util.py (100%) rename {test_runner => xctestrunner/test_runner}/runner_exit_codes.py (100%) rename {test_runner => xctestrunner/test_runner}/test_summaries_util.py (100%) rename {test_runner => xctestrunner/test_runner}/xcodebuild_test_executor.py (100%) rename {test_runner => xctestrunner/test_runner}/xctest_session.py (100%) rename {test_runner => xctestrunner/test_runner}/xctestrun.py (100%) diff --git a/README.md b/README.md index 1f78dcf..fd43ab8 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,14 @@ environment variables, additional arguments. ## Installation You can download the ios_test_runner.par binary in [release](https://github.com/google/xctestrunner/releases) +or build the ios_test_runner.par binary by bazel: +``` +$ git clone https://github.com/google/xctestrunner.git +$ cd xctestrunner +$ bazel build xctestrunner:ios_test_runner.par +$ ls bazel-bin/xctestrunner/ios_test_runner.par +``` + ## Usage - Build your app under test and test bundle. You can use Xcode.app, `xcodebuild` command line tool or [bazel](https://github.com/bazelbuild/bazel). diff --git a/BUILD b/xctestrunner/BUILD similarity index 100% rename from BUILD rename to xctestrunner/BUILD diff --git a/__init__.py b/xctestrunner/__init__.py similarity index 100% rename from __init__.py rename to xctestrunner/__init__.py diff --git a/shared/__init__.py b/xctestrunner/shared/__init__.py similarity index 100% rename from shared/__init__.py rename to xctestrunner/shared/__init__.py diff --git a/shared/bundle_util.py b/xctestrunner/shared/bundle_util.py similarity index 100% rename from shared/bundle_util.py rename to xctestrunner/shared/bundle_util.py diff --git a/shared/ios_constants.py b/xctestrunner/shared/ios_constants.py similarity index 100% rename from shared/ios_constants.py rename to xctestrunner/shared/ios_constants.py diff --git a/shared/ios_errors.py b/xctestrunner/shared/ios_errors.py similarity index 100% rename from shared/ios_errors.py rename to xctestrunner/shared/ios_errors.py diff --git a/shared/plist_util.py b/xctestrunner/shared/plist_util.py similarity index 100% rename from shared/plist_util.py rename to xctestrunner/shared/plist_util.py diff --git a/shared/provisioning_profile.py b/xctestrunner/shared/provisioning_profile.py similarity index 100% rename from shared/provisioning_profile.py rename to xctestrunner/shared/provisioning_profile.py diff --git a/shared/xcode_info_util.py b/xctestrunner/shared/xcode_info_util.py similarity index 98% rename from shared/xcode_info_util.py rename to xctestrunner/shared/xcode_info_util.py index d20db58..9637f55 100644 --- a/shared/xcode_info_util.py +++ b/xctestrunner/shared/xcode_info_util.py @@ -17,7 +17,7 @@ import os import subprocess -from shared import ios_constants +from xctestrunner.shared import ios_constants _xcode_version_number = None diff --git a/simulator_control/__init__.py b/xctestrunner/simulator_control/__init__.py similarity index 100% rename from simulator_control/__init__.py rename to xctestrunner/simulator_control/__init__.py diff --git a/simulator_control/simtype_profile.py b/xctestrunner/simulator_control/simtype_profile.py similarity index 100% rename from simulator_control/simtype_profile.py rename to xctestrunner/simulator_control/simtype_profile.py diff --git a/simulator_control/simulator_util.py b/xctestrunner/simulator_control/simulator_util.py similarity index 100% rename from simulator_control/simulator_util.py rename to xctestrunner/simulator_control/simulator_util.py diff --git a/test_runner/TestProject/TestProject.xcodeproj/project.pbxproj b/xctestrunner/test_runner/TestProject/TestProject.xcodeproj/project.pbxproj similarity index 100% rename from test_runner/TestProject/TestProject.xcodeproj/project.pbxproj rename to xctestrunner/test_runner/TestProject/TestProject.xcodeproj/project.pbxproj diff --git a/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXctest.xcscheme b/xctestrunner/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXctest.xcscheme similarity index 100% rename from test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXctest.xcscheme rename to xctestrunner/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXctest.xcscheme diff --git a/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXcuitest.xcscheme b/xctestrunner/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXcuitest.xcscheme similarity index 100% rename from test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXcuitest.xcscheme rename to xctestrunner/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXcuitest.xcscheme diff --git a/test_runner/__init__.py b/xctestrunner/test_runner/__init__.py similarity index 100% rename from test_runner/__init__.py rename to xctestrunner/test_runner/__init__.py diff --git a/test_runner/dummy_project.py b/xctestrunner/test_runner/dummy_project.py similarity index 100% rename from test_runner/dummy_project.py rename to xctestrunner/test_runner/dummy_project.py diff --git a/test_runner/ios_test_runner.py b/xctestrunner/test_runner/ios_test_runner.py similarity index 100% rename from test_runner/ios_test_runner.py rename to xctestrunner/test_runner/ios_test_runner.py diff --git a/test_runner/logic_test_util.py b/xctestrunner/test_runner/logic_test_util.py similarity index 100% rename from test_runner/logic_test_util.py rename to xctestrunner/test_runner/logic_test_util.py diff --git a/test_runner/runner_exit_codes.py b/xctestrunner/test_runner/runner_exit_codes.py similarity index 100% rename from test_runner/runner_exit_codes.py rename to xctestrunner/test_runner/runner_exit_codes.py diff --git a/test_runner/test_summaries_util.py b/xctestrunner/test_runner/test_summaries_util.py similarity index 100% rename from test_runner/test_summaries_util.py rename to xctestrunner/test_runner/test_summaries_util.py diff --git a/test_runner/xcodebuild_test_executor.py b/xctestrunner/test_runner/xcodebuild_test_executor.py similarity index 100% rename from test_runner/xcodebuild_test_executor.py rename to xctestrunner/test_runner/xcodebuild_test_executor.py diff --git a/test_runner/xctest_session.py b/xctestrunner/test_runner/xctest_session.py similarity index 100% rename from test_runner/xctest_session.py rename to xctestrunner/test_runner/xctest_session.py diff --git a/test_runner/xctestrun.py b/xctestrunner/test_runner/xctestrun.py similarity index 100% rename from test_runner/xctestrun.py rename to xctestrunner/test_runner/xctestrun.py From aa821f33cc501f916678cc6503da2a169bb9b496 Mon Sep 17 00:00:00 2001 From: Weiming Dai Date: Sun, 9 Feb 2020 18:20:10 -0800 Subject: [PATCH 05/45] Update test runner with internal change - Fix the minSupportedVersion issue for simulator creation. - Allow running arm64 test bundle on arm64e iOS device. --- xctestrunner/shared/bundle_util.py | 12 ++++ xctestrunner/shared/ios_constants.py | 7 ++ xctestrunner/shared/xcode_info_util.py | 1 + .../simulator_control/simtype_profile.py | 38 ++++++----- .../simulator_control/simulator_util.py | 68 +++++++++++-------- xctestrunner/test_runner/ios_test_runner.py | 18 ++++- .../test_runner/xcodebuild_test_executor.py | 10 ++- xctestrunner/test_runner/xctest_session.py | 8 ++- xctestrunner/test_runner/xctestrun.py | 22 ++++-- 9 files changed, 131 insertions(+), 53 deletions(-) diff --git a/xctestrunner/shared/bundle_util.py b/xctestrunner/shared/bundle_util.py index 324fde8..343315d 100644 --- a/xctestrunner/shared/bundle_util.py +++ b/xctestrunner/shared/bundle_util.py @@ -215,6 +215,18 @@ def EnableUIFileSharing(bundle_path, resigning=True): CodesignBundle(bundle_path) +def GetFileArchTypes(file_path): + """Gets the architecture types of the file.""" + output = subprocess.check_output(['/usr/bin/lipo', file_path, '-archs']) + return output.split(' ') + + +def RemoveArchType(file_path, arch_type): + """Remove the given architecture types for the file.""" + subprocess.check_call( + ['/usr/bin/lipo', file_path, '-remove', arch_type, '-output', file_path]) + + def _ExtractBundleFile(target_dir, bundle_extension): """Extract single bundle file with given extension. diff --git a/xctestrunner/shared/ios_constants.py b/xctestrunner/shared/ios_constants.py index d28d97e..2136b1c 100644 --- a/xctestrunner/shared/ios_constants.py +++ b/xctestrunner/shared/ios_constants.py @@ -19,6 +19,13 @@ def enum(**enums): return type('Enum', (), enums) +ARCH = enum( + ARMV7='armv7', + ARMV7S='armv7s', + ARM64='arm64', + ARM64E='arm64e', + I386='i386', + X86_64='x86_64') SDK = enum(IPHONEOS='iphoneos', IPHONESIMULATOR='iphonesimulator') # It is consistent with bazel's apple platform: # https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/rules/apple/ApplePlatform.java diff --git a/xctestrunner/shared/xcode_info_util.py b/xctestrunner/shared/xcode_info_util.py index 9637f55..d6f0725 100644 --- a/xctestrunner/shared/xcode_info_util.py +++ b/xctestrunner/shared/xcode_info_util.py @@ -34,6 +34,7 @@ def GetXcodeDeveloperPath(): # path to that fallback directory. # See https://github.com/bazelbuild/rules_apple/issues/684 for context. def GetSwift5FallbackLibsDir(): + """Gets the directory for Swift5 fallback libraries.""" relativePath = "Toolchains/XcodeDefault.xctoolchain/usr/lib/swift-5.0" swiftLibsDir = os.path.join(GetXcodeDeveloperPath(), relativePath) swiftLibPlatformDir = os.path.join(swiftLibsDir, ios_constants.SDK.IPHONESIMULATOR) diff --git a/xctestrunner/simulator_control/simtype_profile.py b/xctestrunner/simulator_control/simtype_profile.py index 1b51cb8..dc4ba21 100644 --- a/xctestrunner/simulator_control/simtype_profile.py +++ b/xctestrunner/simulator_control/simtype_profile.py @@ -47,13 +47,14 @@ def profile_plist_obj(self): profile.plist. """ if not self._profile_plist_obj: - if xcode_info_util.GetXcodeVersionNumber() >= 900: + xcode_version = xcode_info_util.GetXcodeVersionNumber() + if xcode_version >= 900: platform_path = xcode_info_util.GetSdkPlatformPath( ios_constants.SDK.IPHONEOS) else: platform_path = xcode_info_util.GetSdkPlatformPath( ios_constants.SDK.IPHONESIMULATOR) - if xcode_info_util.GetXcodeVersionNumber() >= 1100: + if xcode_version >= 1100: sim_profiles_dir = os.path.join( platform_path, 'Library/Developer/CoreSimulator/Profiles') else: @@ -71,14 +72,12 @@ def min_os_version(self): """Gets the min supported OS version. Returns: - string, the min supported OS version. + float, the min supported OS version. """ if not self._min_os_version: - min_os_version = self.profile_plist_obj.GetPlistField('minRuntimeVersion') - # Cut build version. E.g., cut 9.3.3 to 9.3. - if min_os_version.count('.') > 1: - min_os_version = min_os_version[:min_os_version.rfind('.')] - self._min_os_version = min_os_version + min_os_version_str = self.profile_plist_obj.GetPlistField( + 'minRuntimeVersion') + self._min_os_version = _extra_os_version(min_os_version_str) return self._min_os_version @property @@ -86,19 +85,26 @@ def max_os_version(self): """Gets the max supported OS version. Returns: - string, the max supported OS version. + float, the max supported OS version or None if it is not found. """ if not self._max_os_version: # If the profile.plist does not have maxRuntimeVersion field, it means # it supports the max OS version of current iphonesimulator platform. try: - max_os_version = self.profile_plist_obj.GetPlistField( + max_os_version_str = self.profile_plist_obj.GetPlistField( 'maxRuntimeVersion') except ios_errors.PlistError: - max_os_version = xcode_info_util.GetSdkVersion( - ios_constants.SDK.IPHONESIMULATOR) - # Cut build version. E.g., cut 9.3.3 to 9.3. - if max_os_version.count('.') > 1: - max_os_version = max_os_version[:max_os_version.rfind('.')] - self._max_os_version = max_os_version + return None + self._max_os_version = _extra_os_version(max_os_version_str) return self._max_os_version + + +def _extra_os_version(os_version_str): + """Extracts os version float value from a given string.""" + # Cut build version. E.g., cut 9.3.3 to 9.3. + if os_version_str.count('.') > 1: + os_version_str = os_version_str[:os_version_str.rfind('.')] + # We need to round the os version string in the simulator profile. E.g., + # the maxRuntimeVersion of iPhone 5 is 10.255.255 and we could create iOS 10.3 + # for iPhone 5. + return round(float(os_version_str), 1) diff --git a/xctestrunner/simulator_control/simulator_util.py b/xctestrunner/simulator_control/simulator_util.py index 729e370..177b21a 100644 --- a/xctestrunner/simulator_control/simulator_util.py +++ b/xctestrunner/simulator_control/simulator_util.py @@ -134,12 +134,14 @@ def Shutdown(self): self.WaitUntilStateShutdown() logging.info('Shut down simulator %s.', self.simulator_id) - def Delete(self): - """Deletes the simulator asynchronously. + def Delete(self, asynchronously=True): + """Deletes the simulator. The simulator state should be SHUTDOWN when deleting it. Otherwise, it will raise exception. + Args: + asynchronously: whether deleting the simulator asynchronously. Raises: ios_errors.SimError: The simulator's state is not SHUTDOWN. """ @@ -151,11 +153,21 @@ def Delete(self): raise ios_errors.SimError( 'Can only delete the simulator with state SHUTDOWN. The current ' 'state of simulator %s is %s.' % (self._simulator_id, sim_state)) - logging.info('Deleting simulator %s asynchronously.', self.simulator_id) - subprocess.Popen(['xcrun', 'simctl', 'delete', self.simulator_id], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - preexec_fn=os.setpgrp) + command = ['xcrun', 'simctl', 'delete', self.simulator_id] + if asynchronously: + logging.info('Deleting simulator %s asynchronously.', self.simulator_id) + subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=os.setpgrp) + else: + try: + RunSimctlCommand(command) + logging.info('Deleted simulator %s.', self.simulator_id) + except ios_errors.SimError as e: + raise ios_errors.SimError('Failed to delete simulator %s: %s' % + (self.simulator_id, str(e))) # The delete command won't delete the simulator log directory. if os.path.exists(self.simulator_log_root_dir): shutil.rmtree(self.simulator_log_root_dir, ignore_errors=True) @@ -413,9 +425,8 @@ def GetLastSupportedIphoneSimType(os_version): os_version_float = float(os_version) for sim_type in supported_sim_types: if sim_type.startswith('iPhone'): - min_os_version_float = float( - simtype_profile.SimTypeProfile(sim_type).min_os_version) - if os_version_float >= min_os_version_float: + min_os_version = simtype_profile.SimTypeProfile(sim_type).min_os_version + if os_version_float >= min_os_version: return sim_type raise ios_errors.SimError('Can not find supported iPhone simulator type.') @@ -520,17 +531,19 @@ def GetLastSupportedSimOsVersion(os_type=ios_constants.OS.IOS, if not device_type: return supported_os_versions[-1] - simtype_max_os_version_float = float( - simtype_profile.SimTypeProfile(device_type).max_os_version) + max_os_version = simtype_profile.SimTypeProfile(device_type).max_os_version + # The supported os versions will be from latest to older after reverse(). supported_os_versions.reverse() + if not max_os_version: + return supported_os_versions[0] + for os_version in supported_os_versions: - if float(os_version) <= simtype_max_os_version_float: + if float(os_version) <= max_os_version: return os_version - if not supported_os_versions: - raise ios_errors.IllegalArgumentError( - 'The supported OS version %s can not match simulator type %s. Because ' - 'its max OS version is %s' % - (supported_os_versions, device_type, simtype_max_os_version_float)) + raise ios_errors.IllegalArgumentError( + 'The supported OS version %s can not match simulator type %s. Because ' + 'its max OS version is %s' % + (supported_os_versions, device_type, max_os_version)) def GetOsType(device_type): @@ -598,16 +611,17 @@ def _ValidateSimulatorTypeWithOsVersion(device_type, os_version): """ os_version_float = float(os_version) sim_profile = simtype_profile.SimTypeProfile(device_type) - min_os_version_float = float(sim_profile.min_os_version) - if min_os_version_float > os_version_float: + min_os_version = sim_profile.min_os_version + if min_os_version > os_version_float: raise ios_errors.IllegalArgumentError( - 'The min OS version of %s is %s. But current OS version is %s' % - (device_type, min_os_version_float, os_version)) - max_os_version_float = float(sim_profile.max_os_version) - if max_os_version_float < os_version_float: - raise ios_errors.IllegalArgumentError( - 'The max OS version of %s is %s. But current OS version is %s' % - (device_type, max_os_version_float, os_version)) + 'The min OS version of %s is %f. But current OS version is %s' % + (device_type, min_os_version, os_version)) + max_os_version = sim_profile.max_os_version + if max_os_version: + if max_os_version < os_version_float: + raise ios_errors.IllegalArgumentError( + 'The max OS version of %s is %f. But current OS version is %s' % + (device_type, max_os_version, os_version)) def QuitSimulatorApp(): diff --git a/xctestrunner/test_runner/ios_test_runner.py b/xctestrunner/test_runner/ios_test_runner.py index f2a2f81..a69f827 100644 --- a/xctestrunner/test_runner/ios_test_runner.py +++ b/xctestrunner/test_runner/ios_test_runner.py @@ -107,8 +107,12 @@ def _AddTestSubParser(subparsers): def _Test(args): """The function of sub command `test`.""" sdk = _PlatformToSdk(args.platform) if args.platform else _GetSdk(args.id) + device_arch = _GetDeviceArch(args.id, sdk) with xctest_session.XctestSession( - sdk=sdk, work_dir=args.work_dir, output_dir=args.output_dir) as session: + sdk=sdk, + device_arch=device_arch, + work_dir=args.work_dir, + output_dir=args.output_dir) as session: session.Prepare( app_under_test=args.app_under_test_path, test_bundle=args.test_bundle_path, @@ -142,6 +146,7 @@ def _RunSimulatorTest(args): """The function of running test with new simulator.""" with xctest_session.XctestSession( sdk=ios_constants.SDK.IPHONESIMULATOR, + device_arch=ios_constants.ARCH.X86_64, work_dir=args.work_dir, output_dir=args.output_dir) as session: session.Prepare( app_under_test=args.app_under_test_path, @@ -293,6 +298,17 @@ def _GetSdk(device_id): (device_id, known_devices_output)) +def _GetDeviceArch(device_id, sdk): + """Gets the device architecture.""" + # It is a temporary soluton to get device architecture. Checking i386 and + # armv7/armv7s is not supported. + if sdk == ios_constants.SDK.IPHONESIMULATOR: + return ios_constants.ARCH.X86_64 + if '-' in device_id: + return ios_constants.ARCH.ARM64E + return ios_constants.ARCH.ARM64 + + def main(argv): args = _BuildParser().parse_args(argv[1:]) if args.verbose: diff --git a/xctestrunner/test_runner/xcodebuild_test_executor.py b/xctestrunner/test_runner/xcodebuild_test_executor.py index 4665db4..c4e3800 100644 --- a/xctestrunner/test_runner/xcodebuild_test_executor.py +++ b/xctestrunner/test_runner/xcodebuild_test_executor.py @@ -49,6 +49,7 @@ _TOO_MANY_INSTANCES_ALREADY_RUNNING = ('Too many instances of this service are ' 'already running.') _LOST_CONNECTION_ERROR = 'Lost connection to testmanagerd' +_LOST_CONNECTION_TO_DTSERVICEHUB_ERROR = 'Lost connection to DTServiceHub' class CheckXcodebuildStuckThread(threading.Thread): @@ -211,9 +212,14 @@ def Execute(self, return_output=True): output_str = output.getvalue() if self._sdk == ios_constants.SDK.IPHONEOS: if ((re.search(_DEVICE_TYPE_WAS_NULL_PATTERN, output_str) or - _LOST_CONNECTION_ERROR in output_str) and i < max_attempts - 1): + _LOST_CONNECTION_ERROR in output_str or + _LOST_CONNECTION_TO_DTSERVICEHUB_ERROR in output_str) and + i < max_attempts - 1): logging.warning( - 'Failed to launch test on the device. Will relaunch again.') + 'Failed to launch test on the device. Will relaunch again ' + 'after 5s.' + ) + time.sleep(5) continue if _TOO_MANY_INSTANCES_ALREADY_RUNNING in output_str: return (runner_exit_codes.EXITCODE.NEED_REBOOT_DEVICE, diff --git a/xctestrunner/test_runner/xctest_session.py b/xctestrunner/test_runner/xctest_session.py index f8badef..6e3751f 100644 --- a/xctestrunner/test_runner/xctest_session.py +++ b/xctestrunner/test_runner/xctest_session.py @@ -34,7 +34,7 @@ class XctestSession(object): """The class that runs XCTEST based tests.""" - def __init__(self, sdk, work_dir=None, output_dir=None): + def __init__(self, sdk, device_arch, work_dir=None, output_dir=None): """Initializes the XctestSession object. If work_dir is not provdied, will create a temp direcotry to be work_dir and @@ -43,6 +43,7 @@ def __init__(self, sdk, work_dir=None, output_dir=None): Args: sdk: ios_constants.SDK. The sdk of the target device. + device_arch: ios_constants.ARCH. The architecture of the target device. work_dir: string, the working directory contains runfiles. output_dir: string, The directory where derived data will go, including: 1) the detailed test session log which includes test output and the @@ -51,6 +52,7 @@ def __init__(self, sdk, work_dir=None, output_dir=None): specified, the directory will not be deleted after test ends.' """ self._sdk = sdk + self._device_arch = device_arch self._work_dir = work_dir self._delete_work_dir = True self._output_dir = output_dir @@ -146,8 +148,8 @@ def Prepare(self, app_under_test=None, test_bundle=None, test_type != ios_constants.TestType.LOGIC_TEST and xcode_info_util.GetXcodeVersionNumber() >= 800): xctestrun_factory = xctestrun.XctestRunFactory( - app_under_test_dir, test_bundle_dir, self._sdk, test_type, - signing_options, self._work_dir) + app_under_test_dir, test_bundle_dir, self._sdk, self._device_arch, + test_type, signing_options, self._work_dir) self._xctestrun_obj = xctestrun_factory.GenerateXctestrun() elif test_type == ios_constants.TestType.XCUITEST: raise ios_errors.IllegalArgumentError( diff --git a/xctestrunner/test_runner/xctestrun.py b/xctestrunner/test_runner/xctestrun.py index 8435b0d..0e306bc 100644 --- a/xctestrunner/test_runner/xctestrun.py +++ b/xctestrunner/test_runner/xctestrun.py @@ -251,6 +251,7 @@ class XctestRunFactory(object): def __init__(self, app_under_test_dir, test_bundle_dir, sdk=ios_constants.SDK.IPHONESIMULATOR, + device_arch=ios_constants.ARCH.X86_64, test_type=ios_constants.TestType.XCUITEST, signing_options=None, work_dir=None): """Initializes the XctestRun object. @@ -263,6 +264,7 @@ def __init__(self, app_under_test_dir, test_bundle_dir, test_bundle_dir: string, path of the test bundle. sdk: string, SDKRoot of the test. See supported SDKs in module xctestrunner.shared.ios_constants. + device_arch: ios_constants.ARCH. The architecture of the target device. test_type: string, test type of the test bundle. See supported test types in module xctestrunner.shared.ios_constants. signing_options: dict, the signing app options. See @@ -276,6 +278,7 @@ def __init__(self, app_under_test_dir, test_bundle_dir, self._test_bundle_dir = test_bundle_dir self._test_name = os.path.splitext(os.path.basename(test_bundle_dir))[0] self._sdk = sdk + self._device_arch = device_arch self._test_type = test_type if self._sdk == ios_constants.SDK.IPHONEOS: self._on_device = True @@ -468,7 +471,7 @@ def _GenerateTestRootForXcuitest(self): bundle_util.CodesignBundle(self._app_under_test_dir) platform_name = 'iPhoneOS' if self._on_device else 'iPhoneSimulator' - developer_path = '__PLATFORMS__/%s.platform/Developer/' % platform_name + developer_path = '__PLATFORMS__/%s.platform/Developer' % platform_name test_envs = { 'DYLD_FRAMEWORK_PATH': '__TESTROOT__:{developer}/Library/Frameworks:' '{developer}/Library/PrivateFrameworks'.format( @@ -516,9 +519,20 @@ def _GetUitestRunnerAppFromXcode(self, platform_library_path): uitest_runner_app = os.path.join(self._test_root_dir, uitest_runner_app_name + '.app') shutil.copytree(xctrunner_app, uitest_runner_app) + uitest_runner_exec = os.path.join(uitest_runner_app, uitest_runner_app_name) shutil.move( - os.path.join(uitest_runner_app, 'XCTRunner'), - os.path.join(uitest_runner_app, uitest_runner_app_name)) + os.path.join(uitest_runner_app, 'XCTRunner'), uitest_runner_exec) + # XCTRunner is multi-archs. When launching XCTRunner on arm64e device, it + # will be launched as arm64e process by default. If the test bundle is arm64 + # bundle, the XCTRunner which hosts the test bundle will failed to be + # launched. So removing the arm64e arch from XCTRunner can resolve this + # case. + if self._device_arch == ios_constants.ARCH.ARM64E: + test_executable = os.path.join(self._test_bundle_dir, test_bundle_name) + test_archs = bundle_util.GetFileArchTypes(test_executable) + if ios_constants.ARCH.ARM64E not in test_archs: + bundle_util.RemoveArchType(uitest_runner_exec, + ios_constants.ARCH.ARM64E) runner_app_info_plist_path = os.path.join(uitest_runner_app, 'Info.plist') info_plist = plist_util.Plist(runner_app_info_plist_path) @@ -606,7 +620,7 @@ def _GenerateTestRootForXctest(self): app_under_test_name = os.path.splitext( os.path.basename(self._app_under_test_dir))[0] platform_name = 'iPhoneOS' if self._on_device else 'iPhoneSimulator' - developer_path = '__PLATFORMS__/%s.platform/Developer/' % platform_name + developer_path = '__PLATFORMS__/%s.platform/Developer' % platform_name if xcode_info_util.GetXcodeVersionNumber() < 1000: dyld_insert_libs = ('%s/Library/PrivateFrameworks/' 'IDEBundleInjection.framework/IDEBundleInjection' % From 3776f417f67b6ae78f07fe7ccd0433e7432d6955 Mon Sep 17 00:00:00 2001 From: Weiming Dai Date: Thu, 30 Apr 2020 21:05:25 -0700 Subject: [PATCH 06/45] Update test runner with internal change - Check the OS version when passing the Swift5 fallback libraries (required for Xcode 11.4 or later) - Parse the xcresult bundle under Xcode 11 or later. - Remove the Xcode 7 support. --- README.md | 4 +- xctestrunner/shared/ios_errors.py | 4 + xctestrunner/shared/version_util.py | 26 + xctestrunner/shared/xcode_info_util.py | 33 +- .../simulator_control/simulator_util.py | 1 - .../TestProject.xcodeproj/project.pbxproj | 342 ----------- .../xcschemes/TestProjectXctest.xcscheme | 104 ---- .../xcschemes/TestProjectXcuitest.xcscheme | 105 ---- xctestrunner/test_runner/dummy_project.py | 571 ------------------ xctestrunner/test_runner/logic_test_util.py | 27 +- .../test_runner/test_summaries_util.py | 102 ---- xctestrunner/test_runner/xcresult_util.py | 45 ++ xctestrunner/test_runner/xctest_session.py | 86 +-- xctestrunner/test_runner/xctestrun.py | 43 +- 14 files changed, 150 insertions(+), 1343 deletions(-) create mode 100644 xctestrunner/shared/version_util.py delete mode 100755 xctestrunner/test_runner/TestProject/TestProject.xcodeproj/project.pbxproj delete mode 100755 xctestrunner/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXctest.xcscheme delete mode 100755 xctestrunner/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXcuitest.xcscheme delete mode 100644 xctestrunner/test_runner/dummy_project.py delete mode 100644 xctestrunner/test_runner/test_summaries_util.py create mode 100644 xctestrunner/test_runner/xcresult_util.py diff --git a/README.md b/README.md index fd43ab8..7945cf6 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,10 @@ A tool for running prebuilt iOS tests on iOS real device and simulator. - It supports iOS 7+ iOS real device, iOS simulator. - It supports launch options configuration: test methods to run, additional environment variables, additional arguments. -- It supports Xcode 7+. +- It supports Xcode 8+. ## Prerequisites -- Install Xcode (Xcode 7+). XCUITest support requires Xcode 8+. +- Install Xcode (Xcode 8+). XCUITest support requires Xcode 8+. - [Install bazel](https://docs.bazel.build/install.html) (optional). - py module [biplist](https://github.com/wooster/biplist). diff --git a/xctestrunner/shared/ios_errors.py b/xctestrunner/shared/ios_errors.py index 308065e..cdddc48 100644 --- a/xctestrunner/shared/ios_errors.py +++ b/xctestrunner/shared/ios_errors.py @@ -45,3 +45,7 @@ class SimError(Exception): class XcodebuildTestError(Exception): """Exception class for simulator error.""" + + +class XcresultError(Exception): + """Exception class for parsing xcresult error.""" diff --git a/xctestrunner/shared/version_util.py b/xctestrunner/shared/version_util.py new file mode 100644 index 0000000..a6a5e65 --- /dev/null +++ b/xctestrunner/shared/version_util.py @@ -0,0 +1,26 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utility methods for Apple version.""" + + +def GetVersionNumber(version_str): + """Gets the version number of the given version string.""" + parts = version_str.split('.') + version_number = int(parts[0]) * 100 + if len(parts) > 1: + version_number += int(parts[1]) * 10 + if len(parts) > 2: + version_number += int(parts[2]) + return version_number diff --git a/xctestrunner/shared/xcode_info_util.py b/xctestrunner/shared/xcode_info_util.py index d6f0725..ed4ecab 100644 --- a/xctestrunner/shared/xcode_info_util.py +++ b/xctestrunner/shared/xcode_info_util.py @@ -18,6 +18,8 @@ import subprocess from xctestrunner.shared import ios_constants +from xctestrunner.shared import version_util + _xcode_version_number = None @@ -27,19 +29,20 @@ def GetXcodeDeveloperPath(): return subprocess.check_output(('xcode-select', '-p')).strip() -# Xcode 11+'s Swift dylibs are configured in a way that does not allow them to load the correct -# libswiftFoundation.dylib file from libXCTestSwiftSupport.dylib. This bug only affects tests that -# run on simulators running iOS 12.1 or lower. To fix this bug, we need to provide explicit -# fallbacks to the correct Swift dylibs that have been packaged with Xcode. This method returns the -# path to that fallback directory. +# Xcode 11+'s Swift dylibs are configured in a way that does not allow them to +# load the correct libswiftFoundation.dylib file from +# libXCTestSwiftSupport.dylib. This bug only affects tests that run on fallbacks +# to the correct Swift dylibs that have been packaged with Xcode. This method +# returns the path to that fallback directory. # See https://github.com/bazelbuild/rules_apple/issues/684 for context. def GetSwift5FallbackLibsDir(): - """Gets the directory for Swift5 fallback libraries.""" - relativePath = "Toolchains/XcodeDefault.xctoolchain/usr/lib/swift-5.0" - swiftLibsDir = os.path.join(GetXcodeDeveloperPath(), relativePath) - swiftLibPlatformDir = os.path.join(swiftLibsDir, ios_constants.SDK.IPHONESIMULATOR) - if os.path.exists(swiftLibPlatformDir): - return swiftLibPlatformDir + """Gets the Swift5 fallback libraries directory.""" + relative_path = 'Toolchains/XcodeDefault.xctoolchain/usr/lib/swift-5.0' + swift_libs_dir = os.path.join(GetXcodeDeveloperPath(), relative_path) + swift_lib_platform_dir = os.path.join(swift_libs_dir, + ios_constants.SDK.IPHONESIMULATOR) + if os.path.exists(swift_lib_platform_dir): + return swift_lib_platform_dir return None @@ -60,15 +63,9 @@ def GetXcodeVersionNumber(): # Build version 8C1002 output = subprocess.check_output(('xcodebuild', '-version')) xcode_version = output.split('\n')[0].split(' ')[1] - parts = xcode_version.split('.') - xcode_version_number = int(parts[0]) * 100 - if len(parts) > 1: - xcode_version_number += int(parts[1]) * 10 - if len(parts) > 2: - xcode_version_number += int(parts[2]) # Add cache xcode_version_number to avoid calling subprocess multiple times. # It is expected that no one changes xcode during the test runner working. - _xcode_version_number = xcode_version_number + _xcode_version_number = version_util.GetVersionNumber(xcode_version) return _xcode_version_number diff --git a/xctestrunner/simulator_control/simulator_util.py b/xctestrunner/simulator_control/simulator_util.py index 177b21a..cf1d59c 100644 --- a/xctestrunner/simulator_control/simulator_util.py +++ b/xctestrunner/simulator_control/simulator_util.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """The utility class for simulator.""" import json diff --git a/xctestrunner/test_runner/TestProject/TestProject.xcodeproj/project.pbxproj b/xctestrunner/test_runner/TestProject/TestProject.xcodeproj/project.pbxproj deleted file mode 100755 index 3d20a0b..0000000 --- a/xctestrunner/test_runner/TestProject/TestProject.xcodeproj/project.pbxproj +++ /dev/null @@ -1,342 +0,0 @@ - - - - - archiveVersion - 1 - classes - - objectVersion - 46 - objects - - AppUnderTestBuildConfig - - buildSettings - - CODE_SIGNING_REQUIRED - NO - CODE_SIGN_IDENTITY - - DEVELOPMENT_TEAM - - INFOPLIST_FILE - $(BUILT_PRODUCTS_DIR)/$(APP_UNDER_TEST_NAME).app/Info.plist - PRODUCT_BUNDLE_IDENTIFIER - - PRODUCT_NAME - $(APP_UNDER_TEST_NAME) - PROVISIONING_PROFILE_SPECIFIER - - - isa - XCBuildConfiguration - name - Debug - - AppUnderTestBuildConfigList - - buildConfigurations - - AppUnderTestBuildConfig - - defaultConfigurationIsVisible - 0 - isa - XCConfigurationList - - AppUnderTestFile - - explicitFileType - wrapper.application - includeInIndex - 0 - isa - PBXFileReference - path - .app - sourceTree - BUILT_PRODUCTS_DIR - - AppUnderTestTarget - - buildConfigurationList - AppUnderTestBuildConfigList - buildPhases - - buildRules - - dependencies - - isa - PBXNativeTarget - name - %AppUnderTestName% - productName - %AppUnderTestName% - productReference - AppUnderTestFile - productType - com.apple.product-type.application - - TestProjectBuildConfig - - buildSettings - - IPHONEOS_DEPLOYMENT_TARGET - - SDKROOT - iphoneos - APP_UNDER_TEST_NAME - - XCTEST_BUNDLE_NAME - - XCUITEST_BUNDLE_NAME - - - isa - XCBuildConfiguration - name - Debug - - TestProjectBuildConfigList - - buildConfigurations - - TestProjectBuildConfig - - defaultConfigurationIsVisible - 0 - defaultConfigurationName - Debug - isa - XCConfigurationList - - TestProjectGroup - - children - - TestProjectProducts - - isa - PBXGroup - sourceTree - <group> - - TestProjectObject - - attributes - - LastUpgradeCheck - 0720 - ORGANIZATIONNAME - Google Inc - TargetAttributes - - AppUnderTestTarget - - CreatedOnToolsVersion - 7.2 - - XCTestBundleTarget - - CreatedOnToolsVersion - 7.2 - TestTargetID - AppUnderTestTarget - - XCUITestBundleTarget - - CreatedOnToolsVersion - 8.0 - TestTargetID - AppUnderTestTarget - - - - buildConfigurationList - TestProjectBuildConfigList - compatibilityVersion - Xcode 3.2 - developmentRegion - English - hasScannedForEncodings - 0 - isa - PBXProject - knownRegions - - en - Base - - mainGroup - TestProjectGroup - productRefGroup - TestProjectProducts - projectDirPath - - projectRoot - - targets - - AppUnderTestTarget - XCTestBundleTarget - XCUITestBundleTarget - - - TestProjectProducts - - children - - AppUnderTestFile - XCTestBundleFile - XCUITestBundleFile - - isa - PBXGroup - name - Products - sourceTree - <group> - - XCTestBundleBuildConfig - - buildSettings - - CODE_SIGN_IDENTITY - - DEVELOPMENT_TEAM - - INFOPLIST_FILE - $(BUILT_PRODUCTS_DIR)/$(XCTEST_BUNDLE_NAME).xctest/Info.plist - PRODUCT_NAME - $(XCTEST_BUNDLE_NAME) - TEST_HOST - $(BUILT_PRODUCTS_DIR)/$(APP_UNDER_TEST_NAME).app/$(APP_UNDER_TEST_NAME) - - isa - XCBuildConfiguration - name - Debug - - XCTestBundleBuildConfigList - - buildConfigurations - - XCTestBundleBuildConfig - - defaultConfigurationIsVisible - 0 - isa - XCConfigurationList - - XCTestBundleFile - - explicitFileType - wrapper.cfbundle - includeInIndex - 0 - isa - PBXFileReference - path - .xctest - sourceTree - BUILT_PRODUCTS_DIR - - XCTestBundleTarget - - buildConfigurationList - XCTestBundleBuildConfigList - buildPhases - - buildRules - - dependencies - - isa - PBXNativeTarget - name - %XCTestBundleName% - productName - %XCTestBundleName% - productReference - XCTestBundleFile - productType - com.apple.product-type.bundle.unit-test - - XCUITestBundleBuildConfig - - buildSettings - - CODE_SIGN_IDENTITY - - DEVELOPMENT_TEAM - - INFOPLIST_FILE - $(BUILT_PRODUCTS_DIR)/$(XCUITEST_BUNDLE_NAME).xctest/Info.plist - PRODUCT_BUNDLE_IDENTIFIER - - PRODUCT_NAME - $(XCUITEST_BUNDLE_NAME) - PROVISIONING_PROFILE_SPECIFIER - - TEST_TARGET_NAME - $(APP_UNDER_TEST_NAME) - USES_XCTRUNNER - true - - isa - XCBuildConfiguration - name - Debug - - XCUITestBundleBuildConfigList - - buildConfigurations - - XCUITestBundleBuildConfig - - defaultConfigurationIsVisible - 0 - isa - XCConfigurationList - - XCUITestBundleFile - - explicitFileType - wrapper.cfbundle - includeInIndex - 0 - isa - PBXFileReference - path - UITests.xctest - sourceTree - BUILT_PRODUCTS_DIR - - XCUITestBundleTarget - - buildConfigurationList - XCUITestBundleBuildConfigList - buildPhases - - buildRules - - dependencies - - isa - PBXNativeTarget - name - %XCUITestBundleName% - productName - %XCUITestBundleName% - productReference - XCUITestBundleFile - productType - com.apple.product-type.bundle.ui-testing - - - rootObject - TestProjectObject - - diff --git a/xctestrunner/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXctest.xcscheme b/xctestrunner/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXctest.xcscheme deleted file mode 100755 index 60b405b..0000000 --- a/xctestrunner/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXctest.xcscheme +++ /dev/null @@ -1,104 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/xctestrunner/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXcuitest.xcscheme b/xctestrunner/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXcuitest.xcscheme deleted file mode 100755 index 1abe430..0000000 --- a/xctestrunner/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXcuitest.xcscheme +++ /dev/null @@ -1,105 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/xctestrunner/test_runner/dummy_project.py b/xctestrunner/test_runner/dummy_project.py deleted file mode 100644 index b704d2d..0000000 --- a/xctestrunner/test_runner/dummy_project.py +++ /dev/null @@ -1,571 +0,0 @@ -# Copyright 2017 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Helper class for dummy Xcode project generated by prebuilt bundles. - -The dummy project supports sdk iphonesimulator and test type XCUITest. It can -be run with `xcodebuild build-for-testing`. -""" - -import logging -import os -import pkgutil -import shutil -import subprocess -import tempfile -import xml.etree.ElementTree as ET - -from xctestrunner.shared import bundle_util -from xctestrunner.shared import ios_constants -from xctestrunner.shared import ios_errors -from xctestrunner.shared import plist_util -from xctestrunner.shared import provisioning_profile -from xctestrunner.shared import xcode_info_util -from xctestrunner.test_runner import xcodebuild_test_executor - - -_DEFAULT_PERMS = 0o0777 -_DUMMYPROJECT_DIR_NAME = 'TestProject' -_DUMMYPROJECT_XCODEPROJ_NAME = 'TestProject.xcodeproj' -_DUMMYPROJECT_PBXPROJ_NAME = 'project.pbxproj' -_DUMMYPROJECT_XCTESTS_SCHEME = 'TestProjectXctest' -_DUMMYPROJECT_XCUITESTS_SCHEME = 'TestProjectXcuitest' - -_SIGNAL_BUILD_FOR_TESTING_SUCCEEDED = '** TEST BUILD SUCCEEDED **' -_SIGNAL_XCODEBUILD_TEST_SUCCEEDED = '** TEST SUCCEEDED **' -_SIGNAL_XCODEBUILD_TEST_FAILED = '** TEST FAILED **' - - -class DummyProject(object): - """Handles a dummy project with prebuilt bundles.""" - - def __init__(self, - app_under_test_dir, - test_bundle_dir, - sdk=ios_constants.SDK.IPHONESIMULATOR, - test_type=ios_constants.TestType.XCUITEST, - work_dir=None, - keychain_path=None): - """Initializes the DummyProject object. - - Args: - app_under_test_dir: string, path of the app to be tested in - dummy project. - test_bundle_dir: string, path of the test bundle. - sdk: string, SDKRoot of the dummy project. See supported SDKs in - module shared.ios_constants. - test_type: string, test type of the test bundle. See supported test types - in module shared.ios_constants. - work_dir: string, work directory which contains run files. - keychain_path: string, path of preferred keychain to use. - """ - self._app_under_test_dir = app_under_test_dir - self._test_bundle_dir = test_bundle_dir - self._sdk = sdk - self._test_type = test_type - if work_dir: - self._work_dir = os.path.join(work_dir, 'dummy_project') - else: - self._work_dir = None - self._dummy_project_path = None - self._keychain_path = keychain_path - self._xcodeproj_dir_path = None - self._pbxproj_file_path = None - self._is_dummy_project_generated = False - self._delete_work_dir = False - self._ValidateArguments() - self._test_scheme = None - if test_type == ios_constants.TestType.XCTEST: - self._test_scheme = _DUMMYPROJECT_XCTESTS_SCHEME - elif test_type == ios_constants.TestType.XCUITEST: - self._test_scheme = _DUMMYPROJECT_XCUITESTS_SCHEME - - def __enter__(self): - self.GenerateDummyProject() - return self - - def __exit__(self, unused_type, unused_value, unused_traceback): - """Deletes the temp directories.""" - self.Close() - - @property - def pbxproj_file_path(self): - """Gets the pbxproj file path of the dummy project.""" - return self._pbxproj_file_path - - @property - def test_scheme_path(self): - """Gets the test scheme path of the dummy project.""" - return os.path.join( - self._xcodeproj_dir_path, - 'xcshareddata/xcschemes', - '%s.xcscheme' % self._test_scheme) - - def BuildForTesting(self, built_products_dir, derived_data_dir): - """Runs `xcodebuild build-for-testing` with the dummy project. - - If app under test or test bundle are not in built_products_dir, will copy - the file into built_products_dir. - - Args: - built_products_dir: path of the built products dir in this build session. - derived_data_dir: path of the derived data dir in this build session. - Raises: - BuildFailureError: when failed to build the dummy project. - """ - self.GenerateDummyProject() - self._PrepareBuildProductsDir(built_products_dir) - logging.info('Running `xcodebuild build-for-testing` with dummy project.\n' - '\tbuilt_product_dir = %s\n\tderived_data_path = %s\n', - built_products_dir, - derived_data_dir) - command = ['xcodebuild', 'build-for-testing', - 'BUILT_PRODUCTS_DIR=' + built_products_dir, - 'SDKROOT=' + self._sdk, - '-project', self._xcodeproj_dir_path, - '-scheme', self._test_scheme, - '-derivedDataPath', derived_data_dir] - run_env = dict(os.environ) - run_env['NSUnbufferedIO'] = 'YES' - try: - output = subprocess.check_output( - command, env=run_env, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - raise ios_errors.BuildFailureError('Failed to build the dummy project. ' - 'Output is:\n%s' % e.output) - - if _SIGNAL_BUILD_FOR_TESTING_SUCCEEDED not in output: - raise ios_errors.BuildFailureError('Failed to build the dummy project. ' - 'Output is:\n%s' % output) - - def RunXcTest(self, device_id, built_products_dir, derived_data_dir, - startup_timeout_sec): - """Runs `xcodebuild test` with the dummy project. - - If app under test or test bundle are not in built_products_dir, will copy - the file into built_products_dir. - - Args: - device_id: string, id of the device. - built_products_dir: path of the built products dir in this build session. - derived_data_dir: path of the derived data dir in this build session. - startup_timeout_sec: Seconds until the xcodebuild command is deemed stuck. - - Returns: - A value of type runner_exit_codes.EXITCODE. - - Raises: - IllegalArgumentError: when test type is not xctest. - """ - if self._test_type != ios_constants.TestType.XCTEST: - raise ios_errors.IllegalArgumentError( - 'Only xctest dummy project is supported to run `xcodebuild test`. ' - 'The test type %s is not supported.' % self._test_type) - self.GenerateDummyProject() - # In Xcode 7.3+, the folder structure of app under test is changed. - if xcode_info_util.GetXcodeVersionNumber() >= 730: - app_under_test_plugin_path = os.path.join(self._app_under_test_dir, - 'PlugIns') - if not os.path.exists(app_under_test_plugin_path): - os.mkdir(app_under_test_plugin_path) - test_bundle_under_plugin_path = os.path.join( - app_under_test_plugin_path, os.path.basename(self._test_bundle_dir)) - if not os.path.exists(test_bundle_under_plugin_path): - shutil.copytree(self._test_bundle_dir, test_bundle_under_plugin_path) - self._PrepareBuildProductsDir(built_products_dir) - - logging.info('Running `xcodebuild test` with dummy project.\n' - 'device_id= %s\n' - 'built_product_dir = %s\nderived_data_path = %s\n', - device_id, - built_products_dir, - derived_data_dir) - command = ['xcodebuild', 'test', - 'BUILT_PRODUCTS_DIR=' + built_products_dir, - '-project', self._xcodeproj_dir_path, - '-scheme', self._test_scheme, - '-destination', 'id=' + device_id, - '-derivedDataPath', derived_data_dir] - app_bundle_id = bundle_util.GetBundleId(self._app_under_test_dir) - exit_code, _ = xcodebuild_test_executor.XcodebuildTestExecutor( - command, - succeeded_signal=_SIGNAL_XCODEBUILD_TEST_SUCCEEDED, - failed_signal=_SIGNAL_XCODEBUILD_TEST_FAILED, - sdk=self._sdk, - test_type=self._test_type, - device_id=device_id, - app_bundle_id=app_bundle_id, - startup_timeout_sec=startup_timeout_sec).Execute(return_output=False) - return exit_code - - def GenerateDummyProject(self): - """Generates the dummy project according to the specification. - - Raises: - IllegalArgumentError: when the sdk or test type is not supported. - """ - if self._is_dummy_project_generated: - return - - if self._work_dir: - self._dummy_project_path = os.path.join(self._work_dir, - _DUMMYPROJECT_DIR_NAME) - if os.path.exists(self._dummy_project_path): - logging.info('Skips generating dummy project which is generated.') - self._xcodeproj_dir_path = os.path.join( - self._dummy_project_path, _DUMMYPROJECT_XCODEPROJ_NAME) - self._pbxproj_file_path = os.path.join( - self._xcodeproj_dir_path, _DUMMYPROJECT_PBXPROJ_NAME) - self._is_dummy_project_generated = True - return - - logging.info('Generating dummy project.') - if self._work_dir: - if not os.path.exists(self._work_dir): - os.mkdir(self._work_dir) - else: - self._work_dir = tempfile.mkdtemp() - self._delete_work_dir = True - self._dummy_project_path = os.path.join(self._work_dir, - _DUMMYPROJECT_DIR_NAME) - shutil.copytree(_GetTestProject(self._work_dir), self._dummy_project_path) - for root, dirs, files in os.walk(self._dummy_project_path): - for d in dirs: - os.chmod(os.path.join(root, d), _DEFAULT_PERMS) - for f in files: - os.chmod(os.path.join(root, f), _DEFAULT_PERMS) - self._xcodeproj_dir_path = os.path.join( - self._dummy_project_path, _DUMMYPROJECT_XCODEPROJ_NAME) - self._pbxproj_file_path = os.path.join( - self._xcodeproj_dir_path, _DUMMYPROJECT_PBXPROJ_NAME) - - # Set the iOS deployment target in pbxproj. - # If don't set this field, the default value will be the latest supported - # iOS version which may make the app installation failure. - self._SetIosDeploymentTarget() - - # Overwrite the pbxproj file content for test type specific. - if self._test_type == ios_constants.TestType.XCUITEST: - self._SetPbxprojForXcuitest() - elif self._test_type == ios_constants.TestType.XCTEST: - self._SetPbxprojForXctest() - - self._is_dummy_project_generated = True - logging.info('Dummy project is generated.') - - def Close(self): - """Deletes the temp directories.""" - if self._delete_work_dir and os.path.exists(self._work_dir): - shutil.rmtree(self._work_dir) - - def _ValidateArguments(self): - """Checks whether the arguments of the dummy project is valid. - - Raises: - IllegalArgumentError: when the sdk or test type is not supported. - """ - if self._sdk not in ios_constants.SUPPORTED_SDKS: - raise ios_errors.IllegalArgumentError( - 'The sdk %s is not supported. Supported sdks are %s.' - % (self._sdk, ios_constants.SUPPORTED_SDKS)) - if self._test_type not in ios_constants.SUPPORTED_TEST_TYPES: - raise ios_errors.IllegalArgumentError( - 'The test type %s is not supported. Supported test types are %s.' - % (self._test_type, ios_constants.SUPPORTED_TEST_TYPES)) - - def _PrepareBuildProductsDir(self, built_products_dir): - """Prepares the build products directory for dummy project. - - Args: - built_products_dir: path of the directory to be prepared. - """ - logging.info('Preparing build products directory %s for dummy project.', - built_products_dir) - app_under_test_name = os.path.basename(self._app_under_test_dir) - test_bundle_name = os.path.basename(self._test_bundle_dir) - if not os.path.exists( - os.path.join(built_products_dir, app_under_test_name)): - shutil.copytree(self._app_under_test_dir, - os.path.join(built_products_dir, app_under_test_name)) - if not os.path.exists( - os.path.join(built_products_dir, test_bundle_name)): - shutil.copytree(self._test_bundle_dir, - os.path.join(built_products_dir, test_bundle_name)) - - def _SetIosDeploymentTarget(self): - """Sets the iOS deployment target in dummy project's pbxproj.""" - pbxproj_plist_obj = plist_util.Plist(self.pbxproj_file_path) - pbxproj_plist_obj.SetPlistField( - 'objects:TestProjectBuildConfig:buildSettings:' - 'IPHONEOS_DEPLOYMENT_TARGET', - bundle_util.GetMinimumOSVersion(self._app_under_test_dir)) - - def _SetPbxprojForXcuitest(self): - """Sets the dummy project's pbxproj for xcuitest.""" - pbxproj_plist_obj = plist_util.Plist(self.pbxproj_file_path) - pbxproj_objects = pbxproj_plist_obj.GetPlistField('objects') - - # Sets the build setting of test bundle for generated XCTRunner.app signing. - # 1) If run with iphonesimulator, don't need to set any fields in build - # setting. xcodebuild will sign the XCTRunner.app with identity '-' and no - # provisioning profile by default. - # 2) If runs with iphoneos and the app under test's embedded provisioning - # profile is 'iOS Team Provisioning Profile: *', set build setting for using - # Xcode managed provisioning profile to sign the XCTRunner.app. - # 3) If runs with iphoneos and the app under test's embedded provisioning - # profile is specific, set build setting for using app under test's - # embedded provisioning profile to sign the XCTRunner.app. If the - # provisioning profile is not installed in the Mac machine, also installs - # it. - # 4) The test bundle's provisioning profile can be overwrited by method - # SetTestBundleProvisioningProfile. - if self._sdk == ios_constants.SDK.IPHONEOS: - build_setting = pbxproj_objects[ - 'XCUITestBundleBuildConfig']['buildSettings'] - build_setting['PRODUCT_BUNDLE_IDENTIFIER'] = bundle_util.GetBundleId( - self._test_bundle_dir) - build_setting['DEVELOPMENT_TEAM'] = bundle_util.GetDevelopmentTeam( - self._test_bundle_dir) - embedded_provision = provisioning_profile.ProvisiongProfile( - os.path.join(self._app_under_test_dir, 'embedded.mobileprovision'), - self._work_dir, - keychain_path=self._keychain_path) - embedded_provision.Install() - # Case 2) - if embedded_provision.name.startswith('iOS Team Provisioning Profile: '): - build_setting['CODE_SIGN_IDENTITY'] = 'iPhone Developer' - else: - # Case 3) - build_setting['CODE_SIGN_IDENTITY'] = bundle_util.GetCodesignIdentity( - self._app_under_test_dir) - (build_setting[ - 'PROVISIONING_PROFILE_SPECIFIER']) = embedded_provision.name - - # Sets the app under test and test bundle. - test_project_build_setting = pbxproj_objects[ - 'TestProjectBuildConfig']['buildSettings'] - app_under_test_name = os.path.splitext( - os.path.basename(self._app_under_test_dir))[0] - pbxproj_objects['AppUnderTestTarget']['name'] = app_under_test_name - pbxproj_objects['AppUnderTestTarget']['productName'] = app_under_test_name - test_project_build_setting['APP_UNDER_TEST_NAME'] = app_under_test_name - test_bundle_name = os.path.splitext( - os.path.basename(self._test_bundle_dir))[0] - pbxproj_objects['XCUITestBundleTarget']['name'] = test_bundle_name - pbxproj_objects['XCUITestBundleTarget']['productName'] = test_bundle_name - test_project_build_setting['XCUITEST_BUNDLE_NAME'] = test_bundle_name - - pbxproj_plist_obj.SetPlistField('objects', pbxproj_objects) - - def _SetPbxprojForXctest(self): - """Sets the dummy project's pbxproj for xctest.""" - pbxproj_plist_obj = plist_util.Plist(self.pbxproj_file_path) - pbxproj_objects = pbxproj_plist_obj.GetPlistField('objects') - - # Sets the build setting for app under test and unit test bundle signing. - # 1) If run with iphonesimulator, don't need to set any fields in build - # setting. xcodebuild will sign bundles with identity '-' and no - # provisioning profile by default. - # 2) If runs with iphoneos and the app under test's embedded provisioning - # profile is 'iOS Team Provisioning Profile: *', set build setting for using - # Xcode managed provisioning profile to sign bundles. - # 3) If runs with iphoneos and the app under test's embedded provisioning - # profile is specific, set build setting with using app under test's - # embedded provisioning profile. - if self._sdk == ios_constants.SDK.IPHONEOS: - aut_build_setting = pbxproj_objects[ - 'AppUnderTestBuildConfig']['buildSettings'] - test_build_setting = pbxproj_objects[ - 'XCTestBundleBuildConfig']['buildSettings'] - aut_build_setting['CODE_SIGNING_REQUIRED'] = 'YES' - aut_build_setting['PRODUCT_BUNDLE_IDENTIFIER'] = bundle_util.GetBundleId( - self._app_under_test_dir) - embedded_provision = provisioning_profile.ProvisiongProfile( - os.path.join(self._app_under_test_dir, 'embedded.mobileprovision'), - self._work_dir, - keychain_path=self._keychain_path) - embedded_provision.Install() - # Case 2) - if embedded_provision.name.startswith('iOS Team Provisioning Profile: '): - aut_build_setting['CODE_SIGN_IDENTITY'] = 'iPhone Developer' - test_build_setting['CODE_SIGN_IDENTITY'] = 'iPhone Developer' - app_under_test_dev_team = bundle_util.GetDevelopmentTeam( - self._app_under_test_dir) - aut_build_setting['DEVELOPMENT_TEAM'] = app_under_test_dev_team - test_build_setting['DEVELOPMENT_TEAM'] = app_under_test_dev_team - else: - # Case 3) - app_under_test_sign_identity = bundle_util.GetCodesignIdentity( - self._app_under_test_dir) - aut_build_setting['CODE_SIGN_IDENTITY'] = app_under_test_sign_identity - test_build_setting['CODE_SIGN_IDENTITY'] = app_under_test_sign_identity - (aut_build_setting[ - 'PROVISIONING_PROFILE_SPECIFIER']) = embedded_provision.name - - # Sets the app under test and test bundle. - test_project_build_setting = pbxproj_objects[ - 'TestProjectBuildConfig']['buildSettings'] - app_under_test_name = os.path.splitext( - os.path.basename(self._app_under_test_dir))[0] - pbxproj_objects['AppUnderTestTarget']['name'] = app_under_test_name - pbxproj_objects['AppUnderTestTarget']['productName'] = app_under_test_name - test_project_build_setting['APP_UNDER_TEST_NAME'] = app_under_test_name - test_bundle_name = os.path.splitext( - os.path.basename(self._test_bundle_dir))[0] - pbxproj_objects['XCTestBundleTarget']['name'] = test_bundle_name - pbxproj_objects['XCTestBundleTarget']['productName'] = test_bundle_name - test_project_build_setting['XCTEST_BUNDLE_NAME'] = test_bundle_name - - pbxproj_plist_obj.SetPlistField('objects', pbxproj_objects) - - def SetTestBundleProvisioningProfile(self, test_bundle_provisioning_profile): - """Sets the provisioning profile specifier to the test bundle. - - If the given provisioning profile is a path, will also install it in the - host. - - Args: - test_bundle_provisioning_profile: string, name/path of the provisioning - profile of test bundle. - """ - if not test_bundle_provisioning_profile: - return - provisioning_profile_is_file = False - if (test_bundle_provisioning_profile.startswith('/') and - os.path.exists(test_bundle_provisioning_profile)): - provisioning_profile_is_file = True - - if self._sdk != ios_constants.SDK.IPHONEOS: - logging.warning( - 'Can only set provisioning profile to test bundle in iphoneos SDK. ' - 'But current SDK is %s', self._sdk) - return - self.GenerateDummyProject() - if self._test_type == ios_constants.TestType.XCUITEST: - pbxproj_plist_obj = plist_util.Plist(self.pbxproj_file_path) - pbxproj_objects = pbxproj_plist_obj.GetPlistField('objects') - settings = pbxproj_objects['XCUITestBundleBuildConfig']['buildSettings'] - settings['CODE_SIGN_IDENTITY'] = bundle_util.GetCodesignIdentity( - self._test_bundle_dir) - if not provisioning_profile_is_file: - settings[ - 'PROVISIONING_PROFILE_SPECIFIER'] = test_bundle_provisioning_profile - else: - profile_obj = provisioning_profile.ProvisiongProfile( - test_bundle_provisioning_profile, - self._work_dir, - keychain_path=self._keychain_path) - profile_obj.Install() - settings['PROVISIONING_PROFILE_SPECIFIER'] = profile_obj.name - pbxproj_plist_obj.SetPlistField('objects', pbxproj_objects) - else: - logging.warning( - 'Setting provisioning profile specifier to test bundle in test type ' - '%s is not supported.', self._test_type) - - def SetEnvVars(self, env_vars): - """Sets the additional environment variables in the dummy project's scheme. - - Args: - env_vars: dict. Both key and value is string. - """ - if not env_vars: - return - self.GenerateDummyProject() - scheme_path = self.test_scheme_path - scheme_tree = ET.parse(scheme_path) - root = scheme_tree.getroot() - test_action_element = root.find('TestAction') - test_action_element.set('shouldUseLaunchSchemeArgsEnv', 'NO') - envs_element = ET.SubElement(test_action_element, 'EnvironmentVariables') - for key, value in env_vars.items(): - env_element = ET.SubElement(envs_element, 'EnvironmentVariable') - env_element.set('key', key) - env_element.set('value', value) - env_element.set('isEnabled', 'YES') - scheme_tree.write(scheme_path) - - def SetArgs(self, args): - """Sets the additional arguments in the dummy project's scheme. - - Args: - args: a list of string. Each item is an argument. - """ - if not args: - return - self.GenerateDummyProject() - scheme_path = self.test_scheme_path - scheme_tree = ET.parse(scheme_path) - test_action_element = scheme_tree.getroot().find('TestAction') - test_action_element.set('shouldUseLaunchSchemeArgsEnv', 'NO') - args_element = ET.SubElement(test_action_element, 'CommandLineArguments') - for arg in args: - arg_element = ET.SubElement(args_element, 'CommandLineArgument') - arg_element.set('argument', arg) - arg_element.set('isEnabled', 'YES') - scheme_tree.write(scheme_path) - - def SetSkipTests(self, skip_tests): - """Sets the skip tests in the dummy project's scheme. - - Args: - skip_tests: a list of string. The format of each item is - Test-Class-Name[/Test-Method-Name]. - """ - if not skip_tests: - return - self.GenerateDummyProject() - scheme_path = self.test_scheme_path - scheme_tree = ET.parse(scheme_path) - test_action_element = scheme_tree.getroot().find('TestAction') - testable_reference_element = test_action_element.find( - 'Testables').find('TestableReference') - skip_tests_element = ET.SubElement( - testable_reference_element, 'SkippedTests') - for skip_test in skip_tests: - skip_test_element = ET.SubElement(skip_tests_element, 'Test') - skip_test_element.set('Identifier', skip_test) - scheme_tree.write(scheme_path) - - -def _GetTestProject(work_dir): - """Gets the TestProject path.""" - test_project_path = os.path.join(work_dir, 'Resource/TestProject') - if os.path.exists(test_project_path): - return test_project_path - - xcodeproj_path = os.path.join(test_project_path, 'TestProject.xcodeproj') - os.makedirs(xcodeproj_path) - with open(os.path.join(xcodeproj_path, 'project.pbxproj'), - 'w+') as target_file: - target_file.write( - pkgutil.get_data('xctestrunner.test_runner', - 'TestProject/TestProject.xcodeproj/project.pbxproj')) - xcschemes_path = os.path.join(xcodeproj_path, 'xcshareddata/xcschemes') - os.makedirs(xcschemes_path) - with open(os.path.join(xcschemes_path, 'TestProjectXctest.xcscheme'), - 'w+') as target_file: - target_file.write( - pkgutil.get_data( - 'xctestrunner.test_runner', - 'TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/' - 'TestProjectXctest.xcscheme')) - with open(os.path.join(xcschemes_path, 'TestProjectXcuitest.xcscheme'), - 'w+') as target_file: - target_file.write( - pkgutil.get_data( - 'xctestrunner.test_runner', - 'TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/' - 'TestProjectXcuitest.xcscheme')) - return test_project_path diff --git a/xctestrunner/test_runner/logic_test_util.py b/xctestrunner/test_runner/logic_test_util.py index 60fac3a..8439e4b 100644 --- a/xctestrunner/test_runner/logic_test_util.py +++ b/xctestrunner/test_runner/logic_test_util.py @@ -18,14 +18,19 @@ import sys from xctestrunner.shared import ios_constants +from xctestrunner.shared import version_util from xctestrunner.shared import xcode_info_util from xctestrunner.test_runner import runner_exit_codes _SIMCTL_ENV_VAR_PREFIX = 'SIMCTL_CHILD_' -def RunLogicTestOnSim( - sim_id, test_bundle_path, env_vars=None, args=None, tests_to_run=None): +def RunLogicTestOnSim(sim_id, + test_bundle_path, + env_vars=None, + args=None, + tests_to_run=None, + os_version=None): """Runs logic tests on the simulator. The output prints on system stdout. Args: @@ -36,6 +41,7 @@ def RunLogicTestOnSim( args: array, the additional arguments passing to test's process. tests_to_run: array, the format of each item is TestClass[/TestMethod]. If it is empty, then runs with All methods. + os_version: string, the OS version of the simulator. Returns: exit_code: A value of type runner_exit_codes.EXITCODE. @@ -48,15 +54,14 @@ def RunLogicTestOnSim( for key in env_vars: simctl_env_vars[_SIMCTL_ENV_VAR_PREFIX + key] = env_vars[key] simctl_env_vars['NSUnbufferedIO'] = 'YES' - - # Fixes failures for unit test targets that depend on Swift libraries when running with Xcode 11 - # on pre-iOS 12.2 simulators. - # Example failure message this resolves: "The bundle couldn’t be loaded because it is damaged - # or missing necessary resources." - swift5FallbackLibsDir = xcode_info_util.GetSwift5FallbackLibsDir() - if swift5FallbackLibsDir: - simctl_env_vars[_SIMCTL_ENV_VAR_PREFIX + "DYLD_FALLBACK_LIBRARY_PATH"] = swift5FallbackLibsDir - + # When running tests on iOS 12.1 or earlier simulator under Xcode 11 or later, + # it is required to add swift5 fallback libraries to environment variable. + # See https://github.com/bazelbuild/rules_apple/issues/684 for context. + if (xcode_info_util.GetXcodeVersionNumber() >= 1100 and + os_version and + version_util.GetVersionNumber(os_version) < 1220): + key = _SIMCTL_ENV_VAR_PREFIX + 'DYLD_FALLBACK_LIBRARY_PATH' + simctl_env_vars[key] = xcode_info_util.GetSwift5FallbackLibsDir() command = [ 'xcrun', 'simctl', 'spawn', '-s', sim_id, xcode_info_util.GetXctestToolPath(ios_constants.SDK.IPHONESIMULATOR)] diff --git a/xctestrunner/test_runner/test_summaries_util.py b/xctestrunner/test_runner/test_summaries_util.py deleted file mode 100644 index 32e4923..0000000 --- a/xctestrunner/test_runner/test_summaries_util.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright 2017 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Helper class for parsing TestSummaries.plist file. -""" - -import glob -import os -import shutil -import tempfile - -from xctestrunner.shared import plist_util - - -def GetTestSummariesPaths(derived_data_dir): - """Get the TestSummaries.plist files under the DerivedData directory.""" - return glob.glob('%s/Logs/Test/*_TestSummaries.plist' % derived_data_dir) - - -def ParseTestSummaries( - test_summaries_path, attachments_dir_path, delete_uitest_auto_screenshots): - """Parse the TestSummaries.plist and structure the attachments' files. - - Only the screenshots file from failure test methods and .crash files will be - stored. Other files will be removed. - - Args: - test_summaries_path: string, the path of TestSummaries.plist file. - attachments_dir_path: string, the path of Attachments directory. - delete_uitest_auto_screenshots: bool, whether deletes the auto screenshots. - """ - test_summaries_plist = plist_util.Plist(test_summaries_path) - tests_obj = test_summaries_plist.GetPlistField('TestableSummaries:0:Tests:0') - # Store the required screenshots and crash files under temp directory first. - # Then use the temp directory to replace the original Attachments directory. - # If delete_uitest_auto_screenshots is true, only move crash files to - # temp directory and the left screenshots will be deleted. - temp_dir = tempfile.mkdtemp(dir=os.path.dirname(attachments_dir_path)) - if not delete_uitest_auto_screenshots: - _ParseTestObject(tests_obj, attachments_dir_path, temp_dir) - for crash_file in glob.glob('%s/*.crash' % attachments_dir_path): - shutil.move(crash_file, temp_dir) - shutil.rmtree(attachments_dir_path) - shutil.move(temp_dir, attachments_dir_path) - - -def _ParseTestObject(test_obj, attachments_dir_path, parent_test_obj_dir_path): - """Parse the test method object and structure its attachment files.""" - test_obj_dir_path = os.path.join( - parent_test_obj_dir_path, - test_obj['TestIdentifier'].replace('.', '_').replace('/', '_')) - if 'Subtests' in test_obj: - # If the test suite only has one sub test, don't create extra folder which - # causes extra directory hierarchy. - if len(test_obj['Subtests']) > 1: - if not os.path.exists(test_obj_dir_path): - os.mkdir(test_obj_dir_path) - else: - test_obj_dir_path = parent_test_obj_dir_path - for sub_test_obj in test_obj['Subtests']: - _ParseTestObject(sub_test_obj, attachments_dir_path, test_obj_dir_path) - return - # Only parse the failure test methods. The succeed test method's attachment - # files will be removed later. - if test_obj['TestStatus'] == 'Success': - return - if not os.path.exists(test_obj_dir_path): - os.mkdir(test_obj_dir_path) - test_result_plist_path = os.path.join(test_obj_dir_path, - 'TestMethodResult.plist') - plist_util.Plist(test_result_plist_path).SetPlistField('', test_obj) - if 'ActivitySummaries' in test_obj: - for test_activity_obj in test_obj['ActivitySummaries']: - _ExploreTestActivity( - test_activity_obj, attachments_dir_path, test_obj_dir_path) - - -def _ExploreTestActivity(test_activity_obj, attachments_dir_path, - test_obj_dir_path): - """Move the screenshot files of this method to test object directory.""" - if 'HasScreenshotData' in test_activity_obj: - screenshot_file_paths = glob.glob( - os.path.join( - attachments_dir_path, - 'Screenshot_%s.*' % test_activity_obj['UUID'])) - for path in screenshot_file_paths: - shutil.move(path, test_obj_dir_path) - if 'SubActivities' in test_activity_obj: - for sub_test_activity_obj in test_activity_obj['SubActivities']: - _ExploreTestActivity( - sub_test_activity_obj, attachments_dir_path, test_obj_dir_path) diff --git a/xctestrunner/test_runner/xcresult_util.py b/xctestrunner/test_runner/xcresult_util.py new file mode 100644 index 0000000..dc5a934 --- /dev/null +++ b/xctestrunner/test_runner/xcresult_util.py @@ -0,0 +1,45 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helper class for parsing xcresult under Xcode 11 or later.""" + +import json +import subprocess + +from xctestrunner.shared import ios_errors + + +def ExposeDiagnosticsRef(xcresult_path, output_path): + """Exposes the DiagnosticsRef files from the given xcresult file.""" + output = subprocess.check_output([ + 'xcrun', 'xcresulttool', 'get', '--format', 'json', '--path', + xcresult_path + ]) + result_bundle_json = json.loads(output) + actions = result_bundle_json['actions']['_values'] + action_result = None + for action in actions: + if action['_type']['_name'] == 'ActionRecord': + action_result = action['actionResult'] + break + if action_result is None: + raise ios_errors.XcresultError( + 'Failed to get "ActionResult" from result bundle %s' % output) + + diagnostics_id = action_result['diagnosticsRef']['id']['_value'] + subprocess.check_call([ + 'xcrun', 'xcresulttool', 'export', '--path', xcresult_path, + '--output-path', output_path, '--type', 'directory', '--id', + diagnostics_id + ]) diff --git a/xctestrunner/test_runner/xctest_session.py b/xctestrunner/test_runner/xctest_session.py index 6e3751f..a767a6f 100644 --- a/xctestrunner/test_runner/xctest_session.py +++ b/xctestrunner/test_runner/xctest_session.py @@ -14,6 +14,7 @@ """The module to run XCTEST based tests.""" +import glob import logging import os import shutil @@ -24,10 +25,8 @@ from xctestrunner.shared import ios_constants from xctestrunner.shared import ios_errors from xctestrunner.shared import xcode_info_util -from xctestrunner.test_runner import dummy_project from xctestrunner.test_runner import logic_test_util -from xctestrunner.test_runner import runner_exit_codes -from xctestrunner.test_runner import test_summaries_util +from xctestrunner.test_runner import xcresult_util from xctestrunner.test_runner import xctestrun @@ -60,7 +59,6 @@ def __init__(self, sdk, device_arch, work_dir=None, output_dir=None): self._startup_timeout_sec = None self._destination_timeout_sec = None self._xctestrun_obj = None - self._dummy_project_obj = None self._prepared = False # The following fields are only for Logic Test. self._logic_test_bundle = None @@ -84,7 +82,7 @@ def Prepare(self, app_under_test=None, test_bundle=None, """Prepares the test session. If xctestrun_file is not provided, will use app under test and test bundle - path to generate a new xctest file or dummy project. + path to generate a new xctest file. Args: app_under_test: string, the path of the application to be tested. It can @@ -123,11 +121,6 @@ def Prepare(self, app_under_test=None, test_bundle=None, self._delete_output_dir = True if xctestrun_file_path: - xcode_version_num = xcode_info_util.GetXcodeVersionNumber() - if xcode_version_num < 800: - raise ios_errors.IllegalArgumentError( - 'The xctestrun file is only supported in Xcode 8+. But current ' - 'Xcode version number is %s' % xcode_version_num) self._xctestrun_obj = xctestrun.XctestRun( xctestrun_file_path, test_type) else: @@ -140,37 +133,18 @@ def Prepare(self, app_under_test=None, test_bundle=None, test_bundle_dir, self._sdk, app_under_test_dir=app_under_test_dir, original_test_type=test_type) - # xctestrun can only support in Xcode 8+. - # Since xctestrun approach is more flexiable to local debug and is easy to - # support tests_to_run feature. So in Xcode 8+, use xctestrun approach to - # run XCTest and Logic Test. - if (test_type in ios_constants.SUPPORTED_TEST_TYPES and - test_type != ios_constants.TestType.LOGIC_TEST and - xcode_info_util.GetXcodeVersionNumber() >= 800): + if test_type not in ios_constants.SUPPORTED_TEST_TYPES: + raise ios_errors.IllegalArgumentError( + 'The test type %s is not supported. Supported test types are %s' % + (test_type, ios_constants.SUPPORTED_TEST_TYPES)) + + if test_type != ios_constants.TestType.LOGIC_TEST: xctestrun_factory = xctestrun.XctestRunFactory( app_under_test_dir, test_bundle_dir, self._sdk, self._device_arch, test_type, signing_options, self._work_dir) self._xctestrun_obj = xctestrun_factory.GenerateXctestrun() - elif test_type == ios_constants.TestType.XCUITEST: - raise ios_errors.IllegalArgumentError( - 'Only supports running XCUITest under Xcode 8+. ' - 'Current xcode version is %s' % - xcode_info_util.GetXcodeVersionNumber()) - elif test_type == ios_constants.TestType.XCTEST: - self._dummy_project_obj = dummy_project.DummyProject( - app_under_test_dir, - test_bundle_dir, - self._sdk, - ios_constants.TestType.XCTEST, - self._work_dir, - keychain_path=signing_options.get('keychain_path') or None) - self._dummy_project_obj.GenerateDummyProject() - elif test_type == ios_constants.TestType.LOGIC_TEST: - self._logic_test_bundle = test_bundle_dir else: - raise ios_errors.IllegalArgumentError( - 'The test type %s is not supported. Supported test types are %s' - % (test_type, ios_constants.SUPPORTED_TEST_TYPES)) + self._logic_test_bundle = test_bundle_dir self._prepared = True def SetLaunchOptions(self, launch_options): @@ -207,20 +181,17 @@ def SetLaunchOptions(self, launch_options): self._xctestrun_obj.DeleteXctestrunField('SystemAttachmentLifetime') except ios_errors.PlistError: pass - elif self._dummy_project_obj: - self._dummy_project_obj.SetEnvVars(launch_options.get('env_vars')) - self._dummy_project_obj.SetArgs(launch_options.get('args')) - self._dummy_project_obj.SetSkipTests(launch_options.get('skip_tests')) elif self._logic_test_bundle: self._logic_test_env_vars = launch_options.get('env_vars') self._logic_test_args = launch_options.get('args') self._logic_tests_to_run = launch_options.get('tests_to_run') - def RunTest(self, device_id): + def RunTest(self, device_id, os_version=None): """Runs test on the target device with the given device_id. Args: device_id: string, id of the device. + os_version: string, OS version of the device. Returns: A value of type runner_exit_codes.EXITCODE. @@ -237,27 +208,24 @@ def RunTest(self, device_id): exit_code = self._xctestrun_obj.Run(device_id, self._sdk, self._output_dir, self._startup_timeout_sec, - self._destination_timeout_sec) - for test_summaries_path in test_summaries_util.GetTestSummariesPaths( - self._output_dir): - try: - test_summaries_util.ParseTestSummaries( - test_summaries_path, - os.path.join(self._output_dir, 'Logs/Test/Attachments'), - True if self._disable_uitest_auto_screenshots else - exit_code == runner_exit_codes.EXITCODE.SUCCEEDED) - except ios_errors.PlistError as e: - logging.warning('Failed to parse test summaries %s: %s', - test_summaries_path, str(e)) + self._destination_timeout_sec, + os_version=os_version) + # The xcresult only contains raw data in Xcode 11 or later. + if xcode_info_util.GetXcodeVersionNumber() >= 1100: + test_log_dir = '%s/Logs/Test' % self._output_dir + xcresults = glob.glob('%s/*.xcresult' % test_log_dir) + for xcresult in xcresults: + xcresult_util.ExposeDiagnosticsRef(xcresult, test_log_dir) + shutil.rmtree(xcresult) return exit_code - elif self._dummy_project_obj: - return self._dummy_project_obj.RunXcTest(device_id, self._work_dir, - self._output_dir, - self._startup_timeout_sec) elif self._logic_test_bundle: return logic_test_util.RunLogicTestOnSim( - device_id, self._logic_test_bundle, self._logic_test_env_vars, - self._logic_test_args, self._logic_tests_to_run) + device_id, + self._logic_test_bundle, + self._logic_test_env_vars, + self._logic_test_args, + self._logic_tests_to_run, + os_version=os_version) else: raise ios_errors.XcodebuildTestError('Unexpected runtime error.') diff --git a/xctestrunner/test_runner/xctestrun.py b/xctestrunner/test_runner/xctestrun.py index 0e306bc..27183a1 100644 --- a/xctestrunner/test_runner/xctestrun.py +++ b/xctestrunner/test_runner/xctestrun.py @@ -23,6 +23,7 @@ from xctestrunner.shared import ios_constants from xctestrunner.shared import ios_errors from xctestrunner.shared import plist_util +from xctestrunner.shared import version_util from xctestrunner.shared import xcode_info_util from xctestrunner.test_runner import xcodebuild_test_executor @@ -141,7 +142,7 @@ def SetSkipTests(self, skip_tests): self.SetXctestrunField('SkipTestIdentifiers', skip_tests) def Run(self, device_id, sdk, derived_data_dir, startup_timeout_sec, - destination_timeout_sec=None): + destination_timeout_sec=None, os_version=None): """Runs the test with generated xctestrun file in the specific device. Args: @@ -151,10 +152,23 @@ def Run(self, device_id, sdk, derived_data_dir, startup_timeout_sec, startup_timeout_sec: seconds until the xcodebuild command is deemed stuck. destination_timeout_sec: Wait for the given seconds while searching for the destination device. + os_version: os version of the device. Returns: A value of type runner_exit_codes.EXITCODE. """ + # When running tests on iOS 12.1 or earlier simulator under Xcode 11 or + # later, it is required to add swift5 fallback libraries to environment + # variable. + # See https://github.com/bazelbuild/rules_apple/issues/684 for context. + if (xcode_info_util.GetXcodeVersionNumber() >= 1100 and + sdk == ios_constants.SDK.IPHONESIMULATOR and os_version and + version_util.GetVersionNumber(os_version) < 1220): + new_env_var = { + 'DYLD_FALLBACK_LIBRARY_PATH': + xcode_info_util.GetSwift5FallbackLibsDir() + } + self.SetTestEnvVars(new_env_var) logging.info('Running test-without-building with device %s', device_id) command = ['xcodebuild', 'test-without-building', '-xctestrun', self._xctestrun_file_path, @@ -478,15 +492,6 @@ def _GenerateTestRootForXcuitest(self): developer=developer_path), 'DYLD_LIBRARY_PATH': '__TESTROOT__:%s/usr/lib' % developer_path } - - # Fixes failures for UI test targets that depend on Swift libraries when running with Xcode 11 - # on pre-iOS 12.2 simulators. - # Example failure message this resolves: "The bundle couldn’t be loaded because it is damaged - # or missing necessary resources." - swift5FallbackLibsDir = xcode_info_util.GetSwift5FallbackLibsDir() - if swift5FallbackLibsDir: - test_envs["DYLD_FALLBACK_LIBRARY_PATH"] = swift5FallbackLibsDir - self._xctestrun_dict = { 'IsUITestBundle': True, 'SystemAttachmentLifetime': 'keepNever', @@ -636,15 +641,6 @@ def _GenerateTestRootForXctest(self): 'DYLD_INSERT_LIBRARIES': dyld_insert_libs, 'DYLD_LIBRARY_PATH': '__TESTROOT__:%s/usr/lib:' % developer_path } - - # Fixes failures for test targets that depend on Swift libraries when running with Xcode 11 - # on pre-iOS 12.2 simulators. - # Example failure message this resolves: "The bundle couldn’t be loaded because it is damaged - # or missing necessary resources." - swift5FallbackLibsDir = xcode_info_util.GetSwift5FallbackLibsDir() - if swift5FallbackLibsDir: - test_envs["DYLD_FALLBACK_LIBRARY_PATH"] = swift5FallbackLibsDir - self._xctestrun_dict = { 'TestHostPath': self._app_under_test_dir, 'TestBundlePath': self._test_bundle_dir, @@ -665,15 +661,6 @@ def _GenerateTestRootForLogicTest(self): 'DYLD_FRAMEWORK_PATH': dyld_framework_path, 'DYLD_LIBRARY_PATH': dyld_framework_path } - - # Fixes failures for unit test targets that depend on Swift libraries when running with Xcode 11 - # on pre-iOS 12.2 simulators. - # Example failure message this resolves: "The bundle couldn’t be loaded because it is damaged - # or missing necessary resources." - swift5FallbackLibsDir = xcode_info_util.GetSwift5FallbackLibsDir() - if swift5FallbackLibsDir: - test_envs["DYLD_FALLBACK_LIBRARY_PATH"] = swift5FallbackLibsDir - self._xctestrun_dict = { 'TestBundlePath': self._test_bundle_dir, 'TestHostPath': xcode_info_util.GetXctestToolPath(self._sdk), From 401884575c83a285ee4404b7da9d29dc06d0bc28 Mon Sep 17 00:00:00 2001 From: Weiming Dai Date: Tue, 26 May 2020 17:25:04 -0700 Subject: [PATCH 07/45] Remove python_version attribute With the python_version attribute, the par binary could not work on macOS 10.14. See https://github.com/google/xctestrunner/issues/18 for details. --- xctestrunner/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/xctestrunner/BUILD b/xctestrunner/BUILD index b528fe8..bc06659 100644 --- a/xctestrunner/BUILD +++ b/xctestrunner/BUILD @@ -27,5 +27,4 @@ par_binary( ':simulator', ], data = glob(['test_runner/TestProject/**']), - python_version = 'PY2', ) From 41b18a2640da5dd8403625223e2521e291abd100 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Mon, 22 Jun 2020 09:09:07 -0700 Subject: [PATCH 08/45] Fix python2 interpreter By default when specifying PY2 as the `python_version` the shebang in the produced parfile is `/usr/bin/env python2` which doesn't exist on macOS (unless users have created it). This fix forces the shebang to be `/usr/bin/python2.7` which exists on all recent macOS versions. Once this project is converted to python3, we should change this to `/usr/bin/python3` instead. --- xctestrunner/BUILD | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/xctestrunner/BUILD b/xctestrunner/BUILD index bc06659..f3a3ee2 100644 --- a/xctestrunner/BUILD +++ b/xctestrunner/BUILD @@ -3,28 +3,33 @@ package(default_visibility = ["//visibility:public"]) load("@subpar//:subpar.bzl", "par_binary") py_library( - name = 'shared', - srcs = glob(['shared/*.py']), + name = "shared", + srcs = glob(["shared/*.py"]), ) py_library( - name = 'simulator', - srcs = glob(['simulator_control/*.py']), + name = "simulator", + srcs = glob(["simulator_control/*.py"]), deps = [ - ':shared', + ":shared", ], ) par_binary( - name = 'ios_test_runner', + name = "ios_test_runner", srcs = glob( - ['test_runner/*.py'], - exclude = ['test_runner/TestProject/**'] + ["test_runner/*.py"], + exclude = ["test_runner/TestProject/**"], ), - main = 'test_runner/ios_test_runner.py', + compiler_args = [ + "--interpreter", + "/usr/bin/python2.7", + ], + data = glob(["test_runner/TestProject/**"]), + main = "test_runner/ios_test_runner.py", + python_version = "PY2", deps = [ - ':shared', - ':simulator', + ":shared", + ":simulator", ], - data = glob(['test_runner/TestProject/**']), ) From 67100a8dd85dd65546a9465dfbd7ffd5da962a0e Mon Sep 17 00:00:00 2001 From: Brentley Jones Date: Tue, 30 Jun 2020 13:37:27 -0500 Subject: [PATCH 09/45] Pass DEVELOPER_DIR to `xcrun simctl spawn` In all cases, except `xcrun simctl spawn`, we inherit `os.environ` when running commands. This change passes through DEVELOPER_DIR to allow xcrun to select the correct simctl. Without this change, the only way to get the correct simulator selected, is to use xcode-select, which is a system level change, and prone to errors. With this change, one can run env DEVELOPER_DIR=/Applications/Xcode-12.0b1.app/Contents/Developer bazel test --test_env=DEVELOPER_DIR -- //TestTarget and the correct simulator will be created and used. --- xctestrunner/test_runner/logic_test_util.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/xctestrunner/test_runner/logic_test_util.py b/xctestrunner/test_runner/logic_test_util.py index 8439e4b..c3bd792 100644 --- a/xctestrunner/test_runner/logic_test_util.py +++ b/xctestrunner/test_runner/logic_test_util.py @@ -14,6 +14,7 @@ """The helper classes to run logic test.""" +import os import subprocess import sys @@ -62,6 +63,11 @@ def RunLogicTestOnSim(sim_id, version_util.GetVersionNumber(os_version) < 1220): key = _SIMCTL_ENV_VAR_PREFIX + 'DYLD_FALLBACK_LIBRARY_PATH' simctl_env_vars[key] = xcode_info_util.GetSwift5FallbackLibsDir() + # We need to set the DEVELOPER_DIR to ensure xcrun works correctly + developer_dir = os.environ.get('DEVELOPER_DIR') + if developer_dir: + simctl_env_vars['DEVELOPER_DIR'] = developer_dir + command = [ 'xcrun', 'simctl', 'spawn', '-s', sim_id, xcode_info_util.GetXctestToolPath(ios_constants.SDK.IPHONESIMULATOR)] From 964120cb1364905d8ed3737336c7941dd56b1546 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Wed, 15 Jul 2020 14:50:01 -0700 Subject: [PATCH 10/45] Add xcresult bundle collection This outputs the xcresult bundle to a specific place, and stops removing them so they can be consumed after the test --- xctestrunner/test_runner/xctest_session.py | 12 +++++------- xctestrunner/test_runner/xctestrun.py | 12 ++++++++++-- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/xctestrunner/test_runner/xctest_session.py b/xctestrunner/test_runner/xctest_session.py index a767a6f..7830120 100644 --- a/xctestrunner/test_runner/xctest_session.py +++ b/xctestrunner/test_runner/xctest_session.py @@ -14,7 +14,6 @@ """The module to run XCTEST based tests.""" -import glob import logging import os import shutil @@ -205,18 +204,17 @@ def RunTest(self, device_id, os_version=None): 'XctestSession.Prepare first.') if self._xctestrun_obj: + result_bundle_path = os.path.join( + os.environ['TEST_UNDECLARED_OUTPUTS_DIR'], 'test.xcresult') exit_code = self._xctestrun_obj.Run(device_id, self._sdk, self._output_dir, self._startup_timeout_sec, self._destination_timeout_sec, - os_version=os_version) + os_version=os_version, + result_bundle_path=result_bundle_path) # The xcresult only contains raw data in Xcode 11 or later. if xcode_info_util.GetXcodeVersionNumber() >= 1100: - test_log_dir = '%s/Logs/Test' % self._output_dir - xcresults = glob.glob('%s/*.xcresult' % test_log_dir) - for xcresult in xcresults: - xcresult_util.ExposeDiagnosticsRef(xcresult, test_log_dir) - shutil.rmtree(xcresult) + xcresult_util.ExposeDiagnosticsRef(result_bundle_path, test_log_dir) return exit_code elif self._logic_test_bundle: return logic_test_util.RunLogicTestOnSim( diff --git a/xctestrunner/test_runner/xctestrun.py b/xctestrunner/test_runner/xctestrun.py index 27183a1..6750144 100644 --- a/xctestrunner/test_runner/xctestrun.py +++ b/xctestrunner/test_runner/xctestrun.py @@ -142,7 +142,8 @@ def SetSkipTests(self, skip_tests): self.SetXctestrunField('SkipTestIdentifiers', skip_tests) def Run(self, device_id, sdk, derived_data_dir, startup_timeout_sec, - destination_timeout_sec=None, os_version=None): + destination_timeout_sec=None, os_version=None, + result_bundle_path=None): """Runs the test with generated xctestrun file in the specific device. Args: @@ -153,6 +154,7 @@ def Run(self, device_id, sdk, derived_data_dir, startup_timeout_sec, destination_timeout_sec: Wait for the given seconds while searching for the destination device. os_version: os version of the device. + result_bundle_path: path to output a xcresult bundle to Returns: A value of type runner_exit_codes.EXITCODE. @@ -161,7 +163,8 @@ def Run(self, device_id, sdk, derived_data_dir, startup_timeout_sec, # later, it is required to add swift5 fallback libraries to environment # variable. # See https://github.com/bazelbuild/rules_apple/issues/684 for context. - if (xcode_info_util.GetXcodeVersionNumber() >= 1100 and + xcode_version = xcode_info_util.GetXcodeVersionNumber() + if (xcode_version >= 1100 and sdk == ios_constants.SDK.IPHONESIMULATOR and os_version and version_util.GetVersionNumber(os_version) < 1220): new_env_var = { @@ -174,6 +177,11 @@ def Run(self, device_id, sdk, derived_data_dir, startup_timeout_sec, '-xctestrun', self._xctestrun_file_path, '-destination', 'id=%s' % device_id, '-derivedDataPath', derived_data_dir] + + if xcode_version >= 1100 and result_bundle_path: + shutil.rmtree(result_bundle_path, ignore_errors=True) + command.extend(['-resultBundlePath', result_bundle_path]) + if destination_timeout_sec: command.extend(['-destination-timeout', str(destination_timeout_sec)]) exit_code, _ = xcodebuild_test_executor.XcodebuildTestExecutor( From 74f270efa4c42bd92b8c3bb4d0cab696c2d6c17a Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Wed, 15 Jul 2020 17:41:20 -0700 Subject: [PATCH 11/45] Ignore non diagnosticsRef ids Potential fix for https://github.com/google/xctestrunner/issues/23 --- xctestrunner/test_runner/xcresult_util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xctestrunner/test_runner/xcresult_util.py b/xctestrunner/test_runner/xcresult_util.py index dc5a934..f42a9de 100644 --- a/xctestrunner/test_runner/xcresult_util.py +++ b/xctestrunner/test_runner/xcresult_util.py @@ -37,6 +37,8 @@ def ExposeDiagnosticsRef(xcresult_path, output_path): raise ios_errors.XcresultError( 'Failed to get "ActionResult" from result bundle %s' % output) + if 'diagnosticsRef' not in action_result: + return diagnostics_id = action_result['diagnosticsRef']['id']['_value'] subprocess.check_call([ 'xcrun', 'xcresulttool', 'export', '--path', xcresult_path, From 269e9f809504a7ba41429feb160dc0c3c6efe313 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Tue, 4 Aug 2020 12:28:38 -0700 Subject: [PATCH 12/45] output_dir --- xctestrunner/test_runner/xctest_session.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xctestrunner/test_runner/xctest_session.py b/xctestrunner/test_runner/xctest_session.py index 7830120..71cfc45 100644 --- a/xctestrunner/test_runner/xctest_session.py +++ b/xctestrunner/test_runner/xctest_session.py @@ -204,8 +204,7 @@ def RunTest(self, device_id, os_version=None): 'XctestSession.Prepare first.') if self._xctestrun_obj: - result_bundle_path = os.path.join( - os.environ['TEST_UNDECLARED_OUTPUTS_DIR'], 'test.xcresult') + result_bundle_path = os.path.join(self._output_dir, 'test.xcresult') exit_code = self._xctestrun_obj.Run(device_id, self._sdk, self._output_dir, self._startup_timeout_sec, From 3aeb41941de0ca55247a0e9133bfb96c8144494a Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Tue, 4 Aug 2020 12:57:16 -0700 Subject: [PATCH 13/45] add option --- xctestrunner/test_runner/xctest_session.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/xctestrunner/test_runner/xctest_session.py b/xctestrunner/test_runner/xctest_session.py index 71cfc45..05d6d4c 100644 --- a/xctestrunner/test_runner/xctest_session.py +++ b/xctestrunner/test_runner/xctest_session.py @@ -59,6 +59,7 @@ def __init__(self, sdk, device_arch, work_dir=None, output_dir=None): self._destination_timeout_sec = None self._xctestrun_obj = None self._prepared = False + self._keep_xcresult_data = True # The following fields are only for Logic Test. self._logic_test_bundle = None self._logic_test_env_vars = None @@ -159,6 +160,7 @@ def SetLaunchOptions(self, launch_options): 'XctestSession.Prepare first.') if not launch_options: return + self._keep_xcresult_data = launch_options.get('keep_xcresult_data', True) self._startup_timeout_sec = launch_options.get('startup_timeout_sec') self._destination_timeout_sec = launch_options.get( 'destination_timeout_sec') @@ -214,6 +216,8 @@ def RunTest(self, device_id, os_version=None): # The xcresult only contains raw data in Xcode 11 or later. if xcode_info_util.GetXcodeVersionNumber() >= 1100: xcresult_util.ExposeDiagnosticsRef(result_bundle_path, test_log_dir) + if not self._keep_xcresult_data: + shutil.rmtree(result_bundle_path) return exit_code elif self._logic_test_bundle: return logic_test_util.RunLogicTestOnSim( From cefec9db094fe9de21cc1631d478b633a9f1b29e Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Tue, 4 Aug 2020 15:31:26 -0700 Subject: [PATCH 14/45] docs --- xctestrunner/shared/ios_constants.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/xctestrunner/shared/ios_constants.py b/xctestrunner/shared/ios_constants.py index 2136b1c..54403eb 100644 --- a/xctestrunner/shared/ios_constants.py +++ b/xctestrunner/shared/ios_constants.py @@ -67,6 +67,9 @@ def enum(**enums): In xctest, the functionality is the same as "args". In xcuitest, the process of app under test is different with the process of test. + keep_xcresult_data: bool + Whether or not to keep the xcresult bundle produced by the test run + in the output_dir. tests_to_run : array The specific test classes or test methods to run. Each item should be string and its format is Test-Class-Name[/Test-Method-Name]. It is supported From 81bf46834b11232b14e3db75e672c72c00701230 Mon Sep 17 00:00:00 2001 From: Weiming Dai Date: Tue, 4 Aug 2020 16:38:03 -0700 Subject: [PATCH 15/45] Xcode 12.0 compatible change on device testing. Xcode 12.0 compatible change on device testing. --- xctestrunner/test_runner/xctestrun.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/xctestrunner/test_runner/xctestrun.py b/xctestrunner/test_runner/xctestrun.py index 6750144..1aecad5 100644 --- a/xctestrunner/test_runner/xctestrun.py +++ b/xctestrunner/test_runner/xctestrun.py @@ -634,13 +634,21 @@ def _GenerateTestRootForXctest(self): os.path.basename(self._app_under_test_dir))[0] platform_name = 'iPhoneOS' if self._on_device else 'iPhoneSimulator' developer_path = '__PLATFORMS__/%s.platform/Developer' % platform_name - if xcode_info_util.GetXcodeVersionNumber() < 1000: - dyld_insert_libs = ('%s/Library/PrivateFrameworks/' - 'IDEBundleInjection.framework/IDEBundleInjection' % - developer_path) + + if self._on_device: + if xcode_info_util.GetXcodeVersionNumber() < 1000: + dyld_insert_libs = ('__TESTHOST__/Frameworks/' + 'IDEBundleInjection.framework/IDEBundleInjection') + else: + dyld_insert_libs = '__TESTHOST__/Frameworks/libXCTestBundleInject.dylib' else: - dyld_insert_libs = ('%s/usr/lib/libXCTestBundleInject.dylib' % - developer_path) + if xcode_info_util.GetXcodeVersionNumber() < 1000: + dyld_insert_libs = ('%s/Library/PrivateFrameworks/' + 'IDEBundleInjection.framework/IDEBundleInjection' % + developer_path) + else: + dyld_insert_libs = ('%s/usr/lib/libXCTestBundleInject.dylib' % + developer_path) test_envs = { 'XCInjectBundleInto': os.path.join('__TESTHOST__', app_under_test_name), 'DYLD_FRAMEWORK_PATH': '__TESTROOT__:{developer}/Library/Frameworks:' From a9dd528abf1f2ba2d1a4e92db63b7be2196a963a Mon Sep 17 00:00:00 2001 From: Weiming Dai Date: Tue, 4 Aug 2020 16:44:16 -0700 Subject: [PATCH 16/45] Expose crash reports from xcresult bundle - Expose crash reports from xcresult bundle - Fix a programming error for missing test_log_dir --- xctestrunner/test_runner/xcresult_util.py | 103 +++++++++++++++++++-- xctestrunner/test_runner/xctest_session.py | 10 +- 2 files changed, 101 insertions(+), 12 deletions(-) diff --git a/xctestrunner/test_runner/xcresult_util.py b/xctestrunner/test_runner/xcresult_util.py index f42a9de..599de4e 100644 --- a/xctestrunner/test_runner/xcresult_util.py +++ b/xctestrunner/test_runner/xcresult_util.py @@ -15,19 +15,23 @@ """Helper class for parsing xcresult under Xcode 11 or later.""" import json +import os import subprocess from xctestrunner.shared import ios_errors -def ExposeDiagnosticsRef(xcresult_path, output_path): - """Exposes the DiagnosticsRef files from the given xcresult file.""" - output = subprocess.check_output([ - 'xcrun', 'xcresulttool', 'get', '--format', 'json', '--path', - xcresult_path - ]) - result_bundle_json = json.loads(output) - actions = result_bundle_json['actions']['_values'] +def ExpoesXcresult(xcresult_path, output_path): + """Exposes the files from xcresult. + + The files includes the diagnostics files and attachments files. + + Args: + xcresult_path: string, path of xcresult bundle. + output_path: string, path of output directory. + """ + root_result_bundle = _GetResultBundleObject(xcresult_path, bundle_id=None) + actions = root_result_bundle['actions']['_values'] action_result = None for action in actions: if action['_type']['_name'] == 'ActionRecord': @@ -35,8 +39,14 @@ def ExposeDiagnosticsRef(xcresult_path, output_path): break if action_result is None: raise ios_errors.XcresultError( - 'Failed to get "ActionResult" from result bundle %s' % output) + 'Failed to get "ActionResult" from result bundle %s' % + root_result_bundle) + _ExposeDiagnostics(xcresult_path, output_path, action_result) + _ExposeAttachments(xcresult_path, output_path, action_result) + +def _ExposeDiagnostics(xcresult_path, output_path, action_result): + """Exposes the diagnostics files from the given xcresult file.""" if 'diagnosticsRef' not in action_result: return diagnostics_id = action_result['diagnosticsRef']['id']['_value'] @@ -45,3 +55,78 @@ def ExposeDiagnosticsRef(xcresult_path, output_path): '--output-path', output_path, '--type', 'directory', '--id', diagnostics_id ]) + + +def _ExposeAttachments(xcresult_path, output_path, action_result): + """Exposes the attachments files from the given xcresult file.""" + testsref_id = action_result['testsRef']['id']['_value'] + test_plan_summaries = _GetResultBundleObject( + xcresult_path, bundle_id=testsref_id) + test_plan_summary = test_plan_summaries['summaries']['_values'][0] + testable_summary = test_plan_summary['testableSummaries']['_values'][0] + # If the app under test crashes in unit test (XCTest) before loading the + # tests, the testable summary won't have tests summary. + if 'tests' not in testable_summary: + return + root_tests_summary = testable_summary['tests']['_values'][0] + failure_test_ref_ids = _GetFailureTestRefs(root_tests_summary) + for test_ref_id in failure_test_ref_ids: + test_summary_result = _GetResultBundleObject(xcresult_path, test_ref_id) + activity_summaries = test_summary_result['activitySummaries']['_values'] + for activity_summary in activity_summaries: + if 'attachments' in activity_summary: + test_identifier = test_summary_result['identifier']['_value'] + for attachment in activity_summary['attachments']['_values']: + file_name = attachment['filename']['_value'] + target_file_dir = os.path.join(output_path, 'Attachments', + test_identifier) + if not os.path.exists(target_file_dir): + os.makedirs(target_file_dir) + target_file_path = os.path.join(target_file_dir, file_name) + + payload_ref_id = attachment['payloadRef']['id']['_value'] + subprocess.check_call([ + 'xcrun', 'xcresulttool', 'export', '--path', xcresult_path, + '--output-path', target_file_path, '--type', 'file', '--id', + payload_ref_id + ]) + + +def _GetResultBundleObject(xcresult_path, bundle_id=None): + """Gets the result bundle object in json format. + + Args: + xcresult_path: string, path of xcresult bundle. + bundle_id: string, id of the result bundle object. If it is None, it is + rootID. + Returns: + A dict, result bundle object in json format. + """ + command = [ + 'xcrun', 'xcresulttool', 'get', '--format', 'json', '--path', + xcresult_path + ] + if bundle_id: + command.extend(['--id', bundle_id]) + return json.loads(subprocess.check_output(command)) + + +def _GetFailureTestRefs(test_summary): + """Gets a list of test summaryRef id of all failure test. + + Args: + test_summary: dict, a dict of test summary object. + Returns: + A list of failure test case's summaryRef id. + """ + failure_test_refs = [] + if 'subtests' in test_summary: + for sub_test_summary in test_summary['subtests']['_values']: + failure_test_refs.extend(_GetFailureTestRefs(sub_test_summary)) + else: + if (('testStatus' not in test_summary or + test_summary['testStatus']['_value'] != 'Success') and + 'summaryRef' in test_summary): + summary_ref_id = test_summary['summaryRef']['id']['_value'] + failure_test_refs.append(summary_ref_id) + return failure_test_refs diff --git a/xctestrunner/test_runner/xctest_session.py b/xctestrunner/test_runner/xctest_session.py index 05d6d4c..9e6d694 100644 --- a/xctestrunner/test_runner/xctest_session.py +++ b/xctestrunner/test_runner/xctest_session.py @@ -215,9 +215,13 @@ def RunTest(self, device_id, os_version=None): result_bundle_path=result_bundle_path) # The xcresult only contains raw data in Xcode 11 or later. if xcode_info_util.GetXcodeVersionNumber() >= 1100: - xcresult_util.ExposeDiagnosticsRef(result_bundle_path, test_log_dir) - if not self._keep_xcresult_data: - shutil.rmtree(result_bundle_path) + expose_xcresult = os.path.join(self._output_dir, 'ExposeXcresult') + try: + xcresult_util.ExpoesXcresult(result_bundle_path, expose_xcresult) + if not self._keep_xcresult_data: + shutil.rmtree(result_bundle_path) + except subprocess.CalledProcessError as e: + logging.warning(e.output) return exit_code elif self._logic_test_bundle: return logic_test_util.RunLogicTestOnSim( From 9f4e1ce0134f0b66a3de68a6ea8d798ef7a81d08 Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Fri, 4 Sep 2020 21:37:50 -0700 Subject: [PATCH 17/45] Allow running the test runner binary target from another repository --- xctestrunner/BUILD => BUILD | 2 +- README.md | 4 ++-- WORKSPACE | 2 ++ xctestrunner/__init__.py => __init__.py | 0 {xctestrunner/shared => shared}/__init__.py | 0 {xctestrunner/shared => shared}/bundle_util.py | 0 {xctestrunner/shared => shared}/ios_constants.py | 0 {xctestrunner/shared => shared}/ios_errors.py | 0 {xctestrunner/shared => shared}/plist_util.py | 0 {xctestrunner/shared => shared}/provisioning_profile.py | 0 {xctestrunner/shared => shared}/version_util.py | 0 {xctestrunner/shared => shared}/xcode_info_util.py | 0 .../simulator_control => simulator_control}/__init__.py | 0 .../simtype_profile.py | 0 .../simulator_control => simulator_control}/simulator_util.py | 0 {xctestrunner/test_runner => test_runner}/__init__.py | 0 {xctestrunner/test_runner => test_runner}/ios_test_runner.py | 0 {xctestrunner/test_runner => test_runner}/logic_test_util.py | 0 .../test_runner => test_runner}/runner_exit_codes.py | 0 .../test_runner => test_runner}/xcodebuild_test_executor.py | 0 {xctestrunner/test_runner => test_runner}/xcresult_util.py | 0 {xctestrunner/test_runner => test_runner}/xctest_session.py | 0 {xctestrunner/test_runner => test_runner}/xctestrun.py | 0 23 files changed, 5 insertions(+), 3 deletions(-) rename xctestrunner/BUILD => BUILD (95%) rename xctestrunner/__init__.py => __init__.py (100%) rename {xctestrunner/shared => shared}/__init__.py (100%) rename {xctestrunner/shared => shared}/bundle_util.py (100%) rename {xctestrunner/shared => shared}/ios_constants.py (100%) rename {xctestrunner/shared => shared}/ios_errors.py (100%) rename {xctestrunner/shared => shared}/plist_util.py (100%) rename {xctestrunner/shared => shared}/provisioning_profile.py (100%) rename {xctestrunner/shared => shared}/version_util.py (100%) rename {xctestrunner/shared => shared}/xcode_info_util.py (100%) rename {xctestrunner/simulator_control => simulator_control}/__init__.py (100%) rename {xctestrunner/simulator_control => simulator_control}/simtype_profile.py (100%) rename {xctestrunner/simulator_control => simulator_control}/simulator_util.py (100%) rename {xctestrunner/test_runner => test_runner}/__init__.py (100%) rename {xctestrunner/test_runner => test_runner}/ios_test_runner.py (100%) rename {xctestrunner/test_runner => test_runner}/logic_test_util.py (100%) rename {xctestrunner/test_runner => test_runner}/runner_exit_codes.py (100%) rename {xctestrunner/test_runner => test_runner}/xcodebuild_test_executor.py (100%) rename {xctestrunner/test_runner => test_runner}/xcresult_util.py (100%) rename {xctestrunner/test_runner => test_runner}/xctest_session.py (100%) rename {xctestrunner/test_runner => test_runner}/xctestrun.py (100%) diff --git a/xctestrunner/BUILD b/BUILD similarity index 95% rename from xctestrunner/BUILD rename to BUILD index f3a3ee2..ba603d4 100644 --- a/xctestrunner/BUILD +++ b/BUILD @@ -17,7 +17,7 @@ py_library( par_binary( name = "ios_test_runner", - srcs = glob( + srcs = ["__init__.py"] + glob( ["test_runner/*.py"], exclude = ["test_runner/TestProject/**"], ), diff --git a/README.md b/README.md index 7945cf6..596ac47 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,8 @@ or build the ios_test_runner.par binary by bazel: ``` $ git clone https://github.com/google/xctestrunner.git $ cd xctestrunner -$ bazel build xctestrunner:ios_test_runner.par -$ ls bazel-bin/xctestrunner/ios_test_runner.par +$ bazel build :ios_test_runner.par +$ ls bazel-bin/ios_test_runner.par ``` ## Usage diff --git a/WORKSPACE b/WORKSPACE index 497b765..c29c882 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,3 +1,5 @@ +workspace(name = "xctestrunner") + # For packaging python scripts. load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") diff --git a/xctestrunner/__init__.py b/__init__.py similarity index 100% rename from xctestrunner/__init__.py rename to __init__.py diff --git a/xctestrunner/shared/__init__.py b/shared/__init__.py similarity index 100% rename from xctestrunner/shared/__init__.py rename to shared/__init__.py diff --git a/xctestrunner/shared/bundle_util.py b/shared/bundle_util.py similarity index 100% rename from xctestrunner/shared/bundle_util.py rename to shared/bundle_util.py diff --git a/xctestrunner/shared/ios_constants.py b/shared/ios_constants.py similarity index 100% rename from xctestrunner/shared/ios_constants.py rename to shared/ios_constants.py diff --git a/xctestrunner/shared/ios_errors.py b/shared/ios_errors.py similarity index 100% rename from xctestrunner/shared/ios_errors.py rename to shared/ios_errors.py diff --git a/xctestrunner/shared/plist_util.py b/shared/plist_util.py similarity index 100% rename from xctestrunner/shared/plist_util.py rename to shared/plist_util.py diff --git a/xctestrunner/shared/provisioning_profile.py b/shared/provisioning_profile.py similarity index 100% rename from xctestrunner/shared/provisioning_profile.py rename to shared/provisioning_profile.py diff --git a/xctestrunner/shared/version_util.py b/shared/version_util.py similarity index 100% rename from xctestrunner/shared/version_util.py rename to shared/version_util.py diff --git a/xctestrunner/shared/xcode_info_util.py b/shared/xcode_info_util.py similarity index 100% rename from xctestrunner/shared/xcode_info_util.py rename to shared/xcode_info_util.py diff --git a/xctestrunner/simulator_control/__init__.py b/simulator_control/__init__.py similarity index 100% rename from xctestrunner/simulator_control/__init__.py rename to simulator_control/__init__.py diff --git a/xctestrunner/simulator_control/simtype_profile.py b/simulator_control/simtype_profile.py similarity index 100% rename from xctestrunner/simulator_control/simtype_profile.py rename to simulator_control/simtype_profile.py diff --git a/xctestrunner/simulator_control/simulator_util.py b/simulator_control/simulator_util.py similarity index 100% rename from xctestrunner/simulator_control/simulator_util.py rename to simulator_control/simulator_util.py diff --git a/xctestrunner/test_runner/__init__.py b/test_runner/__init__.py similarity index 100% rename from xctestrunner/test_runner/__init__.py rename to test_runner/__init__.py diff --git a/xctestrunner/test_runner/ios_test_runner.py b/test_runner/ios_test_runner.py similarity index 100% rename from xctestrunner/test_runner/ios_test_runner.py rename to test_runner/ios_test_runner.py diff --git a/xctestrunner/test_runner/logic_test_util.py b/test_runner/logic_test_util.py similarity index 100% rename from xctestrunner/test_runner/logic_test_util.py rename to test_runner/logic_test_util.py diff --git a/xctestrunner/test_runner/runner_exit_codes.py b/test_runner/runner_exit_codes.py similarity index 100% rename from xctestrunner/test_runner/runner_exit_codes.py rename to test_runner/runner_exit_codes.py diff --git a/xctestrunner/test_runner/xcodebuild_test_executor.py b/test_runner/xcodebuild_test_executor.py similarity index 100% rename from xctestrunner/test_runner/xcodebuild_test_executor.py rename to test_runner/xcodebuild_test_executor.py diff --git a/xctestrunner/test_runner/xcresult_util.py b/test_runner/xcresult_util.py similarity index 100% rename from xctestrunner/test_runner/xcresult_util.py rename to test_runner/xcresult_util.py diff --git a/xctestrunner/test_runner/xctest_session.py b/test_runner/xctest_session.py similarity index 100% rename from xctestrunner/test_runner/xctest_session.py rename to test_runner/xctest_session.py diff --git a/xctestrunner/test_runner/xctestrun.py b/test_runner/xctestrun.py similarity index 100% rename from xctestrunner/test_runner/xctestrun.py rename to test_runner/xctestrun.py From e012c920c245a8012dc86dff6e94e97ee4734a7e Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Tue, 8 Sep 2020 22:58:17 -0700 Subject: [PATCH 18/45] Python 3 Support --- BUILD | 4 ++-- shared/bundle_util.py | 6 +++--- shared/plist_util.py | 2 +- shared/xcode_info_util.py | 10 +++++----- simulator_control/simulator_util.py | 2 +- test_runner/ios_test_runner.py | 2 +- test_runner/xcodebuild_test_executor.py | 6 +++--- test_runner/xcresult_util.py | 2 +- test_runner/xctest_session.py | 2 +- test_runner/xctestrun.py | 4 ++-- 10 files changed, 20 insertions(+), 20 deletions(-) diff --git a/BUILD b/BUILD index ba603d4..aed3d72 100644 --- a/BUILD +++ b/BUILD @@ -23,11 +23,11 @@ par_binary( ), compiler_args = [ "--interpreter", - "/usr/bin/python2.7", + "/usr/bin/python3", ], data = glob(["test_runner/TestProject/**"]), main = "test_runner/ios_test_runner.py", - python_version = "PY2", + python_version = "PY3", deps = [ ":shared", ":simulator", diff --git a/shared/bundle_util.py b/shared/bundle_util.py index 343315d..2c09eca 100644 --- a/shared/bundle_util.py +++ b/shared/bundle_util.py @@ -126,7 +126,7 @@ def GetCodesignIdentity(bundle_path): """ command = ('codesign', '-dvv', bundle_path) process = subprocess.Popen(command, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, text=True) output = process.communicate()[0] for line in output.split('\n'): if line.startswith('Authority='): @@ -151,7 +151,7 @@ def GetDevelopmentTeam(bundle_path): """ command = ('codesign', '-dvv', bundle_path) process = subprocess.Popen(command, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, text=True) output = process.communicate()[0] for line in output.split('\n'): if line.startswith('TeamIdentifier='): @@ -217,7 +217,7 @@ def EnableUIFileSharing(bundle_path, resigning=True): def GetFileArchTypes(file_path): """Gets the architecture types of the file.""" - output = subprocess.check_output(['/usr/bin/lipo', file_path, '-archs']) + output = subprocess.check_output(['/usr/bin/lipo', file_path, '-archs'], text=True) return output.split(' ') diff --git a/shared/plist_util.py b/shared/plist_util.py index bf79b5a..d431305 100644 --- a/shared/plist_util.py +++ b/shared/plist_util.py @@ -271,7 +271,7 @@ def _GetPlistFieldByPlistBuddy(plist_path, field): """ command = [PLIST_BUDDY, '-c', 'Print :"%s"' % field, plist_path] try: - return subprocess.check_output(command, stderr=subprocess.STDOUT).strip() + return subprocess.check_output(command, stderr=subprocess.STDOUT, text=True).strip() except subprocess.CalledProcessError as e: raise ios_errors.PlistError( 'Failed to get field %s in plist %s: %s', field, plist_path, e.output) diff --git a/shared/xcode_info_util.py b/shared/xcode_info_util.py index ed4ecab..7ba0989 100644 --- a/shared/xcode_info_util.py +++ b/shared/xcode_info_util.py @@ -26,7 +26,7 @@ def GetXcodeDeveloperPath(): """Gets the active developer path of Xcode command line tools.""" - return subprocess.check_output(('xcode-select', '-p')).strip() + return subprocess.check_output(('xcode-select', '-p'), text=True).strip() # Xcode 11+'s Swift dylibs are configured in a way that does not allow them to @@ -61,7 +61,7 @@ def GetXcodeVersionNumber(): # Example output: # Xcode 8.2.1 # Build version 8C1002 - output = subprocess.check_output(('xcodebuild', '-version')) + output = subprocess.check_output(('xcodebuild', '-version'), text=True) xcode_version = output.split('\n')[0].split(' ')[1] # Add cache xcode_version_number to avoid calling subprocess multiple times. # It is expected that no one changes xcode during the test runner working. @@ -72,13 +72,13 @@ def GetXcodeVersionNumber(): def GetSdkPlatformPath(sdk): """Gets the selected SDK platform path.""" return subprocess.check_output( - ['xcrun', '--sdk', sdk, '--show-sdk-platform-path']).strip() + ['xcrun', '--sdk', sdk, '--show-sdk-platform-path'], text=True).strip() def GetSdkVersion(sdk): """Gets the selected SDK version.""" return subprocess.check_output( - ['xcrun', '--sdk', sdk, '--show-sdk-version']).strip() + ['xcrun', '--sdk', sdk, '--show-sdk-version'], text=True).strip() def GetXctestToolPath(sdk): @@ -89,7 +89,7 @@ def GetXctestToolPath(sdk): def GetDarwinUserCacheDir(): """Gets the path of Darwin user cache directory.""" - return subprocess.check_output(('getconf', 'DARWIN_USER_CACHE_DIR')).rstrip() + return subprocess.check_output(('getconf', 'DARWIN_USER_CACHE_DIR'), text=True).rstrip() def GetXcodeEmbeddedAppDeltasDir(): diff --git a/simulator_control/simulator_util.py b/simulator_control/simulator_util.py index cf1d59c..2317188 100644 --- a/simulator_control/simulator_util.py +++ b/simulator_control/simulator_util.py @@ -681,7 +681,7 @@ def RunSimctlCommand(command): """Runs simctl command.""" for i in range(_SIMCTL_MAX_ATTEMPTS): process = subprocess.Popen( - command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) stdout, stderr = process.communicate() if ios_constants.CORESIMULATOR_CHANGE_ERROR in stderr: output = stdout diff --git a/test_runner/ios_test_runner.py b/test_runner/ios_test_runner.py index a69f827..68104b3 100644 --- a/test_runner/ios_test_runner.py +++ b/test_runner/ios_test_runner.py @@ -288,7 +288,7 @@ def _GetSdk(device_id): return ios_constants.SDK.IPHONESIMULATOR known_devices_output = subprocess.check_output( - ['instruments', '-s', 'devices']) + ['instruments', '-s', 'devices'], text=True) for line in known_devices_output.split('\n'): if device_id in line and '(Simulator)' not in line: return ios_constants.SDK.IPHONEOS diff --git a/test_runner/xcodebuild_test_executor.py b/test_runner/xcodebuild_test_executor.py index c4e3800..cce5c94 100644 --- a/test_runner/xcodebuild_test_executor.py +++ b/test_runner/xcodebuild_test_executor.py @@ -161,11 +161,11 @@ def Execute(self, return_output=True): for i in range(max_attempts): process = subprocess.Popen( self._command, env=run_env, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, text=True) check_xcodebuild_stuck = CheckXcodebuildStuckThread( process, self._startup_timeout_sec) check_xcodebuild_stuck.start() - output = io.BytesIO() + output = io.StringIO() for stdout_line in iter(process.stdout.readline, ''): if not test_started: # Terminates the CheckXcodebuildStuckThread when test has started @@ -346,4 +346,4 @@ def _FetchTestCacheFileDirs(xcodebuild_test_output, max_dir_num=1): def _ReadFileTailInShell(file_path, line): """Tails the file in the last several lines.""" - return subprocess.check_output(['tail', '-%d' % line, file_path]) + return subprocess.check_output(['tail', '-%d' % line, file_path], text=True) diff --git a/test_runner/xcresult_util.py b/test_runner/xcresult_util.py index 599de4e..2ca2e12 100644 --- a/test_runner/xcresult_util.py +++ b/test_runner/xcresult_util.py @@ -108,7 +108,7 @@ def _GetResultBundleObject(xcresult_path, bundle_id=None): ] if bundle_id: command.extend(['--id', bundle_id]) - return json.loads(subprocess.check_output(command)) + return json.loads(subprocess.check_output(command, text=True)) def _GetFailureTestRefs(test_summary): diff --git a/test_runner/xctest_session.py b/test_runner/xctest_session.py index 9e6d694..f2a9f41 100644 --- a/test_runner/xctest_session.py +++ b/test_runner/xctest_session.py @@ -376,7 +376,7 @@ def _DetectTestType(test_bundle_dir): """Detects if the test bundle is XCUITest or XCTest.""" test_bundle_exec_path = os.path.join( test_bundle_dir, os.path.splitext(os.path.basename(test_bundle_dir))[0]) - output = subprocess.check_output(['nm', test_bundle_exec_path]) + output = subprocess.check_output(['nm', test_bundle_exec_path], text=True) if 'XCUIApplication' in output: return ios_constants.TestType.XCUITEST else: diff --git a/test_runner/xctestrun.py b/test_runner/xctestrun.py index 1aecad5..a82fd5e 100644 --- a/test_runner/xctestrun.py +++ b/test_runner/xctestrun.py @@ -55,8 +55,8 @@ def __init__(self, xctestrun_file_path, test_type=None, aut_bundle_id=None): self._xctestrun_file_path = xctestrun_file_path self._xctestrun_file_plist_obj = plist_util.Plist(xctestrun_file_path) # xctestrun file always has only key at root dict. - self._root_key = self._xctestrun_file_plist_obj.GetPlistField( - None).keys()[0] + self._root_key = next(iter(self._xctestrun_file_plist_obj.GetPlistField( + None))) self._test_type = test_type self._aut_bundle_id = aut_bundle_id From 659a147aaf0927d3b9490f18e7bf6d272ef12e6a Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Tue, 8 Sep 2020 22:58:35 -0700 Subject: [PATCH 19/45] Add a :// for_bazel_tests target for rules_apple integration tests --- BUILD | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/BUILD b/BUILD index aed3d72..8cbc406 100644 --- a/BUILD +++ b/BUILD @@ -33,3 +33,14 @@ par_binary( ":simulator", ], ) + +# Consumed by bazel tests. +filegroup( + name = "for_bazel_tests", + testonly = 1, + srcs = glob(["**/*"]), + # Exposed publicly just so other rules can use this if they set up + # integration tests that need to copy all the support files into + # a temporary workspace for the tests. + visibility = ["//visibility:public"], +) From 6a86a20bda6fd93f97e5aff7cdd89518eeabb26f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thi=20Do=C3=A3n?= Date: Wed, 28 Apr 2021 14:26:59 +0900 Subject: [PATCH 20/45] Fix compatibility with Python 3.9 This fixes error like the following when running xctestrunner using Python 3.9: ``` AttributeError: module 'plistlib' has no attribute 'readPlist' ``` `plistlib.readPlist` and `plistlib.writePlist` are no longer available since Python 3.9, and their replacements are available since Python 3.4. This change replaces them with wrapper functions to make xctestrunner compatible with a multiple Python versions. --- shared/plist_util.py | 49 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/shared/plist_util.py b/shared/plist_util.py index d431305..de5b9ef 100644 --- a/shared/plist_util.py +++ b/shared/plist_util.py @@ -61,7 +61,7 @@ def GetPlistField(self, field): """ if self._plistlib_module is None: return _GetPlistFieldByPlistBuddy(self._plist_file_path, field) - plist_root_object = self._plistlib_module.readPlist(self._plist_file_path) + plist_root_object = _ReadPlistByPlistLib(self._plist_file_path) return _GetObjectWithField(plist_root_object, field) def HasPlistField(self, field): @@ -102,11 +102,11 @@ def SetPlistField(self, field, value): _SetPlistFieldByPlistBuddy(self._plist_file_path, field, value) return if not field: - self._plistlib_module.writePlist(value, self._plist_file_path) + _WritePlistByPlistLib(value, self._plist_file_path) return if os.path.exists(self._plist_file_path): - plist_root_object = self._plistlib_module.readPlist(self._plist_file_path) + plist_root_object = _ReadPlistByPlistLib(self._plist_file_path) else: plist_root_object = {} keys_in_field = field.rsplit(':', 1) @@ -123,7 +123,7 @@ def SetPlistField(self, field, value): except (KeyError, IndexError): raise ios_errors.PlistError('Failed to set key %s from object %s.' % (key, target_object)) - self._plistlib_module.writePlist(plist_root_object, self._plist_file_path) + _WritePlistByPlistLib(plist_root_object, self._plist_file_path) def DeletePlistField(self, field): """Delete field in .plist file. @@ -142,7 +142,7 @@ def DeletePlistField(self, field): _DeletePlistFieldByPlistBuddy(self._plist_file_path, field) return - plist_root_object = self._plistlib_module.readPlist(self._plist_file_path) + plist_root_object = _ReadPlistByPlistLib(self._plist_file_path) keys_in_field = field.rsplit(':', 1) if len(keys_in_field) == 1: key = field @@ -159,7 +159,7 @@ def DeletePlistField(self, field): raise ios_errors.PlistError('Failed to delete key %s from object %s.' % (key, target_object)) - self._plistlib_module.writePlist(plist_root_object, self._plist_file_path) + _WritePlistByPlistLib(plist_root_object, self._plist_file_path) def _GetObjectWithField(target_object, field): @@ -241,7 +241,7 @@ def _GetPlistLibModule(plist_file_path): if not os.path.exists(plist_file_path): return plistlib try: - plistlib.readPlist(plist_file_path) + _ReadPlistByPlistLib(plist_file_path) return plistlib except xml.parsers.expat.ExpatError: if biplist is None: @@ -252,6 +252,41 @@ def _GetPlistLibModule(plist_file_path): return biplist +def _ReadPlistByPlistLib(plist_path): + """Wrapper function to read .plist file that is compatible with a wide + variety of Python versions. + + Args: + plist_path: string, full path of the .plist file. + + Returns: + The unpacked root object. + """ + # `plistlib.load` is only available since Python 3.4. + if hasattr(plistlib, "load"): + return plistlib.load(open(plist_path, 'rb')) + # `plistlib.readPlist` was deprecated since Python 3.4 and was deleted since + # Python 3.9. + return plistlib.readPlist(plist_path) + + +def _WritePlistByPlistLib(data, plist_path): + """Wrapper function to write .plist file that is compatible with a wide + variety of Python versions. + + Args: + plist_path: string, full path of the .plist file. + """ + + # `plistlib.dump` is only available since Python 3.4. + if hasattr(plistlib, "dump"): + plistlib.dump(data, open(plist_path, 'wb')) + else: + # `plistlib.writePlist` was deprecated since Python 3.4 and was deleted + # since Python 3.9. + plistlib.writePlist(data, plist_path) + + def _GetPlistFieldByPlistBuddy(plist_path, field): """View specific field in the .plist file by PlistBuddy tool. From 6a3b6f91d0dc17d77b0d49d3494bd4f1ccf9b66b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thi=20Do=C3=A3n?= Date: Wed, 5 May 2021 09:31:21 +0900 Subject: [PATCH 21/45] Add missing arg docs --- shared/plist_util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shared/plist_util.py b/shared/plist_util.py index de5b9ef..d84d81a 100644 --- a/shared/plist_util.py +++ b/shared/plist_util.py @@ -275,6 +275,8 @@ def _WritePlistByPlistLib(data, plist_path): variety of Python versions. Args: + data: an object, the value of the field to be added. It can be integer, + bool, string, array, dict. plist_path: string, full path of the .plist file. """ From 33bb70182159f9a2799dd22bf81312e34eae85d0 Mon Sep 17 00:00:00 2001 From: Jonathan Schear Date: Mon, 19 Apr 2021 17:55:33 -0400 Subject: [PATCH 22/45] Remove glob that doesn't match any files. --- BUILD | 2 -- 1 file changed, 2 deletions(-) diff --git a/BUILD b/BUILD index 8cbc406..f07ed62 100644 --- a/BUILD +++ b/BUILD @@ -19,13 +19,11 @@ par_binary( name = "ios_test_runner", srcs = ["__init__.py"] + glob( ["test_runner/*.py"], - exclude = ["test_runner/TestProject/**"], ), compiler_args = [ "--interpreter", "/usr/bin/python3", ], - data = glob(["test_runner/TestProject/**"]), main = "test_runner/ios_test_runner.py", python_version = "PY3", deps = [ From 88ff3408385331a6070dc84268ffe10f226e3fb5 Mon Sep 17 00:00:00 2001 From: Trinh Ngoc Thuyen Date: Tue, 1 Jun 2021 22:29:28 +0800 Subject: [PATCH 23/45] Fix: selected tests are not picked up --- test_runner/xctestrun.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test_runner/xctestrun.py b/test_runner/xctestrun.py index a82fd5e..5e03f3d 100644 --- a/test_runner/xctestrun.py +++ b/test_runner/xctestrun.py @@ -501,6 +501,7 @@ def _GenerateTestRootForXcuitest(self): 'DYLD_LIBRARY_PATH': '__TESTROOT__:%s/usr/lib' % developer_path } self._xctestrun_dict = { + 'ProductModuleName': self._test_name, 'IsUITestBundle': True, 'SystemAttachmentLifetime': 'keepNever', 'TestBundlePath': self._test_bundle_dir, @@ -658,6 +659,7 @@ def _GenerateTestRootForXctest(self): 'DYLD_LIBRARY_PATH': '__TESTROOT__:%s/usr/lib:' % developer_path } self._xctestrun_dict = { + 'ProductModuleName': self._test_name, 'TestHostPath': self._app_under_test_dir, 'TestBundlePath': self._test_bundle_dir, 'IsAppHostedTestBundle': True, @@ -678,6 +680,7 @@ def _GenerateTestRootForLogicTest(self): 'DYLD_LIBRARY_PATH': dyld_framework_path } self._xctestrun_dict = { + 'ProductModuleName': self._test_name, 'TestBundlePath': self._test_bundle_dir, 'TestHostPath': xcode_info_util.GetXctestToolPath(self._sdk), 'TestingEnvironmentVariables': test_envs, From 2451025e4ab9f65a73e184121aa324d74354d129 Mon Sep 17 00:00:00 2001 From: Bogo Giertler Date: Sat, 9 Oct 2021 12:31:26 -0700 Subject: [PATCH 24/45] Fix typos --- test_runner/xcresult_util.py | 2 +- test_runner/xctest_session.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test_runner/xcresult_util.py b/test_runner/xcresult_util.py index 2ca2e12..840abdf 100644 --- a/test_runner/xcresult_util.py +++ b/test_runner/xcresult_util.py @@ -21,7 +21,7 @@ from xctestrunner.shared import ios_errors -def ExpoesXcresult(xcresult_path, output_path): +def ExposeXcresult(xcresult_path, output_path): """Exposes the files from xcresult. The files includes the diagnostics files and attachments files. diff --git a/test_runner/xctest_session.py b/test_runner/xctest_session.py index f2a9f41..d78026a 100644 --- a/test_runner/xctest_session.py +++ b/test_runner/xctest_session.py @@ -217,7 +217,7 @@ def RunTest(self, device_id, os_version=None): if xcode_info_util.GetXcodeVersionNumber() >= 1100: expose_xcresult = os.path.join(self._output_dir, 'ExposeXcresult') try: - xcresult_util.ExpoesXcresult(result_bundle_path, expose_xcresult) + xcresult_util.ExposeXcresult(result_bundle_path, expose_xcresult) if not self._keep_xcresult_data: shutil.rmtree(result_bundle_path) except subprocess.CalledProcessError as e: From 708ace6efbfc27f91637c067a0e9adc06403dbef Mon Sep 17 00:00:00 2001 From: Bogo Giertler Date: Sat, 9 Oct 2021 15:44:47 -0700 Subject: [PATCH 25/45] Skip results with missing activitySummaries field. --- test_runner/xcresult_util.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test_runner/xcresult_util.py b/test_runner/xcresult_util.py index 840abdf..a241cb7 100644 --- a/test_runner/xcresult_util.py +++ b/test_runner/xcresult_util.py @@ -72,6 +72,11 @@ def _ExposeAttachments(xcresult_path, output_path, action_result): failure_test_ref_ids = _GetFailureTestRefs(root_tests_summary) for test_ref_id in failure_test_ref_ids: test_summary_result = _GetResultBundleObject(xcresult_path, test_ref_id) + # if the test results in an `expectedFailures` entry, there might be an + # `activitySummaries` field present. + if 'activitySummaries' not in test_summary_result: + continue + activity_summaries = test_summary_result['activitySummaries']['_values'] for activity_summary in activity_summaries: if 'attachments' in activity_summary: From 7f8fc81b10c8d93f09f6fe38b2a3f37ba25336a6 Mon Sep 17 00:00:00 2001 From: Weiming Dai Date: Thu, 28 Oct 2021 14:52:58 -0700 Subject: [PATCH 26/45] Incremental changes for test runner Major changes 1. Support Xcode 13.0/13.1. 2. Change the parameter text in subprocess to decode/encoding to be compatible with Python 3.6. 3. Support running x86_64 UITest on arm simulator. 4. Improve the stability of booting simulator. 5. Clean up 1). Remove the Xcode 9 or earlier Xcode support. 2). Remove the usage of '/usr/libexec/PlistBuddy' and `biplist` module since we can get full plist support from plistlib in Python3. --- README.md | 16 +-- shared/__init__.py | 13 ++ shared/bundle_util.py | 14 +- shared/ios_errors.py | 1 + shared/plist_util.py | 175 ++---------------------- shared/provisioning_profile.py | 2 +- shared/xcode_info_util.py | 48 +++---- simulator_control/__init__.py | 13 ++ simulator_control/simtype_profile.py | 8 +- simulator_control/simulator_util.py | 84 ++++++++---- test_runner/__init__.py | 13 ++ test_runner/ios_test_runner.py | 145 ++++++++++---------- test_runner/logic_test_util.py | 6 - test_runner/xcodebuild_test_executor.py | 36 +++-- test_runner/xcresult_util.py | 2 +- test_runner/xctest_session.py | 6 +- test_runner/xctestrun.py | 83 ++++++----- 17 files changed, 295 insertions(+), 370 deletions(-) diff --git a/README.md b/README.md index 596ac47..15551dc 100644 --- a/README.md +++ b/README.md @@ -3,15 +3,14 @@ A tool for running prebuilt iOS tests on iOS real device and simulator. ## Features - It supports XCTest (Xcode Unit Test), XCUITest (Xcode UI Test). -- It supports iOS 7+ iOS real device, iOS simulator. +- It supports iOS 11+ iOS real device, iOS simulator. - It supports launch options configuration: test methods to run, additional environment variables, additional arguments. -- It supports Xcode 8+. +- It supports Xcode 10+. ## Prerequisites -- Install Xcode (Xcode 8+). XCUITest support requires Xcode 8+. +- Install Xcode (Xcode 10+). - [Install bazel](https://docs.bazel.build/install.html) (optional). -- py module [biplist](https://github.com/wooster/biplist). ## Installation You can download the ios_test_runner.par binary in [release](https://github.com/google/xctestrunner/releases) @@ -43,12 +42,3 @@ Disclaimer: This is not an official Google product. XCTestRunner uses Apple native tool `xcodebuild`, `simctl` to control iOS Simulator and launch tests on iOS devices. - -For testing, XCTestRunner injects app under test and test bundle file into a -dummy project. Then the dummy project can be used `xcodebuild test` to run -XCTest (not for XCUITest), or `xcodebuild build-for-testing` to generate -xctestrun file for further testing. - -For iOS 7 real device testing, the latest supported Xcode version is 7.2.1. -For iOS 7 simulator testing, latest supported Xcode version is 7.2.1 and latest -supported MacOS version is Yosemite (10.10.x). diff --git a/shared/__init__.py b/shared/__init__.py index e69de29..74a2d1e 100644 --- a/shared/__init__.py +++ b/shared/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/shared/bundle_util.py b/shared/bundle_util.py index 2c09eca..fc7f8b8 100644 --- a/shared/bundle_util.py +++ b/shared/bundle_util.py @@ -126,8 +126,8 @@ def GetCodesignIdentity(bundle_path): """ command = ('codesign', '-dvv', bundle_path) process = subprocess.Popen(command, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, text=True) - output = process.communicate()[0] + stderr=subprocess.STDOUT) + output = process.communicate()[0].decode('utf-8') for line in output.split('\n'): if line.startswith('Authority='): return line[len('Authority='):] @@ -151,8 +151,8 @@ def GetDevelopmentTeam(bundle_path): """ command = ('codesign', '-dvv', bundle_path) process = subprocess.Popen(command, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, text=True) - output = process.communicate()[0] + stderr=subprocess.STDOUT) + output = process.communicate()[0].decode('utf-8') for line in output.split('\n'): if line.startswith('TeamIdentifier='): return line[len('TeamIdentifier='):] @@ -195,7 +195,8 @@ def CodesignBundle(bundle_path, stderr=subprocess.PIPE) except subprocess.CalledProcessError as e: raise ios_errors.BundleError( - 'Failed to codesign the bundle %s: %s' % (bundle_path, e.output)) + 'Failed to codesign the bundle %s with %s: %s' % + (bundle_path, identity, e.output)) def EnableUIFileSharing(bundle_path, resigning=True): @@ -217,7 +218,8 @@ def EnableUIFileSharing(bundle_path, resigning=True): def GetFileArchTypes(file_path): """Gets the architecture types of the file.""" - output = subprocess.check_output(['/usr/bin/lipo', file_path, '-archs'], text=True) + output = subprocess.check_output(['/usr/bin/lipo', file_path, + '-archs']).decode('utf-8').strip() return output.split(' ') diff --git a/shared/ios_errors.py b/shared/ios_errors.py index cdddc48..b70e077 100644 --- a/shared/ios_errors.py +++ b/shared/ios_errors.py @@ -49,3 +49,4 @@ class XcodebuildTestError(Exception): class XcresultError(Exception): """Exception class for parsing xcresult error.""" + diff --git a/shared/plist_util.py b/shared/plist_util.py index d84d81a..d4faab6 100644 --- a/shared/plist_util.py +++ b/shared/plist_util.py @@ -14,20 +14,10 @@ """Utility class for managing Plist files.""" -import logging import os import plistlib -import subprocess -import xml.parsers.expat from xctestrunner.shared import ios_errors -try: - import biplist -except ImportError: - biplist = None - - -PLIST_BUDDY = '/usr/libexec/PlistBuddy' class Plist(object): @@ -41,7 +31,6 @@ def __init__(self, plist_file_path): """ self._plist_file_path = plist_file_path # Module to read the .plist file. - self._plistlib_module = _GetPlistLibModule(plist_file_path) def GetPlistField(self, field): """View specific field in the .plist file. @@ -59,9 +48,8 @@ def GetPlistField(self, field): Raises: ios_errors.PlistError: the field does not exist in the plist dict. """ - if self._plistlib_module is None: - return _GetPlistFieldByPlistBuddy(self._plist_file_path, field) - plist_root_object = _ReadPlistByPlistLib(self._plist_file_path) + with open(self._plist_file_path, 'rb') as plist_file: + plist_root_object = plistlib.load(plist_file) return _GetObjectWithField(plist_root_object, field) def HasPlistField(self, field): @@ -98,15 +86,14 @@ def SetPlistField(self, field, value): Raises: ios_errors.PlistError: the field does not exist in the .plist file's dict. """ - if self._plistlib_module is None: - _SetPlistFieldByPlistBuddy(self._plist_file_path, field, value) - return if not field: - _WritePlistByPlistLib(value, self._plist_file_path) + with open(self._plist_file_path, 'wb') as plist_file: + plistlib.dump(value, plist_file) return if os.path.exists(self._plist_file_path): - plist_root_object = _ReadPlistByPlistLib(self._plist_file_path) + with open(self._plist_file_path, 'rb') as plist_file: + plist_root_object = plistlib.load(plist_file) else: plist_root_object = {} keys_in_field = field.rsplit(':', 1) @@ -123,7 +110,8 @@ def SetPlistField(self, field, value): except (KeyError, IndexError): raise ios_errors.PlistError('Failed to set key %s from object %s.' % (key, target_object)) - _WritePlistByPlistLib(plist_root_object, self._plist_file_path) + with open(self._plist_file_path, 'wb') as plist_file: + plistlib.dump(plist_root_object, plist_file) def DeletePlistField(self, field): """Delete field in .plist file. @@ -138,11 +126,8 @@ def DeletePlistField(self, field): Raises: ios_errors.PlistError: the field does not exist in the .plist file's dict. """ - if self._plistlib_module is None: - _DeletePlistFieldByPlistBuddy(self._plist_file_path, field) - return - - plist_root_object = _ReadPlistByPlistLib(self._plist_file_path) + with open(self._plist_file_path, 'rb') as plist_file: + plist_root_object = plistlib.load(plist_file) keys_in_field = field.rsplit(':', 1) if len(keys_in_field) == 1: key = field @@ -159,7 +144,8 @@ def DeletePlistField(self, field): raise ios_errors.PlistError('Failed to delete key %s from object %s.' % (key, target_object)) - _WritePlistByPlistLib(plist_root_object, self._plist_file_path) + with open(self._plist_file_path, 'wb') as plist_file: + plistlib.dump(plist_root_object, plist_file) def _GetObjectWithField(target_object, field): @@ -221,140 +207,3 @@ def _ParseKey(target_object, key): % (key, target_object)) raise ios_errors.PlistError('The object %s is not dict or list.' % target_object) - - -def _GetPlistLibModule(plist_file_path): - """Gets the module to read the target .plist file. - - .plist file has two kinds of format: XML format and binary format. If the - .plist file is XML format, it should be read by plistlib and don't use - biplist which will change XML format plist to binary plist. - - Args: - plist_file_path: string, full path of the .plist file. - - Returns: - a module to read the target .plist file or None if the model can not be - imported. - """ - # If the plist file path does not exist, use plistlib by default. - if not os.path.exists(plist_file_path): - return plistlib - try: - _ReadPlistByPlistLib(plist_file_path) - return plistlib - except xml.parsers.expat.ExpatError: - if biplist is None: - logging.info( - 'Failed to import biplist module. Will use tool %s to handle the ' - 'binary format plist.', - PLIST_BUDDY) - return biplist - - -def _ReadPlistByPlistLib(plist_path): - """Wrapper function to read .plist file that is compatible with a wide - variety of Python versions. - - Args: - plist_path: string, full path of the .plist file. - - Returns: - The unpacked root object. - """ - # `plistlib.load` is only available since Python 3.4. - if hasattr(plistlib, "load"): - return plistlib.load(open(plist_path, 'rb')) - # `plistlib.readPlist` was deprecated since Python 3.4 and was deleted since - # Python 3.9. - return plistlib.readPlist(plist_path) - - -def _WritePlistByPlistLib(data, plist_path): - """Wrapper function to write .plist file that is compatible with a wide - variety of Python versions. - - Args: - data: an object, the value of the field to be added. It can be integer, - bool, string, array, dict. - plist_path: string, full path of the .plist file. - """ - - # `plistlib.dump` is only available since Python 3.4. - if hasattr(plistlib, "dump"): - plistlib.dump(data, open(plist_path, 'wb')) - else: - # `plistlib.writePlist` was deprecated since Python 3.4 and was deleted - # since Python 3.9. - plistlib.writePlist(data, plist_path) - - -def _GetPlistFieldByPlistBuddy(plist_path, field): - """View specific field in the .plist file by PlistBuddy tool. - - Args: - plist_path: string, the path of plist file. - field: string, the field consist of property key names delimited by - colons. List(array) items are specified by a zero-based integer index. - Examples - :CFBundleShortVersionString - :CFBundleDocumentTypes:2:CFBundleTypeExtensions - - Returns: - the object of the plist's field. - - Raises: - ios_errors.PlistError: the field does not exist in the plist dict. - """ - command = [PLIST_BUDDY, '-c', 'Print :"%s"' % field, plist_path] - try: - return subprocess.check_output(command, stderr=subprocess.STDOUT, text=True).strip() - except subprocess.CalledProcessError as e: - raise ios_errors.PlistError( - 'Failed to get field %s in plist %s: %s', field, plist_path, e.output) - - -def _SetPlistFieldByPlistBuddy(plist_path, field, value): - """Set field with provided value in .plist file. - - Args: - plist_path: string, the path of plist file. - field: string, the field consist of property key names delimited by - colons. List(array) items are specified by a zero-based integer index. - Examples - :CFBundleShortVersionString - :CFBundleDocumentTypes:2:CFBundleTypeExtensions - value: a object, the value of the field to be added. It can be integer, - bool, string, array, dict. - - Raises: - ios_errors.PlistError: the field does not exist in the .plist file's dict. - """ - command = [PLIST_BUDDY, '-c', 'Set :"%s" "%s"' % (field, value), plist_path] - try: - subprocess.check_output(command, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - raise ios_errors.PlistError('Failed to set field %s in plist %s: %s' - % (field, plist_path, e.output)) - - -def _DeletePlistFieldByPlistBuddy(plist_path, field): - """Delete field in .plist file. - - Args: - plist_path: string, the path of plist file. - field: string, the field consist of property key names delimited by - colons. List(array) items are specified by a zero-based integer index. - Examples - :CFBundleShortVersionString - :CFBundleDocumentTypes:2:CFBundleTypeExtensions - - Raises: - ios_errors.PlistError: the field does not exist in the .plist file's dict. - """ - command = [PLIST_BUDDY, '-c', 'Delete :"%s"' % field, plist_path] - try: - subprocess.check_output(command, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - raise ios_errors.PlistError('Failed to delete field %s in plist %s: %s' - % (field, plist_path, e.output)) diff --git a/shared/provisioning_profile.py b/shared/provisioning_profile.py index bddb9a4..82062b4 100644 --- a/shared/provisioning_profile.py +++ b/shared/provisioning_profile.py @@ -88,7 +88,7 @@ def _DecodeProvisioningProfile(self): command.extend(['-k', self._keychain_path]) process = subprocess.Popen( command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - output = process.communicate() + output = process.communicate()[0].decode('utf-8') if process.poll() != 0: raise ios_errors.ProvisioningProfileError(output) if not os.path.exists(decode_provisioning_profile): diff --git a/shared/xcode_info_util.py b/shared/xcode_info_util.py index 7ba0989..2333e7a 100644 --- a/shared/xcode_info_util.py +++ b/shared/xcode_info_util.py @@ -26,24 +26,7 @@ def GetXcodeDeveloperPath(): """Gets the active developer path of Xcode command line tools.""" - return subprocess.check_output(('xcode-select', '-p'), text=True).strip() - - -# Xcode 11+'s Swift dylibs are configured in a way that does not allow them to -# load the correct libswiftFoundation.dylib file from -# libXCTestSwiftSupport.dylib. This bug only affects tests that run on fallbacks -# to the correct Swift dylibs that have been packaged with Xcode. This method -# returns the path to that fallback directory. -# See https://github.com/bazelbuild/rules_apple/issues/684 for context. -def GetSwift5FallbackLibsDir(): - """Gets the Swift5 fallback libraries directory.""" - relative_path = 'Toolchains/XcodeDefault.xctoolchain/usr/lib/swift-5.0' - swift_libs_dir = os.path.join(GetXcodeDeveloperPath(), relative_path) - swift_lib_platform_dir = os.path.join(swift_libs_dir, - ios_constants.SDK.IPHONESIMULATOR) - if os.path.exists(swift_lib_platform_dir): - return swift_lib_platform_dir - return None + return subprocess.check_output(('xcode-select', '-p')).decode('utf-8').strip() def GetXcodeVersionNumber(): @@ -61,7 +44,7 @@ def GetXcodeVersionNumber(): # Example output: # Xcode 8.2.1 # Build version 8C1002 - output = subprocess.check_output(('xcodebuild', '-version'), text=True) + output = subprocess.check_output(('xcodebuild', '-version')).decode('utf-8') xcode_version = output.split('\n')[0].split(' ')[1] # Add cache xcode_version_number to avoid calling subprocess multiple times. # It is expected that no one changes xcode during the test runner working. @@ -69,16 +52,34 @@ def GetXcodeVersionNumber(): return _xcode_version_number +# Xcode 11+'s Swift dylibs are configured in a way that does not allow them to +# load the correct libswiftFoundation.dylib file from +# libXCTestSwiftSupport.dylib. This bug only affects tests that run on fallbacks +# to the correct Swift dylibs that have been packaged with Xcode. This method +# returns the path to that fallback directory. +# See https://github.com/bazelbuild/rules_apple/issues/684 for context. +def GetSwift5FallbackLibsDir(): + """Gets the Swift5 fallback libraries directory.""" + relative_path = 'Toolchains/XcodeDefault.xctoolchain/usr/lib/swift-5.0' + swift_libs_dir = os.path.join(GetXcodeDeveloperPath(), relative_path) + swift_lib_platform_dir = os.path.join(swift_libs_dir, + ios_constants.SDK.IPHONESIMULATOR) + if os.path.exists(swift_lib_platform_dir): + return swift_lib_platform_dir + return None + + def GetSdkPlatformPath(sdk): """Gets the selected SDK platform path.""" return subprocess.check_output( - ['xcrun', '--sdk', sdk, '--show-sdk-platform-path'], text=True).strip() + ['xcrun', '--sdk', sdk, + '--show-sdk-platform-path']).decode('utf-8').strip() def GetSdkVersion(sdk): """Gets the selected SDK version.""" - return subprocess.check_output( - ['xcrun', '--sdk', sdk, '--show-sdk-version'], text=True).strip() + return subprocess.check_output(['xcrun', '--sdk', sdk, '--show-sdk-version' + ]).decode('utf-8').strip() def GetXctestToolPath(sdk): @@ -89,7 +90,8 @@ def GetXctestToolPath(sdk): def GetDarwinUserCacheDir(): """Gets the path of Darwin user cache directory.""" - return subprocess.check_output(('getconf', 'DARWIN_USER_CACHE_DIR'), text=True).rstrip() + return subprocess.check_output( + ('getconf', 'DARWIN_USER_CACHE_DIR')).decode('utf-8').rstrip() def GetXcodeEmbeddedAppDeltasDir(): diff --git a/simulator_control/__init__.py b/simulator_control/__init__.py index e69de29..74a2d1e 100644 --- a/simulator_control/__init__.py +++ b/simulator_control/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/simulator_control/simtype_profile.py b/simulator_control/simtype_profile.py index dc4ba21..df6673b 100644 --- a/simulator_control/simtype_profile.py +++ b/simulator_control/simtype_profile.py @@ -48,12 +48,8 @@ def profile_plist_obj(self): """ if not self._profile_plist_obj: xcode_version = xcode_info_util.GetXcodeVersionNumber() - if xcode_version >= 900: - platform_path = xcode_info_util.GetSdkPlatformPath( - ios_constants.SDK.IPHONEOS) - else: - platform_path = xcode_info_util.GetSdkPlatformPath( - ios_constants.SDK.IPHONESIMULATOR) + platform_path = xcode_info_util.GetSdkPlatformPath( + ios_constants.SDK.IPHONEOS) if xcode_version >= 1100: sim_profiles_dir = os.path.join( platform_path, 'Library/Developer/CoreSimulator/Profiles') diff --git a/simulator_control/simulator_util.py b/simulator_control/simulator_util.py index 2317188..0f186fe 100644 --- a/simulator_control/simulator_util.py +++ b/simulator_control/simulator_util.py @@ -37,6 +37,7 @@ _SIM_OPERATION_MAX_ATTEMPTS = 3 _SIMCTL_MAX_ATTEMPTS = 2 _SIMULATOR_CREATING_TO_SHUTDOWN_TIMEOUT_SEC = 10 +_SIMULATOR_BOOTED_TIMEOUT_SEC = 10 _SIMULATOR_SHUTDOWN_TIMEOUT_SEC = 30 _SIM_ERROR_RETRY_INTERVAL_SEC = 2 _SIM_CHECK_STATE_INTERVAL_SEC = 0.5 @@ -112,6 +113,28 @@ def device_plist_object(self): self._device_plist_object = plist_util.Plist(device_plist_path) return self._device_plist_object + def Boot(self): + """Boots the simulator as asynchronously. + + Returns: + A subprocess.Popen object of the boot process. + """ + RunSimctlCommand(['xcrun', 'simctl', 'boot', self.simulator_id]) + self.WaitUntilStateBooted() + logging.info('The simulator %s is booted.', self.simulator_id) + + def BootStatus(self): + """Monitor the simulator boot status asynchronously. + + Returns: + A subprocess.Popen object of the boot status process. + """ + return subprocess.Popen( + ['xcrun', 'simctl', 'bootstatus', self.simulator_id, '-b'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding='utf-8') + def Shutdown(self): """Shuts down the simulator.""" sim_state = self.GetSimulatorState() @@ -144,14 +167,6 @@ def Delete(self, asynchronously=True): Raises: ios_errors.SimError: The simulator's state is not SHUTDOWN. """ - # In Xcode 9+, simctl can delete Booted simulator. In prior of Xcode 9, - # we have to shutdown the simulator first before deleting it. - if xcode_info_util.GetXcodeVersionNumber() < 900: - sim_state = self.GetSimulatorState() - if sim_state != ios_constants.SimState.SHUTDOWN: - raise ios_errors.SimError( - 'Can only delete the simulator with state SHUTDOWN. The current ' - 'state of simulator %s is %s.' % (self._simulator_id, sim_state)) command = ['xcrun', 'simctl', 'delete', self.simulator_id] if asynchronously: logging.info('Deleting simulator %s asynchronously.', self.simulator_id) @@ -197,17 +212,16 @@ def FetchLogToFile(self, output_file_path, start_time=None, end_time=None): def GetAppDocumentsPath(self, app_bundle_id): """Gets the path of the app's Documents directory.""" - if xcode_info_util.GetXcodeVersionNumber() >= 830: - try: - app_data_container = RunSimctlCommand([ - 'xcrun', 'simctl', 'get_app_container', self._simulator_id, - app_bundle_id, 'data' - ]) - return os.path.join(app_data_container, 'Documents') - except ios_errors.SimError as e: - raise ios_errors.SimError( - 'Failed to get data container of the app %s in simulator %s: %s' % - (app_bundle_id, self._simulator_id, str(e))) + try: + app_data_container = RunSimctlCommand([ + 'xcrun', 'simctl', 'get_app_container', self._simulator_id, + app_bundle_id, 'data' + ]) + return os.path.join(app_data_container, 'Documents') + except ios_errors.SimError as e: + raise ios_errors.SimError( + 'Failed to get data container of the app %s in simulator %s: %s' % + (app_bundle_id, self._simulator_id, str(e))) apps_dir = os.path.join(self.simulator_root_dir, 'data/Containers/Data/Application') @@ -234,6 +248,25 @@ def IsAppInstalled(self, app_bundle_id): except ios_errors.SimError: return False + def WaitUntilStateBooted(self, timeout_sec=_SIMULATOR_BOOTED_TIMEOUT_SEC): + """Waits until the simulator state becomes BOOTED. + + Args: + timeout_sec: int, timeout of waiting simulator state for becoming BOOTED + in seconds. + + Raises: + ios_errors.SimError: when it is timeout to wait the simulator state + becomes BOOTED. + """ + start_time = time.time() + while start_time + timeout_sec >= time.time(): + time.sleep(_SIM_CHECK_STATE_INTERVAL_SEC) + if self.GetSimulatorState() == ios_constants.SimState.BOOTED: + return + raise ios_errors.SimError('Timeout to wait for simulator booted in %ss.' % + timeout_sec) + def WaitUntilStateShutdown(self, timeout_sec=_SIMULATOR_SHUTDOWN_TIMEOUT_SEC): """Waits until the simulator state becomes SHUTDOWN. @@ -247,9 +280,9 @@ def WaitUntilStateShutdown(self, timeout_sec=_SIMULATOR_SHUTDOWN_TIMEOUT_SEC): """ start_time = time.time() while start_time + timeout_sec >= time.time(): + time.sleep(_SIM_CHECK_STATE_INTERVAL_SEC) if self.GetSimulatorState() == ios_constants.SimState.SHUTDOWN: return - time.sleep(_SIM_CHECK_STATE_INTERVAL_SEC) raise ios_errors.SimError('Timeout to wait for simulator shutdown in %ss.' % timeout_sec) @@ -625,11 +658,7 @@ def _ValidateSimulatorTypeWithOsVersion(device_type, os_version): def QuitSimulatorApp(): """Quits the Simulator.app.""" - if xcode_info_util.GetXcodeVersionNumber() >= 700: - simulator_name = 'Simulator' - else: - simulator_name = 'iOS Simulator' - subprocess.Popen(['killall', simulator_name], + subprocess.Popen(['killall', 'Simulator'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) @@ -681,7 +710,10 @@ def RunSimctlCommand(command): """Runs simctl command.""" for i in range(_SIMCTL_MAX_ATTEMPTS): process = subprocess.Popen( - command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding='utf-8') stdout, stderr = process.communicate() if ios_constants.CORESIMULATOR_CHANGE_ERROR in stderr: output = stdout diff --git a/test_runner/__init__.py b/test_runner/__init__.py index e69de29..74a2d1e 100644 --- a/test_runner/__init__.py +++ b/test_runner/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test_runner/ios_test_runner.py b/test_runner/ios_test_runner.py index 68104b3..c85800d 100644 --- a/test_runner/ios_test_runner.py +++ b/test_runner/ios_test_runner.py @@ -27,7 +27,6 @@ from xctestrunner.shared import ios_constants from xctestrunner.shared import ios_errors -from xctestrunner.shared import xcode_info_util from xctestrunner.simulator_control import simulator_util from xctestrunner.test_runner import runner_exit_codes from xctestrunner.test_runner import xctest_session @@ -102,6 +101,42 @@ def _AddGeneralArguments(parser): 'test ends.') +def _AddPrepareSubParser(subparsers): + """Adds sub parser for sub command `prepare`.""" + def _Prepare(args): + """The function of sub command `prepare`.""" + sdk = _PlatformToSdk(args.platform) if args.platform else _GetSdk(args.id) + device_arch = args.arch + with xctest_session.XctestSession( + sdk=sdk, + device_arch=device_arch, + work_dir=args.work_dir, + output_dir=args.output_dir) as session: + session.Prepare( + app_under_test=args.app_under_test_path, + test_bundle=args.test_bundle_path, + xctestrun_file_path=args.xctestrun, + test_type=args.test_type, + signing_options=_GetJson(args.signing_options_json_path)) + session.SetLaunchOptions(_GetJson(args.launch_options_json_path)) + + test_parser = subparsers.add_parser( + 'prepare', + help='Prepare the working directory to run the test.') + required_arguments = test_parser.add_argument_group('Required arguments') + required_arguments.add_argument( + '--platform', + help='The platform of the device. The value can be ios_device or ' + 'ios_simulator.' + ) + required_arguments.add_argument( + '--arch', + help='The architecture of the device. The value can be x86_64, armv7,' + 'arm64, arm64e' + ) + test_parser.set_defaults(func=_Prepare) + + def _AddTestSubParser(subparsers): """Adds sub parser for sub command `test`.""" def _Test(args): @@ -148,66 +183,32 @@ def _RunSimulatorTest(args): sdk=ios_constants.SDK.IPHONESIMULATOR, device_arch=ios_constants.ARCH.X86_64, work_dir=args.work_dir, output_dir=args.output_dir) as session: - session.Prepare( - app_under_test=args.app_under_test_path, - test_bundle=args.test_bundle_path, - xctestrun_file_path=args.xctestrun, - test_type=args.test_type, - signing_options=_GetJson(args.signing_options_json_path)) - session.SetLaunchOptions(_GetJson(args.launch_options_json_path)) - - # In prior of Xcode 9, `xcodebuild test` will launch the Simulator.app - # process. If there is Simulator.app before running test, it will cause - # error later. - if xcode_info_util.GetXcodeVersionNumber() < 900: - simulator_util.QuitSimulatorApp() - max_attempts = 3 - reboot_sim = False - for i in range(max_attempts): - if not reboot_sim: - simulator_id, _, _, _ = simulator_util.CreateNewSimulator( - device_type=args.device_type, os_version=args.os_version, - name_prefix=args.new_simulator_name_prefix) - reboot_sim = False - - try: - # Don't use command "{Xcode_developer_dir}Applications/ \ - # Simulator.app/Contents/MacOS/Simulator" to launch the Simulator.app. - # 1) `xcodebuild test` will handle the launch Simulator. - # 2) If there are two Simulator.app processes launched by command line - # and `xcodebuild test` starts to run on one of Simulator, the another - # Simulator.app will popup 'Unable to boot device in current state: \ - # Booted' dialog and may cause potential error. - exit_code = session.RunTest(simulator_id) - if i < max_attempts - 1: - if exit_code == runner_exit_codes.EXITCODE.NEED_RECREATE_SIM: - logging.warning( - 'Will create a new simulator to retry running test.') - continue - if exit_code == runner_exit_codes.EXITCODE.NEED_REBOOT_DEVICE: - reboot_sim = True - logging.warning( - 'Will reboot the simulator to retry running test.') - continue - return exit_code - finally: - # 1. In prior of Xcode 9, `xcodebuild test` will launch the - # Simulator.app process. Quit the Simulator.app to avoid side effect. - # 2. Quit Simulator.app can also shutdown the simulator. To make sure - # the Simulator state to be SHUTDOWN, still call shutdown command - # later. - if xcode_info_util.GetXcodeVersionNumber() < 900: - simulator_util.QuitSimulatorApp() - simulator_obj = simulator_util.Simulator(simulator_id) - if reboot_sim: - simulator_obj.Shutdown() - else: - # In Xcode 9+, simctl can delete the Booted simulator. - # In prior of Xcode 9, we have to shutdown the simulator first - # before deleting it. - if xcode_info_util.GetXcodeVersionNumber() < 900: - simulator_obj.Shutdown() - simulator_obj.Delete() + simulator_id, _, os_version, _ = simulator_util.CreateNewSimulator( + device_type=args.device_type, + os_version=args.os_version, + name_prefix=args.new_simulator_name_prefix) + simulator_obj = simulator_util.Simulator(simulator_id) + hostless = args.app_under_test_path is None + try: + if not hostless: + simulator_obj.Boot() + session.Prepare( + app_under_test=args.app_under_test_path, + test_bundle=args.test_bundle_path, + xctestrun_file_path=args.xctestrun, + test_type=args.test_type, + signing_options=_GetJson(args.signing_options_json_path)) + session.SetLaunchOptions(_GetJson(args.launch_options_json_path)) + if not hostless: + try: + simulator_obj.BootStatus().wait(timeout=60) + except subprocess.TimeoutExpired: + logging.warning( + 'The simulator %s could not be booted in 60s. Will try to run ' + 'test directly.', simulator_id) + return session.RunTest(simulator_id, os_version=os_version) + finally: + simulator_obj.Delete() def _SimulatorTest(args): """The function of sub command `simulator_test`.""" @@ -250,6 +251,7 @@ def _BuildParser(): formatter_class=argparse.RawTextHelpFormatter) _AddGeneralArguments(parser) subparsers = parser.add_subparsers(help='Sub-commands help') + _AddPrepareSubParser(subparsers) _AddTestSubParser(subparsers) _AddSimulatorTestSubParser(subparsers) return parser @@ -279,23 +281,16 @@ def _PlatformToSdk(platform): def _GetSdk(device_id): """Gets the sdk of the target device with the given device_id.""" - # The command `instruments -s devices` is much slower than - # `xcrun simctl list devices`. So use `xcrun simctl list devices` to check - # IPHONESIMULATOR SDK first. - simlist_devices_output = simulator_util.RunSimctlCommand( - ['xcrun', 'simctl', 'list', 'devices']) - if device_id in simlist_devices_output: - return ios_constants.SDK.IPHONESIMULATOR - - known_devices_output = subprocess.check_output( - ['instruments', '-s', 'devices'], text=True) - for line in known_devices_output.split('\n'): - if device_id in line and '(Simulator)' not in line: - return ios_constants.SDK.IPHONEOS + devices_list_output = subprocess.check_output( + ['xcrun', 'xcdevice', 'list']).decode('utf-8') + for device_info in json.loads(devices_list_output): + if device_info['identifier'] == device_id: + return ios_constants.SDK.IPHONESIMULATOR if device_info[ + 'simulator'] else ios_constants.SDK.IPHONEOS raise ios_errors.IllegalArgumentError( 'The device with id %s can not be found. The known devices are %s.' % - (device_id, known_devices_output)) + (device_id, devices_list_output)) def _GetDeviceArch(device_id, sdk): diff --git a/test_runner/logic_test_util.py b/test_runner/logic_test_util.py index c3bd792..8439e4b 100644 --- a/test_runner/logic_test_util.py +++ b/test_runner/logic_test_util.py @@ -14,7 +14,6 @@ """The helper classes to run logic test.""" -import os import subprocess import sys @@ -63,11 +62,6 @@ def RunLogicTestOnSim(sim_id, version_util.GetVersionNumber(os_version) < 1220): key = _SIMCTL_ENV_VAR_PREFIX + 'DYLD_FALLBACK_LIBRARY_PATH' simctl_env_vars[key] = xcode_info_util.GetSwift5FallbackLibsDir() - # We need to set the DEVELOPER_DIR to ensure xcrun works correctly - developer_dir = os.environ.get('DEVELOPER_DIR') - if developer_dir: - simctl_env_vars['DEVELOPER_DIR'] = developer_dir - command = [ 'xcrun', 'simctl', 'spawn', '-s', sim_id, xcode_info_util.GetXctestToolPath(ios_constants.SDK.IPHONESIMULATOR)] diff --git a/test_runner/xcodebuild_test_executor.py b/test_runner/xcodebuild_test_executor.py index cce5c94..56b5fcb 100644 --- a/test_runner/xcodebuild_test_executor.py +++ b/test_runner/xcodebuild_test_executor.py @@ -21,7 +21,6 @@ import re import shutil import subprocess -import sys import threading import time @@ -50,6 +49,9 @@ 'already running.') _LOST_CONNECTION_ERROR = 'Lost connection to testmanagerd' _LOST_CONNECTION_TO_DTSERVICEHUB_ERROR = 'Lost connection to DTServiceHub' +_DEVICE_NO_LONGER_CONNECTED = 'This device is no longer connected' +_UNABLE_FIND_DEVICE_IDENTIFIER = 'Unable to find device with identifier' +_BUNDLE_DAMAGED = 'The bundle is damaged or missing necessary resources.' class CheckXcodebuildStuckThread(threading.Thread): @@ -130,11 +132,12 @@ def __init__(self, self._startup_timeout_sec = ( startup_timeout_sec or _XCODEBUILD_TEST_STARTUP_TIMEOUT_SEC) - def Execute(self, return_output=True): + def Execute(self, return_output=True, result_bundle_path=None): """Executes the xcodebuild test command. Args: return_output: bool, whether save output in the execution result. + result_bundle_path: string, path of the result bundle. Returns: a tuple of two fields: @@ -159,14 +162,16 @@ def Execute(self, return_output=True): test_failed = False for i in range(max_attempts): + if result_bundle_path and os.path.exists(result_bundle_path): + shutil.rmtree(result_bundle_path, ignore_errors=True) process = subprocess.Popen( self._command, env=run_env, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, text=True) + stderr=subprocess.STDOUT, encoding='ascii', errors='ignore') check_xcodebuild_stuck = CheckXcodebuildStuckThread( process, self._startup_timeout_sec) check_xcodebuild_stuck.start() output = io.StringIO() - for stdout_line in iter(process.stdout.readline, ''): + for stdout_line in process.stdout: if not test_started: # Terminates the CheckXcodebuildStuckThread when test has started # or XCTRunner.app has started. @@ -188,8 +193,7 @@ def Execute(self, return_output=True): if self._failed_signal and self._failed_signal in stdout_line: test_failed = True - sys.stdout.write(stdout_line) - sys.stdout.flush() + print(stdout_line, flush=True, end='') # If return_output is false, the output is only used for checking error # cause and deleting cached files (_DeleteTestCacheFileDirs method). if return_output or not test_started: @@ -210,10 +214,13 @@ def Execute(self, return_output=True): return self._GetResultForXcodebuildStuck(output, return_output) output_str = output.getvalue() + # Don't need to retry the case for the damaged test bundle. + if _BUNDLE_DAMAGED in output_str: + return (runner_exit_codes.EXITCODE.TEST_NOT_START, + output_str if return_output else None) + if self._sdk == ios_constants.SDK.IPHONEOS: - if ((re.search(_DEVICE_TYPE_WAS_NULL_PATTERN, output_str) or - _LOST_CONNECTION_ERROR in output_str or - _LOST_CONNECTION_TO_DTSERVICEHUB_ERROR in output_str) and + if (self._NeedRetryForDeviceTesting(output_str) and i < max_attempts - 1): logging.warning( 'Failed to launch test on the device. Will relaunch again ' @@ -298,6 +305,14 @@ def _NeedRecreateSim(self, output_str): return True return False + def _NeedRetryForDeviceTesting(self, output_str): + """Returns true if the device testing needs retry.""" + return (re.search(_DEVICE_TYPE_WAS_NULL_PATTERN, output_str) or + _LOST_CONNECTION_ERROR in output_str or + _LOST_CONNECTION_TO_DTSERVICEHUB_ERROR in output_str or + _DEVICE_NO_LONGER_CONNECTED in output_str or + _UNABLE_FIND_DEVICE_IDENTIFIER in output_str) + def _DeleteTestCacheFileDirs(xcodebuild_test_output, sdk, test_type): """Deletes the cache files of the test session according to arguments.""" @@ -346,4 +361,5 @@ def _FetchTestCacheFileDirs(xcodebuild_test_output, max_dir_num=1): def _ReadFileTailInShell(file_path, line): """Tails the file in the last several lines.""" - return subprocess.check_output(['tail', '-%d' % line, file_path], text=True) + return subprocess.check_output(['tail', '-%d' % line, + file_path]).decode('utf-8') diff --git a/test_runner/xcresult_util.py b/test_runner/xcresult_util.py index a241cb7..7e253a6 100644 --- a/test_runner/xcresult_util.py +++ b/test_runner/xcresult_util.py @@ -113,7 +113,7 @@ def _GetResultBundleObject(xcresult_path, bundle_id=None): ] if bundle_id: command.extend(['--id', bundle_id]) - return json.loads(subprocess.check_output(command, text=True)) + return json.loads(subprocess.check_output(command).decode('utf-8')) def _GetFailureTestRefs(test_summary): diff --git a/test_runner/xctest_session.py b/test_runner/xctest_session.py index d78026a..1e879db 100644 --- a/test_runner/xctest_session.py +++ b/test_runner/xctest_session.py @@ -375,8 +375,10 @@ def _FinalizeTestType( def _DetectTestType(test_bundle_dir): """Detects if the test bundle is XCUITest or XCTest.""" test_bundle_exec_path = os.path.join( - test_bundle_dir, os.path.splitext(os.path.basename(test_bundle_dir))[0]) - output = subprocess.check_output(['nm', test_bundle_exec_path], text=True) + test_bundle_dir, + os.path.splitext(os.path.basename(test_bundle_dir))[0]) + output = subprocess.check_output(['nm', + test_bundle_exec_path]).decode('utf-8') if 'XCUIApplication' in output: return ios_constants.TestType.XCUITEST else: diff --git a/test_runner/xctestrun.py b/test_runner/xctestrun.py index a82fd5e..a904f56 100644 --- a/test_runner/xctestrun.py +++ b/test_runner/xctestrun.py @@ -55,8 +55,8 @@ def __init__(self, xctestrun_file_path, test_type=None, aut_bundle_id=None): self._xctestrun_file_path = xctestrun_file_path self._xctestrun_file_plist_obj = plist_util.Plist(xctestrun_file_path) # xctestrun file always has only key at root dict. - self._root_key = next(iter(self._xctestrun_file_plist_obj.GetPlistField( - None))) + self._root_key = list( + self._xctestrun_file_plist_obj.GetPlistField(None).keys())[0] self._test_type = test_type self._aut_bundle_id = aut_bundle_id @@ -193,7 +193,7 @@ def Run(self, device_id, sdk, derived_data_dir, startup_timeout_sec, device_id=device_id, app_bundle_id=self._aut_bundle_id, startup_timeout_sec=startup_timeout_sec).Execute( - return_output=False) + return_output=False, result_bundle_path=result_bundle_path) return exit_code @property @@ -474,13 +474,11 @@ def _GenerateTestRootForXcuitest(self): _CopyAndSignFramework( os.path.join(platform_library_path, 'Frameworks/XCTest.framework'), runner_app_frameworks_dir, test_bundle_signing_identity) - xcode_version_num = xcode_info_util.GetXcodeVersionNumber() - if xcode_version_num >= 900: - _CopyAndSignFramework( - os.path.join(platform_library_path, - 'PrivateFrameworks/XCTAutomationSupport.framework'), - runner_app_frameworks_dir, test_bundle_signing_identity) - if xcode_version_num >= 1100: + _CopyAndSignFramework( + os.path.join(platform_library_path, + 'PrivateFrameworks/XCTAutomationSupport.framework'), + runner_app_frameworks_dir, test_bundle_signing_identity) + if xcode_info_util.GetXcodeVersionNumber() >= 1100: _CopyAndSignLibFile( os.path.join(platform_path, _LIB_XCTEST_SWIFT_RELATIVE_PATH), runner_app_frameworks_dir, test_bundle_signing_identity) @@ -531,21 +529,32 @@ def _GetUitestRunnerAppFromXcode(self, platform_library_path): uitest_runner_app_name = '%s-Runner' % test_bundle_name uitest_runner_app = os.path.join(self._test_root_dir, uitest_runner_app_name + '.app') + if os.path.exists(uitest_runner_app): + shutil.rmtree(uitest_runner_app) shutil.copytree(xctrunner_app, uitest_runner_app) uitest_runner_exec = os.path.join(uitest_runner_app, uitest_runner_app_name) shutil.move( os.path.join(uitest_runner_app, 'XCTRunner'), uitest_runner_exec) # XCTRunner is multi-archs. When launching XCTRunner on arm64e device, it # will be launched as arm64e process by default. If the test bundle is arm64 - # bundle, the XCTRunner which hosts the test bundle will failed to be + # bundle, the XCTRunner which hosts the test bundle will fail to be # launched. So removing the arm64e arch from XCTRunner can resolve this # case. + test_executable = os.path.join(self._test_bundle_dir, test_bundle_name) if self._device_arch == ios_constants.ARCH.ARM64E: - test_executable = os.path.join(self._test_bundle_dir, test_bundle_name) test_archs = bundle_util.GetFileArchTypes(test_executable) if ios_constants.ARCH.ARM64E not in test_archs: bundle_util.RemoveArchType(uitest_runner_exec, ios_constants.ARCH.ARM64E) + # XCTRunner is multi-archs. When launching XCTRunner on Apple silicon + # simulator, it will be launched as arm64 process by default. If the test + # bundle is still x86_64, the XCTRunner which hosts the test bundle will + # fail to be launched. So removing the arm64 arch from XCTRunner can + # resolve this case. + elif not self._on_device: + test_archs = bundle_util.GetFileArchTypes(test_executable) + if ios_constants.ARCH.X86_64 in test_archs: + bundle_util.RemoveArchType(uitest_runner_exec, ios_constants.ARCH.ARM64) runner_app_info_plist_path = os.path.join(uitest_runner_app, 'Info.plist') info_plist = plist_util.Plist(runner_app_info_plist_path) @@ -605,20 +614,11 @@ def _GenerateTestRootForXctest(self): os.path.join(platform_path, 'Developer/Library/Frameworks/XCTest.framework'), app_under_test_frameworks_dir, app_under_test_signing_identity) - xcode_version_num = xcode_info_util.GetXcodeVersionNumber() - if xcode_version_num < 1000: - bundle_injection_lib = os.path.join( - platform_path, 'Developer/Library/PrivateFrameworks/' - 'IDEBundleInjection.framework') - _CopyAndSignFramework(bundle_injection_lib, - app_under_test_frameworks_dir, - app_under_test_signing_identity) - else: - bundle_injection_lib = os.path.join( - platform_path, 'Developer/usr/lib/libXCTestBundleInject.dylib') - _CopyAndSignLibFile(bundle_injection_lib, app_under_test_frameworks_dir, - app_under_test_signing_identity) - if xcode_version_num >= 1100: + bundle_injection_lib = os.path.join( + platform_path, 'Developer/usr/lib/libXCTestBundleInject.dylib') + _CopyAndSignLibFile(bundle_injection_lib, app_under_test_frameworks_dir, + app_under_test_signing_identity) + if xcode_info_util.GetXcodeVersionNumber() >= 1100: _CopyAndSignFramework( os.path.join( platform_path, 'Developer/Library/PrivateFrameworks/' @@ -627,6 +627,22 @@ def _GenerateTestRootForXctest(self): _CopyAndSignLibFile( os.path.join(platform_path, _LIB_XCTEST_SWIFT_RELATIVE_PATH), app_under_test_frameworks_dir, app_under_test_signing_identity) + if xcode_info_util.GetXcodeVersionNumber() >= 1300: + _CopyAndSignFramework( + os.path.join( + platform_path, 'Developer/Library/PrivateFrameworks/' + 'XCUIAutomation.framework'), + app_under_test_frameworks_dir, app_under_test_signing_identity) + _CopyAndSignFramework( + os.path.join( + platform_path, 'Developer/Library/PrivateFrameworks/' + 'XCTestCore.framework'), + app_under_test_frameworks_dir, app_under_test_signing_identity) + _CopyAndSignFramework( + os.path.join( + platform_path, 'Developer/Library/PrivateFrameworks/' + 'XCUnit.framework'), + app_under_test_frameworks_dir, app_under_test_signing_identity) bundle_util.CodesignBundle(self._test_bundle_dir) bundle_util.CodesignBundle(self._app_under_test_dir) @@ -636,19 +652,10 @@ def _GenerateTestRootForXctest(self): developer_path = '__PLATFORMS__/%s.platform/Developer' % platform_name if self._on_device: - if xcode_info_util.GetXcodeVersionNumber() < 1000: - dyld_insert_libs = ('__TESTHOST__/Frameworks/' - 'IDEBundleInjection.framework/IDEBundleInjection') - else: - dyld_insert_libs = '__TESTHOST__/Frameworks/libXCTestBundleInject.dylib' + dyld_insert_libs = '__TESTHOST__/Frameworks/libXCTestBundleInject.dylib' else: - if xcode_info_util.GetXcodeVersionNumber() < 1000: - dyld_insert_libs = ('%s/Library/PrivateFrameworks/' - 'IDEBundleInjection.framework/IDEBundleInjection' % - developer_path) - else: - dyld_insert_libs = ('%s/usr/lib/libXCTestBundleInject.dylib' % - developer_path) + dyld_insert_libs = ('%s/usr/lib/libXCTestBundleInject.dylib' % + developer_path) test_envs = { 'XCInjectBundleInto': os.path.join('__TESTHOST__', app_under_test_name), 'DYLD_FRAMEWORK_PATH': '__TESTROOT__:{developer}/Library/Frameworks:' From a13ec9de45d225620dd4b5e30b23bdc663631f31 Mon Sep 17 00:00:00 2001 From: Weiming Dai Date: Mon, 1 Nov 2021 10:47:22 -0700 Subject: [PATCH 27/45] Add DEVELOPER_DIR env back to execute logic test This change was from 67100a8dd85dd65546a9465dfbd7ffd5da962a0e and removed in 7f8fc81b10c8d93f09f6fe38b2a3f37ba25336a6. --- test_runner/logic_test_util.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test_runner/logic_test_util.py b/test_runner/logic_test_util.py index 8439e4b..c464e76 100644 --- a/test_runner/logic_test_util.py +++ b/test_runner/logic_test_util.py @@ -14,6 +14,7 @@ """The helper classes to run logic test.""" +import os import subprocess import sys @@ -62,6 +63,10 @@ def RunLogicTestOnSim(sim_id, version_util.GetVersionNumber(os_version) < 1220): key = _SIMCTL_ENV_VAR_PREFIX + 'DYLD_FALLBACK_LIBRARY_PATH' simctl_env_vars[key] = xcode_info_util.GetSwift5FallbackLibsDir() + # We need to set the DEVELOPER_DIR to ensure xcrun works correctly + developer_dir = os.environ.get('DEVELOPER_DIR') + if developer_dir: + simctl_env_vars['DEVELOPER_DIR'] = developer_dir command = [ 'xcrun', 'simctl', 'spawn', '-s', sim_id, xcode_info_util.GetXctestToolPath(ios_constants.SDK.IPHONESIMULATOR)] From a171066b8728f9f219761a11bc084437c9d004cb Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Thu, 27 Jan 2022 13:19:55 -0800 Subject: [PATCH 28/45] Stop including stderr in simulator output On M1 macs running anything with xcrun prints a bunch of these warnings to stderr (FB9089778): ``` objc[35011]: Class AMSupportURLConnectionDelegate is implemented in both /usr/lib/libauthinstall.dylib (0x1ffc8ab90) and /Library/Apple/System/Library/PrivateFrameworks/MobileDevice.framework/Versions/A/MobileDevice (0x1044502c8). One of the two will be used. Which one is undefined. ``` So capturing this output, and then using it as json, is invalid on M1 machines when this happens. None of the consumers of this function check the output for any error strings, so this change should only improve the other use cases. --- shared/ios_constants.py | 2 -- simulator_control/simulator_util.py | 9 +++------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/shared/ios_constants.py b/shared/ios_constants.py index 54403eb..f8d57b5 100644 --- a/shared/ios_constants.py +++ b/shared/ios_constants.py @@ -44,8 +44,6 @@ def enum(**enums): XCTRUNNER_STARTED_SIGNAL = 'Running tests...' CORESIMULATOR_INTERRUPTED_ERROR = 'CoreSimulatorService connection interrupted' -CORESIMULATOR_CHANGE_ERROR = ('CoreSimulator detected Xcode.app relocation or ' - 'CoreSimulatorService version change.') LAUNCH_OPTIONS_JSON_HELP = ( """The path of json file, which contains options of launching test. diff --git a/simulator_control/simulator_util.py b/simulator_control/simulator_util.py index 0f186fe..b2014c6 100644 --- a/simulator_control/simulator_util.py +++ b/simulator_control/simulator_util.py @@ -715,14 +715,11 @@ def RunSimctlCommand(command): stderr=subprocess.PIPE, encoding='utf-8') stdout, stderr = process.communicate() - if ios_constants.CORESIMULATOR_CHANGE_ERROR in stderr: - output = stdout - else: - output = '\n'.join([stdout, stderr]) - output = output.strip() + all_output = '\n'.join([stdout, stderr]) + output = stdout.strip() if process.poll() != 0: if (i < (_SIMCTL_MAX_ATTEMPTS - 1) and - ios_constants.CORESIMULATOR_INTERRUPTED_ERROR in output): + ios_constants.CORESIMULATOR_INTERRUPTED_ERROR in all_output): continue raise ios_errors.SimError(output) return output From b7585cacbaaeeece3e493a7aadb0aa3b9e066f7e Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Fri, 4 Feb 2022 09:03:05 -0800 Subject: [PATCH 29/45] Fix ProductModuleName with dashes https://github.com/google/xctestrunner/pull/33#issuecomment-957971036 --- test_runner/xctestrun.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_runner/xctestrun.py b/test_runner/xctestrun.py index 9f4cea2..4390489 100644 --- a/test_runner/xctestrun.py +++ b/test_runner/xctestrun.py @@ -499,7 +499,7 @@ def _GenerateTestRootForXcuitest(self): 'DYLD_LIBRARY_PATH': '__TESTROOT__:%s/usr/lib' % developer_path } self._xctestrun_dict = { - 'ProductModuleName': self._test_name, + 'ProductModuleName': self._test_name.replace("-", "_"), 'IsUITestBundle': True, 'SystemAttachmentLifetime': 'keepNever', 'TestBundlePath': self._test_bundle_dir, @@ -666,7 +666,7 @@ def _GenerateTestRootForXctest(self): 'DYLD_LIBRARY_PATH': '__TESTROOT__:%s/usr/lib:' % developer_path } self._xctestrun_dict = { - 'ProductModuleName': self._test_name, + 'ProductModuleName': self._test_name.replace("-", "_"), 'TestHostPath': self._app_under_test_dir, 'TestBundlePath': self._test_bundle_dir, 'IsAppHostedTestBundle': True, @@ -687,7 +687,7 @@ def _GenerateTestRootForLogicTest(self): 'DYLD_LIBRARY_PATH': dyld_framework_path } self._xctestrun_dict = { - 'ProductModuleName': self._test_name, + 'ProductModuleName': self._test_name.replace("-", "_"), 'TestBundlePath': self._test_bundle_dir, 'TestHostPath': xcode_info_util.GetXctestToolPath(self._sdk), 'TestingEnvironmentVariables': test_envs, From b8788ca73e75c25f43822b7c0d7ef9909ba528a3 Mon Sep 17 00:00:00 2001 From: Frank Escobedo Date: Mon, 21 Mar 2022 14:12:46 -0700 Subject: [PATCH 30/45] xcode13: copy private frameworks for xcuitest tests --- test_runner/xctestrun.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/test_runner/xctestrun.py b/test_runner/xctestrun.py index 4390489..f409fd6 100644 --- a/test_runner/xctestrun.py +++ b/test_runner/xctestrun.py @@ -486,7 +486,19 @@ def _GenerateTestRootForXcuitest(self): uitest_runner_app, entitlements_plist_path=entitlements_plist_path, identity=test_bundle_signing_identity) - + if xcode_info_util.GetXcodeVersionNumber() >= 1300: + _CopyAndSignFramework( + os.path.join(platform_library_path, + 'PrivateFrameworks/XCUIAutomation.framework'), + runner_app_frameworks_dir, test_bundle_signing_identity) + _CopyAndSignFramework( + os.path.join(platform_library_path, + 'PrivateFrameworks/XCTestCore.framework'), + runner_app_frameworks_dir, test_bundle_signing_identity) + _CopyAndSignFramework( + os.path.join(platform_library_path, + 'PrivateFrameworks/XCUnit.framework'), + runner_app_frameworks_dir, test_bundle_signing_identity) bundle_util.CodesignBundle(self._test_bundle_dir) bundle_util.CodesignBundle(self._app_under_test_dir) From 13e9f8f27c98e96c7b9be59dcf9c4b7307092484 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Mon, 28 Nov 2022 11:13:32 -0800 Subject: [PATCH 31/45] Remove use of subpar This project was recently officially marked as deprecated https://github.com/google/subpar/pull/136 rules_apple has supported referencing non-single-file targets for this use case for a few years, so this drop in replacement as a py_binary seems to work fine. --- BUILD | 8 +------- WORKSPACE | 9 --------- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/BUILD b/BUILD index f07ed62..0691865 100644 --- a/BUILD +++ b/BUILD @@ -1,7 +1,5 @@ package(default_visibility = ["//visibility:public"]) -load("@subpar//:subpar.bzl", "par_binary") - py_library( name = "shared", srcs = glob(["shared/*.py"]), @@ -15,15 +13,11 @@ py_library( ], ) -par_binary( +py_binary( name = "ios_test_runner", srcs = ["__init__.py"] + glob( ["test_runner/*.py"], ), - compiler_args = [ - "--interpreter", - "/usr/bin/python3", - ], main = "test_runner/ios_test_runner.py", python_version = "PY3", deps = [ diff --git a/WORKSPACE b/WORKSPACE index c29c882..ab7d480 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,10 +1 @@ workspace(name = "xctestrunner") - -# For packaging python scripts. -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") - -git_repository( - name = "subpar", - remote = "https://github.com/google/subpar", - tag = "2.0.0", -) From 5bb42a527fa472d1c6d27727dea80e32a324126a Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Thu, 12 Jan 2023 15:34:32 -0800 Subject: [PATCH 32/45] Add initial MODULE.bazel --- MODULE.bazel | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 MODULE.bazel diff --git a/MODULE.bazel b/MODULE.bazel new file mode 100644 index 0000000..0ed8806 --- /dev/null +++ b/MODULE.bazel @@ -0,0 +1,5 @@ +module( + name = "xctestrunner", + version = "0.2.15", + compatibility_level = 1, +) From ade9c4346790545201a6d52aa4eb5a7f19ebafcd Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Wed, 18 Jan 2023 20:12:41 -0800 Subject: [PATCH 33/45] Fix startup json var spelling --- shared/ios_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/ios_constants.py b/shared/ios_constants.py index f8d57b5..fa95120 100644 --- a/shared/ios_constants.py +++ b/shared/ios_constants.py @@ -80,7 +80,7 @@ def enum(**enums): Whether captures screenshots automatically in ui test. If yes, will save the screenshots when the test failed. By default, it is false. Prior Xcode 9, this option does not work and the auto screenshot is enable by default. - startup_timeout_seconds: int + startup_timeout_sec: int Seconds until the xcodebuild command is deemed stuck. destination_timeout_sec: int Wait for the given seconds while searching for the destination device. From 85a5d3146544c3540951c633a510353300aa1113 Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Wed, 9 Nov 2022 13:46:11 +0100 Subject: [PATCH 34/45] Do not rely on repository imports With Bzlmod, on disk ("canonical") repository names are an implementation detail and can thus no longer be used in Python imports. Instead, similar to non-Bazel Python projects, the import paths should be based on actual directories under the repo root - all repo roots are implicitly added to the Python path by Bazel. --- BUILD | 10 +++++----- __init__.py => xctestrunner/__init__.py | 0 {shared => xctestrunner/shared}/__init__.py | 0 {shared => xctestrunner/shared}/bundle_util.py | 0 {shared => xctestrunner/shared}/ios_constants.py | 0 {shared => xctestrunner/shared}/ios_errors.py | 0 {shared => xctestrunner/shared}/plist_util.py | 0 .../shared}/provisioning_profile.py | 0 {shared => xctestrunner/shared}/version_util.py | 0 {shared => xctestrunner/shared}/xcode_info_util.py | 0 .../simulator_control}/__init__.py | 0 .../simulator_control}/simtype_profile.py | 0 .../simulator_control}/simulator_util.py | 0 {test_runner => xctestrunner/test_runner}/__init__.py | 0 .../test_runner}/ios_test_runner.py | 0 .../test_runner}/logic_test_util.py | 0 .../test_runner}/runner_exit_codes.py | 0 .../test_runner}/xcodebuild_test_executor.py | 0 .../test_runner}/xcresult_util.py | 0 .../test_runner}/xctest_session.py | 0 {test_runner => xctestrunner/test_runner}/xctestrun.py | 0 21 files changed, 5 insertions(+), 5 deletions(-) rename __init__.py => xctestrunner/__init__.py (100%) rename {shared => xctestrunner/shared}/__init__.py (100%) rename {shared => xctestrunner/shared}/bundle_util.py (100%) rename {shared => xctestrunner/shared}/ios_constants.py (100%) rename {shared => xctestrunner/shared}/ios_errors.py (100%) rename {shared => xctestrunner/shared}/plist_util.py (100%) rename {shared => xctestrunner/shared}/provisioning_profile.py (100%) rename {shared => xctestrunner/shared}/version_util.py (100%) rename {shared => xctestrunner/shared}/xcode_info_util.py (100%) rename {simulator_control => xctestrunner/simulator_control}/__init__.py (100%) rename {simulator_control => xctestrunner/simulator_control}/simtype_profile.py (100%) rename {simulator_control => xctestrunner/simulator_control}/simulator_util.py (100%) rename {test_runner => xctestrunner/test_runner}/__init__.py (100%) rename {test_runner => xctestrunner/test_runner}/ios_test_runner.py (100%) rename {test_runner => xctestrunner/test_runner}/logic_test_util.py (100%) rename {test_runner => xctestrunner/test_runner}/runner_exit_codes.py (100%) rename {test_runner => xctestrunner/test_runner}/xcodebuild_test_executor.py (100%) rename {test_runner => xctestrunner/test_runner}/xcresult_util.py (100%) rename {test_runner => xctestrunner/test_runner}/xctest_session.py (100%) rename {test_runner => xctestrunner/test_runner}/xctestrun.py (100%) diff --git a/BUILD b/BUILD index 0691865..61d9ca7 100644 --- a/BUILD +++ b/BUILD @@ -2,12 +2,12 @@ package(default_visibility = ["//visibility:public"]) py_library( name = "shared", - srcs = glob(["shared/*.py"]), + srcs = glob(["xctestrunner/shared/*.py"]), ) py_library( name = "simulator", - srcs = glob(["simulator_control/*.py"]), + srcs = glob(["xctestrunner/simulator_control/*.py"]), deps = [ ":shared", ], @@ -15,10 +15,10 @@ py_library( py_binary( name = "ios_test_runner", - srcs = ["__init__.py"] + glob( - ["test_runner/*.py"], + srcs = ["xctestrunner/__init__.py"] + glob( + ["xctestrunner/test_runner/*.py"], ), - main = "test_runner/ios_test_runner.py", + main = "xctestrunner/test_runner/ios_test_runner.py", python_version = "PY3", deps = [ ":shared", diff --git a/__init__.py b/xctestrunner/__init__.py similarity index 100% rename from __init__.py rename to xctestrunner/__init__.py diff --git a/shared/__init__.py b/xctestrunner/shared/__init__.py similarity index 100% rename from shared/__init__.py rename to xctestrunner/shared/__init__.py diff --git a/shared/bundle_util.py b/xctestrunner/shared/bundle_util.py similarity index 100% rename from shared/bundle_util.py rename to xctestrunner/shared/bundle_util.py diff --git a/shared/ios_constants.py b/xctestrunner/shared/ios_constants.py similarity index 100% rename from shared/ios_constants.py rename to xctestrunner/shared/ios_constants.py diff --git a/shared/ios_errors.py b/xctestrunner/shared/ios_errors.py similarity index 100% rename from shared/ios_errors.py rename to xctestrunner/shared/ios_errors.py diff --git a/shared/plist_util.py b/xctestrunner/shared/plist_util.py similarity index 100% rename from shared/plist_util.py rename to xctestrunner/shared/plist_util.py diff --git a/shared/provisioning_profile.py b/xctestrunner/shared/provisioning_profile.py similarity index 100% rename from shared/provisioning_profile.py rename to xctestrunner/shared/provisioning_profile.py diff --git a/shared/version_util.py b/xctestrunner/shared/version_util.py similarity index 100% rename from shared/version_util.py rename to xctestrunner/shared/version_util.py diff --git a/shared/xcode_info_util.py b/xctestrunner/shared/xcode_info_util.py similarity index 100% rename from shared/xcode_info_util.py rename to xctestrunner/shared/xcode_info_util.py diff --git a/simulator_control/__init__.py b/xctestrunner/simulator_control/__init__.py similarity index 100% rename from simulator_control/__init__.py rename to xctestrunner/simulator_control/__init__.py diff --git a/simulator_control/simtype_profile.py b/xctestrunner/simulator_control/simtype_profile.py similarity index 100% rename from simulator_control/simtype_profile.py rename to xctestrunner/simulator_control/simtype_profile.py diff --git a/simulator_control/simulator_util.py b/xctestrunner/simulator_control/simulator_util.py similarity index 100% rename from simulator_control/simulator_util.py rename to xctestrunner/simulator_control/simulator_util.py diff --git a/test_runner/__init__.py b/xctestrunner/test_runner/__init__.py similarity index 100% rename from test_runner/__init__.py rename to xctestrunner/test_runner/__init__.py diff --git a/test_runner/ios_test_runner.py b/xctestrunner/test_runner/ios_test_runner.py similarity index 100% rename from test_runner/ios_test_runner.py rename to xctestrunner/test_runner/ios_test_runner.py diff --git a/test_runner/logic_test_util.py b/xctestrunner/test_runner/logic_test_util.py similarity index 100% rename from test_runner/logic_test_util.py rename to xctestrunner/test_runner/logic_test_util.py diff --git a/test_runner/runner_exit_codes.py b/xctestrunner/test_runner/runner_exit_codes.py similarity index 100% rename from test_runner/runner_exit_codes.py rename to xctestrunner/test_runner/runner_exit_codes.py diff --git a/test_runner/xcodebuild_test_executor.py b/xctestrunner/test_runner/xcodebuild_test_executor.py similarity index 100% rename from test_runner/xcodebuild_test_executor.py rename to xctestrunner/test_runner/xcodebuild_test_executor.py diff --git a/test_runner/xcresult_util.py b/xctestrunner/test_runner/xcresult_util.py similarity index 100% rename from test_runner/xcresult_util.py rename to xctestrunner/test_runner/xcresult_util.py diff --git a/test_runner/xctest_session.py b/xctestrunner/test_runner/xctest_session.py similarity index 100% rename from test_runner/xctest_session.py rename to xctestrunner/test_runner/xctest_session.py diff --git a/test_runner/xctestrun.py b/xctestrunner/test_runner/xctestrun.py similarity index 100% rename from test_runner/xctestrun.py rename to xctestrunner/test_runner/xctestrun.py From de26ac93220231100f598d18606fa1413b3f42fb Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Sat, 11 Feb 2023 01:56:55 +0100 Subject: [PATCH 35/45] Fix __init__.py generation causing wrong sys.path entry to be picked up --- BUILD | 2 +- __init__.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 __init__.py diff --git a/BUILD b/BUILD index 61d9ca7..e73145e 100644 --- a/BUILD +++ b/BUILD @@ -15,7 +15,7 @@ py_library( py_binary( name = "ios_test_runner", - srcs = ["xctestrunner/__init__.py"] + glob( + srcs = ["__init__.py", "xctestrunner/__init__.py"] + glob( ["xctestrunner/test_runner/*.py"], ), main = "xctestrunner/test_runner/ios_test_runner.py", diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..2988dd4 --- /dev/null +++ b/__init__.py @@ -0,0 +1,3 @@ +import sys +from . import xctestrunner +sys.modules['xctestrunner'] = xctestrunner From 24629f3e6c0dda397f14924b64eb45d04433c07e Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Sat, 11 Feb 2023 18:08:03 +0100 Subject: [PATCH 36/45] Update __init__.py Co-authored-by: Richard Levasseur --- __init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/__init__.py b/__init__.py index 2988dd4..0144164 100644 --- a/__init__.py +++ b/__init__.py @@ -1,3 +1,7 @@ +# This file is just here to work around Bazel auto-generating an empty file here. +# What happens is `import xctestrunner` loads this file instead of the +# subdirectory because the root runfiles directory comes before the subdirectory +# on sys.path import sys from . import xctestrunner sys.modules['xctestrunner'] = xctestrunner From a7e336b08e2606ca19c57216f8af9a78b379d7d8 Mon Sep 17 00:00:00 2001 From: Jerry Marino Date: Fri, 10 Feb 2023 15:38:00 -0800 Subject: [PATCH 37/45] [Xcode 14] Don't collect diagnostics Unless people are explicitly looking at this, it adds more overhead and increases result bundle size. If someone wants to have this, then perhaps we'd add a flag to have it back but seems like a sensible default --- xctestrunner/test_runner/xctestrun.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/xctestrunner/test_runner/xctestrun.py b/xctestrunner/test_runner/xctestrun.py index f409fd6..35757ad 100644 --- a/xctestrunner/test_runner/xctestrun.py +++ b/xctestrunner/test_runner/xctestrun.py @@ -182,6 +182,9 @@ def Run(self, device_id, sdk, derived_data_dir, startup_timeout_sec, shutil.rmtree(result_bundle_path, ignore_errors=True) command.extend(['-resultBundlePath', result_bundle_path]) + if xcode_version >= 1410: + command.extend(['-collect-test-diagnostics', 'Never']) + if destination_timeout_sec: command.extend(['-destination-timeout', str(destination_timeout_sec)]) exit_code, _ = xcodebuild_test_executor.XcodebuildTestExecutor( From e5086851e55e7b058ac76308ed1e9cdd595db4f8 Mon Sep 17 00:00:00 2001 From: Aditya Atul Tirodkar Date: Fri, 12 May 2023 23:19:10 -0700 Subject: [PATCH 38/45] Update xctestrun.py with XCTestSupport.framework that has issues with Xcode 14.3 fixes https://github.com/google/xctestrunner/issues/57 --- xctestrunner/test_runner/xctestrun.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/xctestrunner/test_runner/xctestrun.py b/xctestrunner/test_runner/xctestrun.py index 35757ad..07a9327 100644 --- a/xctestrunner/test_runner/xctestrun.py +++ b/xctestrunner/test_runner/xctestrun.py @@ -502,6 +502,11 @@ def _GenerateTestRootForXcuitest(self): os.path.join(platform_library_path, 'PrivateFrameworks/XCUnit.framework'), runner_app_frameworks_dir, test_bundle_signing_identity) + if xcode_info_util.GetXcodeVersionNumber() >= 1430: + _CopyAndSignFramework( + os.path.join(platform_library_path, + 'PrivateFrameworks/XCTestSupport.framework'), + runner_app_frameworks_dir, test_bundle_signing_identity) bundle_util.CodesignBundle(self._test_bundle_dir) bundle_util.CodesignBundle(self._app_under_test_dir) @@ -659,6 +664,15 @@ def _GenerateTestRootForXctest(self): platform_path, 'Developer/Library/PrivateFrameworks/' 'XCUnit.framework'), app_under_test_frameworks_dir, app_under_test_signing_identity) + if xcode_info_util.GetXcodeVersionNumber() >= 1430: + _CopyAndSignFramework( + os.path.join( + platform_path, + 'Developer/Library/PrivateFrameworks/XCTestSupport.framework', + ), + app_under_test_frameworks_dir, + app_under_test_signing_identity, + ) bundle_util.CodesignBundle(self._test_bundle_dir) bundle_util.CodesignBundle(self._app_under_test_dir) From 4c5709da9444eae6bba2425734b8654635bed0a6 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Wed, 17 May 2023 09:41:07 -0700 Subject: [PATCH 39/45] Fix -collect-test-diagnostics arg for Xcode 14.1 / 14.2 Seems like Xcode 14.1 and 14.2 require an equals sign here, otherwise they fail with: ``` xcodebuild: error: option -collect-test-diagnostics requires one of two values: on-failure or never ``` Xcode 14.3 seems to have fixed this. --- xctestrunner/test_runner/xctestrun.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xctestrunner/test_runner/xctestrun.py b/xctestrunner/test_runner/xctestrun.py index 07a9327..28c2e0c 100644 --- a/xctestrunner/test_runner/xctestrun.py +++ b/xctestrunner/test_runner/xctestrun.py @@ -183,7 +183,7 @@ def Run(self, device_id, sdk, derived_data_dir, startup_timeout_sec, command.extend(['-resultBundlePath', result_bundle_path]) if xcode_version >= 1410: - command.extend(['-collect-test-diagnostics', 'Never']) + command.extend(['-collect-test-diagnostics=never']) if destination_timeout_sec: command.extend(['-destination-timeout', str(destination_timeout_sec)]) From b7698df3d435b6491b4b4c0f9fc7a63fbed5e3a6 Mon Sep 17 00:00:00 2001 From: aircraft-cerier Date: Wed, 16 Aug 2023 15:32:33 -0700 Subject: [PATCH 40/45] Create .xctestrun file instead of .plist --- xctestrunner/test_runner/xctest_session.py | 2 +- xctestrunner/test_runner/xctestrun.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/xctestrunner/test_runner/xctest_session.py b/xctestrunner/test_runner/xctest_session.py index 1e879db..347d014 100644 --- a/xctestrunner/test_runner/xctest_session.py +++ b/xctestrunner/test_runner/xctest_session.py @@ -177,7 +177,7 @@ def SetLaunchOptions(self, launch_options): if launch_options.get('uitest_auto_screenshots'): self._disable_uitest_auto_screenshots = False # By default, this SystemAttachmentLifetime field is in the generated - # xctestrun.plist. + # test.xctestrun. try: self._xctestrun_obj.DeleteXctestrunField('SystemAttachmentLifetime') except ios_errors.PlistError: diff --git a/xctestrunner/test_runner/xctestrun.py b/xctestrunner/test_runner/xctestrun.py index 28c2e0c..e0f2ef5 100644 --- a/xctestrunner/test_runner/xctestrun.py +++ b/xctestrunner/test_runner/xctestrun.py @@ -342,7 +342,7 @@ def GenerateXctestrun(self): return self._xctestrun_obj if self._work_dir: self._test_root_dir = os.path.join(self._work_dir, 'TEST_ROOT') - xctestrun_file_path = os.path.join(self._test_root_dir, 'xctestrun.plist') + xctestrun_file_path = os.path.join(self._test_root_dir, 'test.xctestrun') if os.path.exists(xctestrun_file_path): logging.info('Skips generating xctestrun file which is generated.') self._xctestrun_obj = XctestRun(xctestrun_file_path) @@ -370,7 +370,7 @@ def GenerateXctestrun(self): elif self._test_type == ios_constants.TestType.LOGIC_TEST: self._GenerateTestRootForLogicTest() - xctestrun_file_path = os.path.join(self._test_root_dir, 'xctestrun.plist') + xctestrun_file_path = os.path.join(self._test_root_dir, 'test.xctestrun') plist_util.Plist(xctestrun_file_path).SetPlistField('Runner', self._xctestrun_dict) @@ -418,8 +418,8 @@ def _ValidateArguments(self): def _GenerateTestRootForXcuitest(self): """Generates the test root for XCUITest. - The approach constructs xctestrun.plist and uitest runner app from Xcode. - Then copies app under test, test bundle, xctestrun.plist and uitest + The approach constructs test.xctestrun and uitest runner app from Xcode. + Then copies app under test, test bundle, test.xctestrun and uitest runner app to test root directory. """ platform_path = xcode_info_util.GetSdkPlatformPath(self._sdk) @@ -605,8 +605,8 @@ def _PrepareUitestInRunerApp(self, uitest_runner_app): def _GenerateTestRootForXctest(self): """Generates the test root for XCTest. - The approach constructs xctestrun.plist from Xcode. Then copies app under - test, test bundle and xctestrun.plist to test root directory. + The approach constructs test.xctestrun from Xcode. Then copies app under + test, test bundle and test.xctestrun to test root directory. """ app_under_test_plugins_dir = os.path.join( self._app_under_test_dir, 'PlugIns') @@ -705,8 +705,8 @@ def _GenerateTestRootForXctest(self): def _GenerateTestRootForLogicTest(self): """Generates the test root for Logic test. - The approach constructs xctestrun.plist from Xcode. Then copies test bundle - and xctestrun.plist to test root directory. + The approach constructs test.xctestrun from Xcode. Then copies test bundle + and test.xctestrun to test root directory. """ dyld_framework_path = os.path.join( xcode_info_util.GetSdkPlatformPath(self._sdk), From ad26f06db1777da41b4e282b8c919aa1553cda54 Mon Sep 17 00:00:00 2001 From: Andrei Raifura Date: Mon, 14 Apr 2025 11:52:07 -0700 Subject: [PATCH 41/45] Update path to DeviceTypes for Xcode 16.3 --- xctestrunner/simulator_control/simtype_profile.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/xctestrunner/simulator_control/simtype_profile.py b/xctestrunner/simulator_control/simtype_profile.py index df6673b..944ada4 100644 --- a/xctestrunner/simulator_control/simtype_profile.py +++ b/xctestrunner/simulator_control/simtype_profile.py @@ -50,7 +50,9 @@ def profile_plist_obj(self): xcode_version = xcode_info_util.GetXcodeVersionNumber() platform_path = xcode_info_util.GetSdkPlatformPath( ios_constants.SDK.IPHONEOS) - if xcode_version >= 1100: + if xcode_version >= 1630: + sim_profiles_dir = '/Library/Developer/CoreSimulator/Profiles' + elif xcode_version >= 1100: sim_profiles_dir = os.path.join( platform_path, 'Library/Developer/CoreSimulator/Profiles') else: From 429e167e7da7497053ff8256015cc81f72ce7c0b Mon Sep 17 00:00:00 2001 From: Vakhid Betrakhmadov Date: Mon, 6 Jan 2025 19:37:11 +0000 Subject: [PATCH 42/45] Fix xcresulttool for Xcode 16 Resolve: #65 --- xctestrunner/test_runner/xcresult_util.py | 40 ++++++++++++++++------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/xctestrunner/test_runner/xcresult_util.py b/xctestrunner/test_runner/xcresult_util.py index 7e253a6..57b3696 100644 --- a/xctestrunner/test_runner/xcresult_util.py +++ b/xctestrunner/test_runner/xcresult_util.py @@ -19,6 +19,7 @@ import subprocess from xctestrunner.shared import ios_errors +from xctestrunner.shared import xcode_info_util def ExposeXcresult(xcresult_path, output_path): @@ -50,11 +51,12 @@ def _ExposeDiagnostics(xcresult_path, output_path, action_result): if 'diagnosticsRef' not in action_result: return diagnostics_id = action_result['diagnosticsRef']['id']['_value'] - subprocess.check_call([ - 'xcrun', 'xcresulttool', 'export', '--path', xcresult_path, - '--output-path', output_path, '--type', 'directory', '--id', - diagnostics_id + export_command = _MakeXcresulttoolCommand([ + 'export', '--path', xcresult_path, + '--output-path', output_path, '--type', 'directory', '--id', + diagnostics_id ]) + subprocess.check_call(export_command) def _ExposeAttachments(xcresult_path, output_path, action_result): @@ -90,11 +92,12 @@ def _ExposeAttachments(xcresult_path, output_path, action_result): target_file_path = os.path.join(target_file_dir, file_name) payload_ref_id = attachment['payloadRef']['id']['_value'] - subprocess.check_call([ - 'xcrun', 'xcresulttool', 'export', '--path', xcresult_path, - '--output-path', target_file_path, '--type', 'file', '--id', - payload_ref_id + export_command = _MakeXcresulttoolCommand([ + 'export', '--path', xcresult_path, + '--output-path', target_file_path, '--type', 'file', '--id', + payload_ref_id ]) + subprocess.check_call(export_command) def _GetResultBundleObject(xcresult_path, bundle_id=None): @@ -107,10 +110,9 @@ def _GetResultBundleObject(xcresult_path, bundle_id=None): Returns: A dict, result bundle object in json format. """ - command = [ - 'xcrun', 'xcresulttool', 'get', '--format', 'json', '--path', - xcresult_path - ] + command = _MakeXcresulttoolCommand([ + 'get', '--format', 'json', '--path', xcresult_path + ]) if bundle_id: command.extend(['--id', bundle_id]) return json.loads(subprocess.check_output(command).decode('utf-8')) @@ -135,3 +137,17 @@ def _GetFailureTestRefs(test_summary): summary_ref_id = test_summary['summaryRef']['id']['_value'] failure_test_refs.append(summary_ref_id) return failure_test_refs + +def _MakeXcresulttoolCommand(args): + """Constructs xcresulttool command for selected Xcode version. + + Args: + args: array, a list of arguments to pass to xcresulttool. + Returns: + The xcresulttool command. + """ + command = ['xcrun', 'xcresulttool'] + args + xcode_version = xcode_info_util.GetXcodeVersionNumber() + if xcode_version >= 1600: + command.extend(['--legacy']) + return command From 5f53db6acbc094635d1a9468cf16af089246226a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nenad=20Miks=CC=8Ca?= Date: Tue, 22 Jul 2025 17:09:47 +0200 Subject: [PATCH 43/45] fix xctestrunner test runner for Xcode 16.3 and above --- xctestrunner/test_runner/xctestrun.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/xctestrunner/test_runner/xctestrun.py b/xctestrunner/test_runner/xctestrun.py index e0f2ef5..4dbae1c 100644 --- a/xctestrunner/test_runner/xctestrun.py +++ b/xctestrunner/test_runner/xctestrun.py @@ -649,11 +649,18 @@ def _GenerateTestRootForXctest(self): os.path.join(platform_path, _LIB_XCTEST_SWIFT_RELATIVE_PATH), app_under_test_frameworks_dir, app_under_test_signing_identity) if xcode_info_util.GetXcodeVersionNumber() >= 1300: - _CopyAndSignFramework( - os.path.join( - platform_path, 'Developer/Library/PrivateFrameworks/' - 'XCUIAutomation.framework'), - app_under_test_frameworks_dir, app_under_test_signing_identity) + if xcode_info_util.GetXcodeVersionNumber() >= 1640: + _CopyAndSignFramework( + os.path.join( + platform_path, 'Developer/Library/Frameworks/' + 'XCUIAutomation.framework'), + app_under_test_frameworks_dir, app_under_test_signing_identity) + else: + _CopyAndSignFramework( + os.path.join( + platform_path, 'Developer/Library/PrivateFrameworks/' + 'XCUIAutomation.framework'), + app_under_test_frameworks_dir, app_under_test_signing_identity) _CopyAndSignFramework( os.path.join( platform_path, 'Developer/Library/PrivateFrameworks/' From a33ff27ba873a61416261a99e7073443ab6de0ff Mon Sep 17 00:00:00 2001 From: JP Simard Date: Tue, 30 Sep 2025 11:06:57 -0400 Subject: [PATCH 44/45] Retry `xcodebuild` by rebooting simulator when stuck When `xcodebuild` times out waiting for tests to start, reboot the simulator and retry up to 3 times before failing. This handles intermittent issues where the simulator gets into a bad state. --- xctestrunner/test_runner/xcodebuild_test_executor.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/xctestrunner/test_runner/xcodebuild_test_executor.py b/xctestrunner/test_runner/xcodebuild_test_executor.py index 56b5fcb..db987c6 100644 --- a/xctestrunner/test_runner/xcodebuild_test_executor.py +++ b/xctestrunner/test_runner/xcodebuild_test_executor.py @@ -211,6 +211,16 @@ def Execute(self, return_output=True, result_bundle_path=None): check_xcodebuild_stuck.Terminate() if check_xcodebuild_stuck.is_xcodebuild_stuck: + if self._sdk == ios_constants.SDK.IPHONESIMULATOR and i < max_attempts - 1: + logging.warning( + 'xcodebuild stuck on simulator (attempt %d/%d). ' + 'Will reboot simulator and retry.', + i + 1, max_attempts) + simulator = simulator_util.Simulator(self._device_id) + simulator.Shutdown() + simulator.Boot() + time.sleep(2) + continue return self._GetResultForXcodebuildStuck(output, return_output) output_str = output.getvalue() From 6749b2d5fca05d8ddd728fa44e326aa1bda01833 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Thu, 30 Oct 2025 12:25:37 -0700 Subject: [PATCH 45/45] Add explicit load for python rules This is required for bazel @ HEAD and upcoming 9.x --- BUILD | 7 ++++++- MODULE.bazel | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/BUILD b/BUILD index e73145e..050376c 100644 --- a/BUILD +++ b/BUILD @@ -1,3 +1,5 @@ +load("@rules_python//python:defs.bzl", "py_binary", "py_library") + package(default_visibility = ["//visibility:public"]) py_library( @@ -15,7 +17,10 @@ py_library( py_binary( name = "ios_test_runner", - srcs = ["__init__.py", "xctestrunner/__init__.py"] + glob( + srcs = [ + "__init__.py", + "xctestrunner/__init__.py", + ] + glob( ["xctestrunner/test_runner/*.py"], ), main = "xctestrunner/test_runner/ios_test_runner.py", diff --git a/MODULE.bazel b/MODULE.bazel index 0ed8806..258f0a4 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -3,3 +3,5 @@ module( version = "0.2.15", compatibility_level = 1, ) + +bazel_dep(name = "rules_python", version = "1.0.0")