diff --git a/BUILD b/BUILD index bc06659..050376c 100644 --- a/BUILD +++ b/BUILD @@ -1,30 +1,43 @@ -package(default_visibility = ["//visibility:public"]) +load("@rules_python//python:defs.bzl", "py_binary", "py_library") -load("@subpar//:subpar.bzl", "par_binary") +package(default_visibility = ["//visibility:public"]) py_library( - name = 'shared', - srcs = glob(['shared/*.py']), + name = "shared", + srcs = glob(["xctestrunner/shared/*.py"]), ) py_library( - name = 'simulator', - srcs = glob(['simulator_control/*.py']), + name = "simulator", + srcs = glob(["xctestrunner/simulator_control/*.py"]), deps = [ - ':shared', + ":shared", ], ) -par_binary( - name = 'ios_test_runner', - srcs = glob( - ['test_runner/*.py'], - exclude = ['test_runner/TestProject/**'] +py_binary( + name = "ios_test_runner", + srcs = [ + "__init__.py", + "xctestrunner/__init__.py", + ] + glob( + ["xctestrunner/test_runner/*.py"], ), - main = 'test_runner/ios_test_runner.py', + main = "xctestrunner/test_runner/ios_test_runner.py", + python_version = "PY3", deps = [ - ':shared', - ':simulator', + ":shared", + ":simulator", ], - data = glob(['test_runner/TestProject/**']), +) + +# Consumed by bazel tests. +filegroup( + name = "for_bazel_tests", + testonly = 1, + srcs = glob(["**/*"]), + # Exposed publicly just so other rules can use this if they set up + # integration tests that need to copy all the support files into + # a temporary workspace for the tests. + visibility = ["//visibility:public"], ) diff --git a/MODULE.bazel b/MODULE.bazel new file mode 100644 index 0000000..258f0a4 --- /dev/null +++ b/MODULE.bazel @@ -0,0 +1,7 @@ +module( + name = "xctestrunner", + version = "0.2.15", + compatibility_level = 1, +) + +bazel_dep(name = "rules_python", version = "1.0.0") diff --git a/README.md b/README.md index 1f78dcf..15551dc 100644 --- a/README.md +++ b/README.md @@ -3,19 +3,26 @@ A tool for running prebuilt iOS tests on iOS real device and simulator. ## Features - It supports XCTest (Xcode Unit Test), XCUITest (Xcode UI Test). -- It supports iOS 7+ iOS real device, iOS simulator. +- It supports iOS 11+ iOS real device, iOS simulator. - It supports launch options configuration: test methods to run, additional environment variables, additional arguments. -- It supports Xcode 7+. +- It supports Xcode 10+. ## Prerequisites -- Install Xcode (Xcode 7+). XCUITest support requires Xcode 8+. +- Install Xcode (Xcode 10+). - [Install bazel](https://docs.bazel.build/install.html) (optional). -- py module [biplist](https://github.com/wooster/biplist). ## Installation You can download the ios_test_runner.par binary in [release](https://github.com/google/xctestrunner/releases) +or build the ios_test_runner.par binary by bazel: +``` +$ git clone https://github.com/google/xctestrunner.git +$ cd xctestrunner +$ bazel build :ios_test_runner.par +$ ls bazel-bin/ios_test_runner.par +``` + ## Usage - Build your app under test and test bundle. You can use Xcode.app, `xcodebuild` command line tool or [bazel](https://github.com/bazelbuild/bazel). @@ -35,12 +42,3 @@ Disclaimer: This is not an official Google product. XCTestRunner uses Apple native tool `xcodebuild`, `simctl` to control iOS Simulator and launch tests on iOS devices. - -For testing, XCTestRunner injects app under test and test bundle file into a -dummy project. Then the dummy project can be used `xcodebuild test` to run -XCTest (not for XCUITest), or `xcodebuild build-for-testing` to generate -xctestrun file for further testing. - -For iOS 7 real device testing, the latest supported Xcode version is 7.2.1. -For iOS 7 simulator testing, latest supported Xcode version is 7.2.1 and latest -supported MacOS version is Yosemite (10.10.x). diff --git a/WORKSPACE b/WORKSPACE index 497b765..ab7d480 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,8 +1 @@ -# For packaging python scripts. -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") - -git_repository( - name = "subpar", - remote = "https://github.com/google/subpar", - tag = "2.0.0", -) +workspace(name = "xctestrunner") diff --git a/__init__.py b/__init__.py index e69de29..0144164 100644 --- a/__init__.py +++ b/__init__.py @@ -0,0 +1,7 @@ +# This file is just here to work around Bazel auto-generating an empty file here. +# What happens is `import xctestrunner` loads this file instead of the +# subdirectory because the root runfiles directory comes before the subdirectory +# on sys.path +import sys +from . import xctestrunner +sys.modules['xctestrunner'] = xctestrunner diff --git a/simulator_control/__init__.py b/simulator_control/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test_runner/TestProject/TestProject.xcodeproj/project.pbxproj b/test_runner/TestProject/TestProject.xcodeproj/project.pbxproj deleted file mode 100755 index 3d20a0b..0000000 --- a/test_runner/TestProject/TestProject.xcodeproj/project.pbxproj +++ /dev/null @@ -1,342 +0,0 @@ - - - - - archiveVersion - 1 - classes - - objectVersion - 46 - objects - - AppUnderTestBuildConfig - - buildSettings - - CODE_SIGNING_REQUIRED - NO - CODE_SIGN_IDENTITY - - DEVELOPMENT_TEAM - - INFOPLIST_FILE - $(BUILT_PRODUCTS_DIR)/$(APP_UNDER_TEST_NAME).app/Info.plist - PRODUCT_BUNDLE_IDENTIFIER - - PRODUCT_NAME - $(APP_UNDER_TEST_NAME) - PROVISIONING_PROFILE_SPECIFIER - - - isa - XCBuildConfiguration - name - Debug - - AppUnderTestBuildConfigList - - buildConfigurations - - AppUnderTestBuildConfig - - defaultConfigurationIsVisible - 0 - isa - XCConfigurationList - - AppUnderTestFile - - explicitFileType - wrapper.application - includeInIndex - 0 - isa - PBXFileReference - path - .app - sourceTree - BUILT_PRODUCTS_DIR - - AppUnderTestTarget - - buildConfigurationList - AppUnderTestBuildConfigList - buildPhases - - buildRules - - dependencies - - isa - PBXNativeTarget - name - %AppUnderTestName% - productName - %AppUnderTestName% - productReference - AppUnderTestFile - productType - com.apple.product-type.application - - TestProjectBuildConfig - - buildSettings - - IPHONEOS_DEPLOYMENT_TARGET - - SDKROOT - iphoneos - APP_UNDER_TEST_NAME - - XCTEST_BUNDLE_NAME - - XCUITEST_BUNDLE_NAME - - - isa - XCBuildConfiguration - name - Debug - - TestProjectBuildConfigList - - buildConfigurations - - TestProjectBuildConfig - - defaultConfigurationIsVisible - 0 - defaultConfigurationName - Debug - isa - XCConfigurationList - - TestProjectGroup - - children - - TestProjectProducts - - isa - PBXGroup - sourceTree - <group> - - TestProjectObject - - attributes - - LastUpgradeCheck - 0720 - ORGANIZATIONNAME - Google Inc - TargetAttributes - - AppUnderTestTarget - - CreatedOnToolsVersion - 7.2 - - XCTestBundleTarget - - CreatedOnToolsVersion - 7.2 - TestTargetID - AppUnderTestTarget - - XCUITestBundleTarget - - CreatedOnToolsVersion - 8.0 - TestTargetID - AppUnderTestTarget - - - - buildConfigurationList - TestProjectBuildConfigList - compatibilityVersion - Xcode 3.2 - developmentRegion - English - hasScannedForEncodings - 0 - isa - PBXProject - knownRegions - - en - Base - - mainGroup - TestProjectGroup - productRefGroup - TestProjectProducts - projectDirPath - - projectRoot - - targets - - AppUnderTestTarget - XCTestBundleTarget - XCUITestBundleTarget - - - TestProjectProducts - - children - - AppUnderTestFile - XCTestBundleFile - XCUITestBundleFile - - isa - PBXGroup - name - Products - sourceTree - <group> - - XCTestBundleBuildConfig - - buildSettings - - CODE_SIGN_IDENTITY - - DEVELOPMENT_TEAM - - INFOPLIST_FILE - $(BUILT_PRODUCTS_DIR)/$(XCTEST_BUNDLE_NAME).xctest/Info.plist - PRODUCT_NAME - $(XCTEST_BUNDLE_NAME) - TEST_HOST - $(BUILT_PRODUCTS_DIR)/$(APP_UNDER_TEST_NAME).app/$(APP_UNDER_TEST_NAME) - - isa - XCBuildConfiguration - name - Debug - - XCTestBundleBuildConfigList - - buildConfigurations - - XCTestBundleBuildConfig - - defaultConfigurationIsVisible - 0 - isa - XCConfigurationList - - XCTestBundleFile - - explicitFileType - wrapper.cfbundle - includeInIndex - 0 - isa - PBXFileReference - path - .xctest - sourceTree - BUILT_PRODUCTS_DIR - - XCTestBundleTarget - - buildConfigurationList - XCTestBundleBuildConfigList - buildPhases - - buildRules - - dependencies - - isa - PBXNativeTarget - name - %XCTestBundleName% - productName - %XCTestBundleName% - productReference - XCTestBundleFile - productType - com.apple.product-type.bundle.unit-test - - XCUITestBundleBuildConfig - - buildSettings - - CODE_SIGN_IDENTITY - - DEVELOPMENT_TEAM - - INFOPLIST_FILE - $(BUILT_PRODUCTS_DIR)/$(XCUITEST_BUNDLE_NAME).xctest/Info.plist - PRODUCT_BUNDLE_IDENTIFIER - - PRODUCT_NAME - $(XCUITEST_BUNDLE_NAME) - PROVISIONING_PROFILE_SPECIFIER - - TEST_TARGET_NAME - $(APP_UNDER_TEST_NAME) - USES_XCTRUNNER - true - - isa - XCBuildConfiguration - name - Debug - - XCUITestBundleBuildConfigList - - buildConfigurations - - XCUITestBundleBuildConfig - - defaultConfigurationIsVisible - 0 - isa - XCConfigurationList - - XCUITestBundleFile - - explicitFileType - wrapper.cfbundle - includeInIndex - 0 - isa - PBXFileReference - path - UITests.xctest - sourceTree - BUILT_PRODUCTS_DIR - - XCUITestBundleTarget - - buildConfigurationList - XCUITestBundleBuildConfigList - buildPhases - - buildRules - - dependencies - - isa - PBXNativeTarget - name - %XCUITestBundleName% - productName - %XCUITestBundleName% - productReference - XCUITestBundleFile - productType - com.apple.product-type.bundle.ui-testing - - - rootObject - TestProjectObject - - diff --git a/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXctest.xcscheme b/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXctest.xcscheme deleted file mode 100755 index 60b405b..0000000 --- a/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXctest.xcscheme +++ /dev/null @@ -1,104 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXcuitest.xcscheme b/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXcuitest.xcscheme deleted file mode 100755 index 1abe430..0000000 --- a/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXcuitest.xcscheme +++ /dev/null @@ -1,105 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/test_runner/__init__.py b/test_runner/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test_runner/dummy_project.py b/test_runner/dummy_project.py deleted file mode 100644 index b704d2d..0000000 --- a/test_runner/dummy_project.py +++ /dev/null @@ -1,571 +0,0 @@ -# Copyright 2017 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Helper class for dummy Xcode project generated by prebuilt bundles. - -The dummy project supports sdk iphonesimulator and test type XCUITest. It can -be run with `xcodebuild build-for-testing`. -""" - -import logging -import os -import pkgutil -import shutil -import subprocess -import tempfile -import xml.etree.ElementTree as ET - -from xctestrunner.shared import bundle_util -from xctestrunner.shared import ios_constants -from xctestrunner.shared import ios_errors -from xctestrunner.shared import plist_util -from xctestrunner.shared import provisioning_profile -from xctestrunner.shared import xcode_info_util -from xctestrunner.test_runner import xcodebuild_test_executor - - -_DEFAULT_PERMS = 0o0777 -_DUMMYPROJECT_DIR_NAME = 'TestProject' -_DUMMYPROJECT_XCODEPROJ_NAME = 'TestProject.xcodeproj' -_DUMMYPROJECT_PBXPROJ_NAME = 'project.pbxproj' -_DUMMYPROJECT_XCTESTS_SCHEME = 'TestProjectXctest' -_DUMMYPROJECT_XCUITESTS_SCHEME = 'TestProjectXcuitest' - -_SIGNAL_BUILD_FOR_TESTING_SUCCEEDED = '** TEST BUILD SUCCEEDED **' -_SIGNAL_XCODEBUILD_TEST_SUCCEEDED = '** TEST SUCCEEDED **' -_SIGNAL_XCODEBUILD_TEST_FAILED = '** TEST FAILED **' - - -class DummyProject(object): - """Handles a dummy project with prebuilt bundles.""" - - def __init__(self, - app_under_test_dir, - test_bundle_dir, - sdk=ios_constants.SDK.IPHONESIMULATOR, - test_type=ios_constants.TestType.XCUITEST, - work_dir=None, - keychain_path=None): - """Initializes the DummyProject object. - - Args: - app_under_test_dir: string, path of the app to be tested in - dummy project. - test_bundle_dir: string, path of the test bundle. - sdk: string, SDKRoot of the dummy project. See supported SDKs in - module shared.ios_constants. - test_type: string, test type of the test bundle. See supported test types - in module shared.ios_constants. - work_dir: string, work directory which contains run files. - keychain_path: string, path of preferred keychain to use. - """ - self._app_under_test_dir = app_under_test_dir - self._test_bundle_dir = test_bundle_dir - self._sdk = sdk - self._test_type = test_type - if work_dir: - self._work_dir = os.path.join(work_dir, 'dummy_project') - else: - self._work_dir = None - self._dummy_project_path = None - self._keychain_path = keychain_path - self._xcodeproj_dir_path = None - self._pbxproj_file_path = None - self._is_dummy_project_generated = False - self._delete_work_dir = False - self._ValidateArguments() - self._test_scheme = None - if test_type == ios_constants.TestType.XCTEST: - self._test_scheme = _DUMMYPROJECT_XCTESTS_SCHEME - elif test_type == ios_constants.TestType.XCUITEST: - self._test_scheme = _DUMMYPROJECT_XCUITESTS_SCHEME - - def __enter__(self): - self.GenerateDummyProject() - return self - - def __exit__(self, unused_type, unused_value, unused_traceback): - """Deletes the temp directories.""" - self.Close() - - @property - def pbxproj_file_path(self): - """Gets the pbxproj file path of the dummy project.""" - return self._pbxproj_file_path - - @property - def test_scheme_path(self): - """Gets the test scheme path of the dummy project.""" - return os.path.join( - self._xcodeproj_dir_path, - 'xcshareddata/xcschemes', - '%s.xcscheme' % self._test_scheme) - - def BuildForTesting(self, built_products_dir, derived_data_dir): - """Runs `xcodebuild build-for-testing` with the dummy project. - - If app under test or test bundle are not in built_products_dir, will copy - the file into built_products_dir. - - Args: - built_products_dir: path of the built products dir in this build session. - derived_data_dir: path of the derived data dir in this build session. - Raises: - BuildFailureError: when failed to build the dummy project. - """ - self.GenerateDummyProject() - self._PrepareBuildProductsDir(built_products_dir) - logging.info('Running `xcodebuild build-for-testing` with dummy project.\n' - '\tbuilt_product_dir = %s\n\tderived_data_path = %s\n', - built_products_dir, - derived_data_dir) - command = ['xcodebuild', 'build-for-testing', - 'BUILT_PRODUCTS_DIR=' + built_products_dir, - 'SDKROOT=' + self._sdk, - '-project', self._xcodeproj_dir_path, - '-scheme', self._test_scheme, - '-derivedDataPath', derived_data_dir] - run_env = dict(os.environ) - run_env['NSUnbufferedIO'] = 'YES' - try: - output = subprocess.check_output( - command, env=run_env, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - raise ios_errors.BuildFailureError('Failed to build the dummy project. ' - 'Output is:\n%s' % e.output) - - if _SIGNAL_BUILD_FOR_TESTING_SUCCEEDED not in output: - raise ios_errors.BuildFailureError('Failed to build the dummy project. ' - 'Output is:\n%s' % output) - - def RunXcTest(self, device_id, built_products_dir, derived_data_dir, - startup_timeout_sec): - """Runs `xcodebuild test` with the dummy project. - - If app under test or test bundle are not in built_products_dir, will copy - the file into built_products_dir. - - Args: - device_id: string, id of the device. - built_products_dir: path of the built products dir in this build session. - derived_data_dir: path of the derived data dir in this build session. - startup_timeout_sec: Seconds until the xcodebuild command is deemed stuck. - - Returns: - A value of type runner_exit_codes.EXITCODE. - - Raises: - IllegalArgumentError: when test type is not xctest. - """ - if self._test_type != ios_constants.TestType.XCTEST: - raise ios_errors.IllegalArgumentError( - 'Only xctest dummy project is supported to run `xcodebuild test`. ' - 'The test type %s is not supported.' % self._test_type) - self.GenerateDummyProject() - # In Xcode 7.3+, the folder structure of app under test is changed. - if xcode_info_util.GetXcodeVersionNumber() >= 730: - app_under_test_plugin_path = os.path.join(self._app_under_test_dir, - 'PlugIns') - if not os.path.exists(app_under_test_plugin_path): - os.mkdir(app_under_test_plugin_path) - test_bundle_under_plugin_path = os.path.join( - app_under_test_plugin_path, os.path.basename(self._test_bundle_dir)) - if not os.path.exists(test_bundle_under_plugin_path): - shutil.copytree(self._test_bundle_dir, test_bundle_under_plugin_path) - self._PrepareBuildProductsDir(built_products_dir) - - logging.info('Running `xcodebuild test` with dummy project.\n' - 'device_id= %s\n' - 'built_product_dir = %s\nderived_data_path = %s\n', - device_id, - built_products_dir, - derived_data_dir) - command = ['xcodebuild', 'test', - 'BUILT_PRODUCTS_DIR=' + built_products_dir, - '-project', self._xcodeproj_dir_path, - '-scheme', self._test_scheme, - '-destination', 'id=' + device_id, - '-derivedDataPath', derived_data_dir] - app_bundle_id = bundle_util.GetBundleId(self._app_under_test_dir) - exit_code, _ = xcodebuild_test_executor.XcodebuildTestExecutor( - command, - succeeded_signal=_SIGNAL_XCODEBUILD_TEST_SUCCEEDED, - failed_signal=_SIGNAL_XCODEBUILD_TEST_FAILED, - sdk=self._sdk, - test_type=self._test_type, - device_id=device_id, - app_bundle_id=app_bundle_id, - startup_timeout_sec=startup_timeout_sec).Execute(return_output=False) - return exit_code - - def GenerateDummyProject(self): - """Generates the dummy project according to the specification. - - Raises: - IllegalArgumentError: when the sdk or test type is not supported. - """ - if self._is_dummy_project_generated: - return - - if self._work_dir: - self._dummy_project_path = os.path.join(self._work_dir, - _DUMMYPROJECT_DIR_NAME) - if os.path.exists(self._dummy_project_path): - logging.info('Skips generating dummy project which is generated.') - self._xcodeproj_dir_path = os.path.join( - self._dummy_project_path, _DUMMYPROJECT_XCODEPROJ_NAME) - self._pbxproj_file_path = os.path.join( - self._xcodeproj_dir_path, _DUMMYPROJECT_PBXPROJ_NAME) - self._is_dummy_project_generated = True - return - - logging.info('Generating dummy project.') - if self._work_dir: - if not os.path.exists(self._work_dir): - os.mkdir(self._work_dir) - else: - self._work_dir = tempfile.mkdtemp() - self._delete_work_dir = True - self._dummy_project_path = os.path.join(self._work_dir, - _DUMMYPROJECT_DIR_NAME) - shutil.copytree(_GetTestProject(self._work_dir), self._dummy_project_path) - for root, dirs, files in os.walk(self._dummy_project_path): - for d in dirs: - os.chmod(os.path.join(root, d), _DEFAULT_PERMS) - for f in files: - os.chmod(os.path.join(root, f), _DEFAULT_PERMS) - self._xcodeproj_dir_path = os.path.join( - self._dummy_project_path, _DUMMYPROJECT_XCODEPROJ_NAME) - self._pbxproj_file_path = os.path.join( - self._xcodeproj_dir_path, _DUMMYPROJECT_PBXPROJ_NAME) - - # Set the iOS deployment target in pbxproj. - # If don't set this field, the default value will be the latest supported - # iOS version which may make the app installation failure. - self._SetIosDeploymentTarget() - - # Overwrite the pbxproj file content for test type specific. - if self._test_type == ios_constants.TestType.XCUITEST: - self._SetPbxprojForXcuitest() - elif self._test_type == ios_constants.TestType.XCTEST: - self._SetPbxprojForXctest() - - self._is_dummy_project_generated = True - logging.info('Dummy project is generated.') - - def Close(self): - """Deletes the temp directories.""" - if self._delete_work_dir and os.path.exists(self._work_dir): - shutil.rmtree(self._work_dir) - - def _ValidateArguments(self): - """Checks whether the arguments of the dummy project is valid. - - Raises: - IllegalArgumentError: when the sdk or test type is not supported. - """ - if self._sdk not in ios_constants.SUPPORTED_SDKS: - raise ios_errors.IllegalArgumentError( - 'The sdk %s is not supported. Supported sdks are %s.' - % (self._sdk, ios_constants.SUPPORTED_SDKS)) - if self._test_type not in ios_constants.SUPPORTED_TEST_TYPES: - raise ios_errors.IllegalArgumentError( - 'The test type %s is not supported. Supported test types are %s.' - % (self._test_type, ios_constants.SUPPORTED_TEST_TYPES)) - - def _PrepareBuildProductsDir(self, built_products_dir): - """Prepares the build products directory for dummy project. - - Args: - built_products_dir: path of the directory to be prepared. - """ - logging.info('Preparing build products directory %s for dummy project.', - built_products_dir) - app_under_test_name = os.path.basename(self._app_under_test_dir) - test_bundle_name = os.path.basename(self._test_bundle_dir) - if not os.path.exists( - os.path.join(built_products_dir, app_under_test_name)): - shutil.copytree(self._app_under_test_dir, - os.path.join(built_products_dir, app_under_test_name)) - if not os.path.exists( - os.path.join(built_products_dir, test_bundle_name)): - shutil.copytree(self._test_bundle_dir, - os.path.join(built_products_dir, test_bundle_name)) - - def _SetIosDeploymentTarget(self): - """Sets the iOS deployment target in dummy project's pbxproj.""" - pbxproj_plist_obj = plist_util.Plist(self.pbxproj_file_path) - pbxproj_plist_obj.SetPlistField( - 'objects:TestProjectBuildConfig:buildSettings:' - 'IPHONEOS_DEPLOYMENT_TARGET', - bundle_util.GetMinimumOSVersion(self._app_under_test_dir)) - - def _SetPbxprojForXcuitest(self): - """Sets the dummy project's pbxproj for xcuitest.""" - pbxproj_plist_obj = plist_util.Plist(self.pbxproj_file_path) - pbxproj_objects = pbxproj_plist_obj.GetPlistField('objects') - - # Sets the build setting of test bundle for generated XCTRunner.app signing. - # 1) If run with iphonesimulator, don't need to set any fields in build - # setting. xcodebuild will sign the XCTRunner.app with identity '-' and no - # provisioning profile by default. - # 2) If runs with iphoneos and the app under test's embedded provisioning - # profile is 'iOS Team Provisioning Profile: *', set build setting for using - # Xcode managed provisioning profile to sign the XCTRunner.app. - # 3) If runs with iphoneos and the app under test's embedded provisioning - # profile is specific, set build setting for using app under test's - # embedded provisioning profile to sign the XCTRunner.app. If the - # provisioning profile is not installed in the Mac machine, also installs - # it. - # 4) The test bundle's provisioning profile can be overwrited by method - # SetTestBundleProvisioningProfile. - if self._sdk == ios_constants.SDK.IPHONEOS: - build_setting = pbxproj_objects[ - 'XCUITestBundleBuildConfig']['buildSettings'] - build_setting['PRODUCT_BUNDLE_IDENTIFIER'] = bundle_util.GetBundleId( - self._test_bundle_dir) - build_setting['DEVELOPMENT_TEAM'] = bundle_util.GetDevelopmentTeam( - self._test_bundle_dir) - embedded_provision = provisioning_profile.ProvisiongProfile( - os.path.join(self._app_under_test_dir, 'embedded.mobileprovision'), - self._work_dir, - keychain_path=self._keychain_path) - embedded_provision.Install() - # Case 2) - if embedded_provision.name.startswith('iOS Team Provisioning Profile: '): - build_setting['CODE_SIGN_IDENTITY'] = 'iPhone Developer' - else: - # Case 3) - build_setting['CODE_SIGN_IDENTITY'] = bundle_util.GetCodesignIdentity( - self._app_under_test_dir) - (build_setting[ - 'PROVISIONING_PROFILE_SPECIFIER']) = embedded_provision.name - - # Sets the app under test and test bundle. - test_project_build_setting = pbxproj_objects[ - 'TestProjectBuildConfig']['buildSettings'] - app_under_test_name = os.path.splitext( - os.path.basename(self._app_under_test_dir))[0] - pbxproj_objects['AppUnderTestTarget']['name'] = app_under_test_name - pbxproj_objects['AppUnderTestTarget']['productName'] = app_under_test_name - test_project_build_setting['APP_UNDER_TEST_NAME'] = app_under_test_name - test_bundle_name = os.path.splitext( - os.path.basename(self._test_bundle_dir))[0] - pbxproj_objects['XCUITestBundleTarget']['name'] = test_bundle_name - pbxproj_objects['XCUITestBundleTarget']['productName'] = test_bundle_name - test_project_build_setting['XCUITEST_BUNDLE_NAME'] = test_bundle_name - - pbxproj_plist_obj.SetPlistField('objects', pbxproj_objects) - - def _SetPbxprojForXctest(self): - """Sets the dummy project's pbxproj for xctest.""" - pbxproj_plist_obj = plist_util.Plist(self.pbxproj_file_path) - pbxproj_objects = pbxproj_plist_obj.GetPlistField('objects') - - # Sets the build setting for app under test and unit test bundle signing. - # 1) If run with iphonesimulator, don't need to set any fields in build - # setting. xcodebuild will sign bundles with identity '-' and no - # provisioning profile by default. - # 2) If runs with iphoneos and the app under test's embedded provisioning - # profile is 'iOS Team Provisioning Profile: *', set build setting for using - # Xcode managed provisioning profile to sign bundles. - # 3) If runs with iphoneos and the app under test's embedded provisioning - # profile is specific, set build setting with using app under test's - # embedded provisioning profile. - if self._sdk == ios_constants.SDK.IPHONEOS: - aut_build_setting = pbxproj_objects[ - 'AppUnderTestBuildConfig']['buildSettings'] - test_build_setting = pbxproj_objects[ - 'XCTestBundleBuildConfig']['buildSettings'] - aut_build_setting['CODE_SIGNING_REQUIRED'] = 'YES' - aut_build_setting['PRODUCT_BUNDLE_IDENTIFIER'] = bundle_util.GetBundleId( - self._app_under_test_dir) - embedded_provision = provisioning_profile.ProvisiongProfile( - os.path.join(self._app_under_test_dir, 'embedded.mobileprovision'), - self._work_dir, - keychain_path=self._keychain_path) - embedded_provision.Install() - # Case 2) - if embedded_provision.name.startswith('iOS Team Provisioning Profile: '): - aut_build_setting['CODE_SIGN_IDENTITY'] = 'iPhone Developer' - test_build_setting['CODE_SIGN_IDENTITY'] = 'iPhone Developer' - app_under_test_dev_team = bundle_util.GetDevelopmentTeam( - self._app_under_test_dir) - aut_build_setting['DEVELOPMENT_TEAM'] = app_under_test_dev_team - test_build_setting['DEVELOPMENT_TEAM'] = app_under_test_dev_team - else: - # Case 3) - app_under_test_sign_identity = bundle_util.GetCodesignIdentity( - self._app_under_test_dir) - aut_build_setting['CODE_SIGN_IDENTITY'] = app_under_test_sign_identity - test_build_setting['CODE_SIGN_IDENTITY'] = app_under_test_sign_identity - (aut_build_setting[ - 'PROVISIONING_PROFILE_SPECIFIER']) = embedded_provision.name - - # Sets the app under test and test bundle. - test_project_build_setting = pbxproj_objects[ - 'TestProjectBuildConfig']['buildSettings'] - app_under_test_name = os.path.splitext( - os.path.basename(self._app_under_test_dir))[0] - pbxproj_objects['AppUnderTestTarget']['name'] = app_under_test_name - pbxproj_objects['AppUnderTestTarget']['productName'] = app_under_test_name - test_project_build_setting['APP_UNDER_TEST_NAME'] = app_under_test_name - test_bundle_name = os.path.splitext( - os.path.basename(self._test_bundle_dir))[0] - pbxproj_objects['XCTestBundleTarget']['name'] = test_bundle_name - pbxproj_objects['XCTestBundleTarget']['productName'] = test_bundle_name - test_project_build_setting['XCTEST_BUNDLE_NAME'] = test_bundle_name - - pbxproj_plist_obj.SetPlistField('objects', pbxproj_objects) - - def SetTestBundleProvisioningProfile(self, test_bundle_provisioning_profile): - """Sets the provisioning profile specifier to the test bundle. - - If the given provisioning profile is a path, will also install it in the - host. - - Args: - test_bundle_provisioning_profile: string, name/path of the provisioning - profile of test bundle. - """ - if not test_bundle_provisioning_profile: - return - provisioning_profile_is_file = False - if (test_bundle_provisioning_profile.startswith('/') and - os.path.exists(test_bundle_provisioning_profile)): - provisioning_profile_is_file = True - - if self._sdk != ios_constants.SDK.IPHONEOS: - logging.warning( - 'Can only set provisioning profile to test bundle in iphoneos SDK. ' - 'But current SDK is %s', self._sdk) - return - self.GenerateDummyProject() - if self._test_type == ios_constants.TestType.XCUITEST: - pbxproj_plist_obj = plist_util.Plist(self.pbxproj_file_path) - pbxproj_objects = pbxproj_plist_obj.GetPlistField('objects') - settings = pbxproj_objects['XCUITestBundleBuildConfig']['buildSettings'] - settings['CODE_SIGN_IDENTITY'] = bundle_util.GetCodesignIdentity( - self._test_bundle_dir) - if not provisioning_profile_is_file: - settings[ - 'PROVISIONING_PROFILE_SPECIFIER'] = test_bundle_provisioning_profile - else: - profile_obj = provisioning_profile.ProvisiongProfile( - test_bundle_provisioning_profile, - self._work_dir, - keychain_path=self._keychain_path) - profile_obj.Install() - settings['PROVISIONING_PROFILE_SPECIFIER'] = profile_obj.name - pbxproj_plist_obj.SetPlistField('objects', pbxproj_objects) - else: - logging.warning( - 'Setting provisioning profile specifier to test bundle in test type ' - '%s is not supported.', self._test_type) - - def SetEnvVars(self, env_vars): - """Sets the additional environment variables in the dummy project's scheme. - - Args: - env_vars: dict. Both key and value is string. - """ - if not env_vars: - return - self.GenerateDummyProject() - scheme_path = self.test_scheme_path - scheme_tree = ET.parse(scheme_path) - root = scheme_tree.getroot() - test_action_element = root.find('TestAction') - test_action_element.set('shouldUseLaunchSchemeArgsEnv', 'NO') - envs_element = ET.SubElement(test_action_element, 'EnvironmentVariables') - for key, value in env_vars.items(): - env_element = ET.SubElement(envs_element, 'EnvironmentVariable') - env_element.set('key', key) - env_element.set('value', value) - env_element.set('isEnabled', 'YES') - scheme_tree.write(scheme_path) - - def SetArgs(self, args): - """Sets the additional arguments in the dummy project's scheme. - - Args: - args: a list of string. Each item is an argument. - """ - if not args: - return - self.GenerateDummyProject() - scheme_path = self.test_scheme_path - scheme_tree = ET.parse(scheme_path) - test_action_element = scheme_tree.getroot().find('TestAction') - test_action_element.set('shouldUseLaunchSchemeArgsEnv', 'NO') - args_element = ET.SubElement(test_action_element, 'CommandLineArguments') - for arg in args: - arg_element = ET.SubElement(args_element, 'CommandLineArgument') - arg_element.set('argument', arg) - arg_element.set('isEnabled', 'YES') - scheme_tree.write(scheme_path) - - def SetSkipTests(self, skip_tests): - """Sets the skip tests in the dummy project's scheme. - - Args: - skip_tests: a list of string. The format of each item is - Test-Class-Name[/Test-Method-Name]. - """ - if not skip_tests: - return - self.GenerateDummyProject() - scheme_path = self.test_scheme_path - scheme_tree = ET.parse(scheme_path) - test_action_element = scheme_tree.getroot().find('TestAction') - testable_reference_element = test_action_element.find( - 'Testables').find('TestableReference') - skip_tests_element = ET.SubElement( - testable_reference_element, 'SkippedTests') - for skip_test in skip_tests: - skip_test_element = ET.SubElement(skip_tests_element, 'Test') - skip_test_element.set('Identifier', skip_test) - scheme_tree.write(scheme_path) - - -def _GetTestProject(work_dir): - """Gets the TestProject path.""" - test_project_path = os.path.join(work_dir, 'Resource/TestProject') - if os.path.exists(test_project_path): - return test_project_path - - xcodeproj_path = os.path.join(test_project_path, 'TestProject.xcodeproj') - os.makedirs(xcodeproj_path) - with open(os.path.join(xcodeproj_path, 'project.pbxproj'), - 'w+') as target_file: - target_file.write( - pkgutil.get_data('xctestrunner.test_runner', - 'TestProject/TestProject.xcodeproj/project.pbxproj')) - xcschemes_path = os.path.join(xcodeproj_path, 'xcshareddata/xcschemes') - os.makedirs(xcschemes_path) - with open(os.path.join(xcschemes_path, 'TestProjectXctest.xcscheme'), - 'w+') as target_file: - target_file.write( - pkgutil.get_data( - 'xctestrunner.test_runner', - 'TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/' - 'TestProjectXctest.xcscheme')) - with open(os.path.join(xcschemes_path, 'TestProjectXcuitest.xcscheme'), - 'w+') as target_file: - target_file.write( - pkgutil.get_data( - 'xctestrunner.test_runner', - 'TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/' - 'TestProjectXcuitest.xcscheme')) - return test_project_path diff --git a/test_runner/test_summaries_util.py b/test_runner/test_summaries_util.py deleted file mode 100644 index 32e4923..0000000 --- a/test_runner/test_summaries_util.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright 2017 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Helper class for parsing TestSummaries.plist file. -""" - -import glob -import os -import shutil -import tempfile - -from xctestrunner.shared import plist_util - - -def GetTestSummariesPaths(derived_data_dir): - """Get the TestSummaries.plist files under the DerivedData directory.""" - return glob.glob('%s/Logs/Test/*_TestSummaries.plist' % derived_data_dir) - - -def ParseTestSummaries( - test_summaries_path, attachments_dir_path, delete_uitest_auto_screenshots): - """Parse the TestSummaries.plist and structure the attachments' files. - - Only the screenshots file from failure test methods and .crash files will be - stored. Other files will be removed. - - Args: - test_summaries_path: string, the path of TestSummaries.plist file. - attachments_dir_path: string, the path of Attachments directory. - delete_uitest_auto_screenshots: bool, whether deletes the auto screenshots. - """ - test_summaries_plist = plist_util.Plist(test_summaries_path) - tests_obj = test_summaries_plist.GetPlistField('TestableSummaries:0:Tests:0') - # Store the required screenshots and crash files under temp directory first. - # Then use the temp directory to replace the original Attachments directory. - # If delete_uitest_auto_screenshots is true, only move crash files to - # temp directory and the left screenshots will be deleted. - temp_dir = tempfile.mkdtemp(dir=os.path.dirname(attachments_dir_path)) - if not delete_uitest_auto_screenshots: - _ParseTestObject(tests_obj, attachments_dir_path, temp_dir) - for crash_file in glob.glob('%s/*.crash' % attachments_dir_path): - shutil.move(crash_file, temp_dir) - shutil.rmtree(attachments_dir_path) - shutil.move(temp_dir, attachments_dir_path) - - -def _ParseTestObject(test_obj, attachments_dir_path, parent_test_obj_dir_path): - """Parse the test method object and structure its attachment files.""" - test_obj_dir_path = os.path.join( - parent_test_obj_dir_path, - test_obj['TestIdentifier'].replace('.', '_').replace('/', '_')) - if 'Subtests' in test_obj: - # If the test suite only has one sub test, don't create extra folder which - # causes extra directory hierarchy. - if len(test_obj['Subtests']) > 1: - if not os.path.exists(test_obj_dir_path): - os.mkdir(test_obj_dir_path) - else: - test_obj_dir_path = parent_test_obj_dir_path - for sub_test_obj in test_obj['Subtests']: - _ParseTestObject(sub_test_obj, attachments_dir_path, test_obj_dir_path) - return - # Only parse the failure test methods. The succeed test method's attachment - # files will be removed later. - if test_obj['TestStatus'] == 'Success': - return - if not os.path.exists(test_obj_dir_path): - os.mkdir(test_obj_dir_path) - test_result_plist_path = os.path.join(test_obj_dir_path, - 'TestMethodResult.plist') - plist_util.Plist(test_result_plist_path).SetPlistField('', test_obj) - if 'ActivitySummaries' in test_obj: - for test_activity_obj in test_obj['ActivitySummaries']: - _ExploreTestActivity( - test_activity_obj, attachments_dir_path, test_obj_dir_path) - - -def _ExploreTestActivity(test_activity_obj, attachments_dir_path, - test_obj_dir_path): - """Move the screenshot files of this method to test object directory.""" - if 'HasScreenshotData' in test_activity_obj: - screenshot_file_paths = glob.glob( - os.path.join( - attachments_dir_path, - 'Screenshot_%s.*' % test_activity_obj['UUID'])) - for path in screenshot_file_paths: - shutil.move(path, test_obj_dir_path) - if 'SubActivities' in test_activity_obj: - for sub_test_activity_obj in test_activity_obj['SubActivities']: - _ExploreTestActivity( - sub_test_activity_obj, attachments_dir_path, test_obj_dir_path) diff --git a/shared/__init__.py b/xctestrunner/__init__.py similarity index 100% rename from shared/__init__.py rename to xctestrunner/__init__.py diff --git a/xctestrunner/shared/__init__.py b/xctestrunner/shared/__init__.py new file mode 100644 index 0000000..74a2d1e --- /dev/null +++ b/xctestrunner/shared/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/shared/bundle_util.py b/xctestrunner/shared/bundle_util.py similarity index 92% rename from shared/bundle_util.py rename to xctestrunner/shared/bundle_util.py index 324fde8..fc7f8b8 100644 --- a/shared/bundle_util.py +++ b/xctestrunner/shared/bundle_util.py @@ -127,7 +127,7 @@ def GetCodesignIdentity(bundle_path): command = ('codesign', '-dvv', bundle_path) process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - output = process.communicate()[0] + output = process.communicate()[0].decode('utf-8') for line in output.split('\n'): if line.startswith('Authority='): return line[len('Authority='):] @@ -152,7 +152,7 @@ def GetDevelopmentTeam(bundle_path): command = ('codesign', '-dvv', bundle_path) process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - output = process.communicate()[0] + output = process.communicate()[0].decode('utf-8') for line in output.split('\n'): if line.startswith('TeamIdentifier='): return line[len('TeamIdentifier='):] @@ -195,7 +195,8 @@ def CodesignBundle(bundle_path, stderr=subprocess.PIPE) except subprocess.CalledProcessError as e: raise ios_errors.BundleError( - 'Failed to codesign the bundle %s: %s' % (bundle_path, e.output)) + 'Failed to codesign the bundle %s with %s: %s' % + (bundle_path, identity, e.output)) def EnableUIFileSharing(bundle_path, resigning=True): @@ -215,6 +216,19 @@ def EnableUIFileSharing(bundle_path, resigning=True): CodesignBundle(bundle_path) +def GetFileArchTypes(file_path): + """Gets the architecture types of the file.""" + output = subprocess.check_output(['/usr/bin/lipo', file_path, + '-archs']).decode('utf-8').strip() + return output.split(' ') + + +def RemoveArchType(file_path, arch_type): + """Remove the given architecture types for the file.""" + subprocess.check_call( + ['/usr/bin/lipo', file_path, '-remove', arch_type, '-output', file_path]) + + def _ExtractBundleFile(target_dir, bundle_extension): """Extract single bundle file with given extension. diff --git a/shared/ios_constants.py b/xctestrunner/shared/ios_constants.py similarity index 93% rename from shared/ios_constants.py rename to xctestrunner/shared/ios_constants.py index d28d97e..fa95120 100644 --- a/shared/ios_constants.py +++ b/xctestrunner/shared/ios_constants.py @@ -19,6 +19,13 @@ def enum(**enums): return type('Enum', (), enums) +ARCH = enum( + ARMV7='armv7', + ARMV7S='armv7s', + ARM64='arm64', + ARM64E='arm64e', + I386='i386', + X86_64='x86_64') SDK = enum(IPHONEOS='iphoneos', IPHONESIMULATOR='iphonesimulator') # It is consistent with bazel's apple platform: # https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/rules/apple/ApplePlatform.java @@ -37,8 +44,6 @@ def enum(**enums): XCTRUNNER_STARTED_SIGNAL = 'Running tests...' CORESIMULATOR_INTERRUPTED_ERROR = 'CoreSimulatorService connection interrupted' -CORESIMULATOR_CHANGE_ERROR = ('CoreSimulator detected Xcode.app relocation or ' - 'CoreSimulatorService version change.') LAUNCH_OPTIONS_JSON_HELP = ( """The path of json file, which contains options of launching test. @@ -60,6 +65,9 @@ def enum(**enums): In xctest, the functionality is the same as "args". In xcuitest, the process of app under test is different with the process of test. + keep_xcresult_data: bool + Whether or not to keep the xcresult bundle produced by the test run + in the output_dir. tests_to_run : array The specific test classes or test methods to run. Each item should be string and its format is Test-Class-Name[/Test-Method-Name]. It is supported @@ -72,7 +80,7 @@ def enum(**enums): Whether captures screenshots automatically in ui test. If yes, will save the screenshots when the test failed. By default, it is false. Prior Xcode 9, this option does not work and the auto screenshot is enable by default. - startup_timeout_seconds: int + startup_timeout_sec: int Seconds until the xcodebuild command is deemed stuck. destination_timeout_sec: int Wait for the given seconds while searching for the destination device. diff --git a/shared/ios_errors.py b/xctestrunner/shared/ios_errors.py similarity index 93% rename from shared/ios_errors.py rename to xctestrunner/shared/ios_errors.py index 308065e..b70e077 100644 --- a/shared/ios_errors.py +++ b/xctestrunner/shared/ios_errors.py @@ -45,3 +45,8 @@ class SimError(Exception): class XcodebuildTestError(Exception): """Exception class for simulator error.""" + + +class XcresultError(Exception): + """Exception class for parsing xcresult error.""" + diff --git a/shared/plist_util.py b/xctestrunner/shared/plist_util.py similarity index 57% rename from shared/plist_util.py rename to xctestrunner/shared/plist_util.py index bf79b5a..d4faab6 100644 --- a/shared/plist_util.py +++ b/xctestrunner/shared/plist_util.py @@ -14,20 +14,10 @@ """Utility class for managing Plist files.""" -import logging import os import plistlib -import subprocess -import xml.parsers.expat from xctestrunner.shared import ios_errors -try: - import biplist -except ImportError: - biplist = None - - -PLIST_BUDDY = '/usr/libexec/PlistBuddy' class Plist(object): @@ -41,7 +31,6 @@ def __init__(self, plist_file_path): """ self._plist_file_path = plist_file_path # Module to read the .plist file. - self._plistlib_module = _GetPlistLibModule(plist_file_path) def GetPlistField(self, field): """View specific field in the .plist file. @@ -59,9 +48,8 @@ def GetPlistField(self, field): Raises: ios_errors.PlistError: the field does not exist in the plist dict. """ - if self._plistlib_module is None: - return _GetPlistFieldByPlistBuddy(self._plist_file_path, field) - plist_root_object = self._plistlib_module.readPlist(self._plist_file_path) + with open(self._plist_file_path, 'rb') as plist_file: + plist_root_object = plistlib.load(plist_file) return _GetObjectWithField(plist_root_object, field) def HasPlistField(self, field): @@ -98,15 +86,14 @@ def SetPlistField(self, field, value): Raises: ios_errors.PlistError: the field does not exist in the .plist file's dict. """ - if self._plistlib_module is None: - _SetPlistFieldByPlistBuddy(self._plist_file_path, field, value) - return if not field: - self._plistlib_module.writePlist(value, self._plist_file_path) + with open(self._plist_file_path, 'wb') as plist_file: + plistlib.dump(value, plist_file) return if os.path.exists(self._plist_file_path): - plist_root_object = self._plistlib_module.readPlist(self._plist_file_path) + with open(self._plist_file_path, 'rb') as plist_file: + plist_root_object = plistlib.load(plist_file) else: plist_root_object = {} keys_in_field = field.rsplit(':', 1) @@ -123,7 +110,8 @@ def SetPlistField(self, field, value): except (KeyError, IndexError): raise ios_errors.PlistError('Failed to set key %s from object %s.' % (key, target_object)) - self._plistlib_module.writePlist(plist_root_object, self._plist_file_path) + with open(self._plist_file_path, 'wb') as plist_file: + plistlib.dump(plist_root_object, plist_file) def DeletePlistField(self, field): """Delete field in .plist file. @@ -138,11 +126,8 @@ def DeletePlistField(self, field): Raises: ios_errors.PlistError: the field does not exist in the .plist file's dict. """ - if self._plistlib_module is None: - _DeletePlistFieldByPlistBuddy(self._plist_file_path, field) - return - - plist_root_object = self._plistlib_module.readPlist(self._plist_file_path) + with open(self._plist_file_path, 'rb') as plist_file: + plist_root_object = plistlib.load(plist_file) keys_in_field = field.rsplit(':', 1) if len(keys_in_field) == 1: key = field @@ -159,7 +144,8 @@ def DeletePlistField(self, field): raise ios_errors.PlistError('Failed to delete key %s from object %s.' % (key, target_object)) - self._plistlib_module.writePlist(plist_root_object, self._plist_file_path) + with open(self._plist_file_path, 'wb') as plist_file: + plistlib.dump(plist_root_object, plist_file) def _GetObjectWithField(target_object, field): @@ -221,103 +207,3 @@ def _ParseKey(target_object, key): % (key, target_object)) raise ios_errors.PlistError('The object %s is not dict or list.' % target_object) - - -def _GetPlistLibModule(plist_file_path): - """Gets the module to read the target .plist file. - - .plist file has two kinds of format: XML format and binary format. If the - .plist file is XML format, it should be read by plistlib and don't use - biplist which will change XML format plist to binary plist. - - Args: - plist_file_path: string, full path of the .plist file. - - Returns: - a module to read the target .plist file or None if the model can not be - imported. - """ - # If the plist file path does not exist, use plistlib by default. - if not os.path.exists(plist_file_path): - return plistlib - try: - plistlib.readPlist(plist_file_path) - return plistlib - except xml.parsers.expat.ExpatError: - if biplist is None: - logging.info( - 'Failed to import biplist module. Will use tool %s to handle the ' - 'binary format plist.', - PLIST_BUDDY) - return biplist - - -def _GetPlistFieldByPlistBuddy(plist_path, field): - """View specific field in the .plist file by PlistBuddy tool. - - Args: - plist_path: string, the path of plist file. - field: string, the field consist of property key names delimited by - colons. List(array) items are specified by a zero-based integer index. - Examples - :CFBundleShortVersionString - :CFBundleDocumentTypes:2:CFBundleTypeExtensions - - Returns: - the object of the plist's field. - - Raises: - ios_errors.PlistError: the field does not exist in the plist dict. - """ - command = [PLIST_BUDDY, '-c', 'Print :"%s"' % field, plist_path] - try: - return subprocess.check_output(command, stderr=subprocess.STDOUT).strip() - except subprocess.CalledProcessError as e: - raise ios_errors.PlistError( - 'Failed to get field %s in plist %s: %s', field, plist_path, e.output) - - -def _SetPlistFieldByPlistBuddy(plist_path, field, value): - """Set field with provided value in .plist file. - - Args: - plist_path: string, the path of plist file. - field: string, the field consist of property key names delimited by - colons. List(array) items are specified by a zero-based integer index. - Examples - :CFBundleShortVersionString - :CFBundleDocumentTypes:2:CFBundleTypeExtensions - value: a object, the value of the field to be added. It can be integer, - bool, string, array, dict. - - Raises: - ios_errors.PlistError: the field does not exist in the .plist file's dict. - """ - command = [PLIST_BUDDY, '-c', 'Set :"%s" "%s"' % (field, value), plist_path] - try: - subprocess.check_output(command, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - raise ios_errors.PlistError('Failed to set field %s in plist %s: %s' - % (field, plist_path, e.output)) - - -def _DeletePlistFieldByPlistBuddy(plist_path, field): - """Delete field in .plist file. - - Args: - plist_path: string, the path of plist file. - field: string, the field consist of property key names delimited by - colons. List(array) items are specified by a zero-based integer index. - Examples - :CFBundleShortVersionString - :CFBundleDocumentTypes:2:CFBundleTypeExtensions - - Raises: - ios_errors.PlistError: the field does not exist in the .plist file's dict. - """ - command = [PLIST_BUDDY, '-c', 'Delete :"%s"' % field, plist_path] - try: - subprocess.check_output(command, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - raise ios_errors.PlistError('Failed to delete field %s in plist %s: %s' - % (field, plist_path, e.output)) diff --git a/shared/provisioning_profile.py b/xctestrunner/shared/provisioning_profile.py similarity index 98% rename from shared/provisioning_profile.py rename to xctestrunner/shared/provisioning_profile.py index bddb9a4..82062b4 100644 --- a/shared/provisioning_profile.py +++ b/xctestrunner/shared/provisioning_profile.py @@ -88,7 +88,7 @@ def _DecodeProvisioningProfile(self): command.extend(['-k', self._keychain_path]) process = subprocess.Popen( command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - output = process.communicate() + output = process.communicate()[0].decode('utf-8') if process.poll() != 0: raise ios_errors.ProvisioningProfileError(output) if not os.path.exists(decode_provisioning_profile): diff --git a/xctestrunner/shared/version_util.py b/xctestrunner/shared/version_util.py new file mode 100644 index 0000000..a6a5e65 --- /dev/null +++ b/xctestrunner/shared/version_util.py @@ -0,0 +1,26 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utility methods for Apple version.""" + + +def GetVersionNumber(version_str): + """Gets the version number of the given version string.""" + parts = version_str.split('.') + version_number = int(parts[0]) * 100 + if len(parts) > 1: + version_number += int(parts[1]) * 10 + if len(parts) > 2: + version_number += int(parts[2]) + return version_number diff --git a/shared/xcode_info_util.py b/xctestrunner/shared/xcode_info_util.py similarity index 58% rename from shared/xcode_info_util.py rename to xctestrunner/shared/xcode_info_util.py index 134d73d..2333e7a 100644 --- a/shared/xcode_info_util.py +++ b/xctestrunner/shared/xcode_info_util.py @@ -17,13 +17,16 @@ import os import subprocess +from xctestrunner.shared import ios_constants +from xctestrunner.shared import version_util + _xcode_version_number = None def GetXcodeDeveloperPath(): """Gets the active developer path of Xcode command line tools.""" - return subprocess.check_output(('xcode-select', '-p')).strip() + return subprocess.check_output(('xcode-select', '-p')).decode('utf-8').strip() def GetXcodeVersionNumber(): @@ -41,30 +44,42 @@ def GetXcodeVersionNumber(): # Example output: # Xcode 8.2.1 # Build version 8C1002 - output = subprocess.check_output(('xcodebuild', '-version')) + output = subprocess.check_output(('xcodebuild', '-version')).decode('utf-8') xcode_version = output.split('\n')[0].split(' ')[1] - parts = xcode_version.split('.') - xcode_version_number = int(parts[0]) * 100 - if len(parts) > 1: - xcode_version_number += int(parts[1]) * 10 - if len(parts) > 2: - xcode_version_number += int(parts[2]) # Add cache xcode_version_number to avoid calling subprocess multiple times. # It is expected that no one changes xcode during the test runner working. - _xcode_version_number = xcode_version_number + _xcode_version_number = version_util.GetVersionNumber(xcode_version) return _xcode_version_number +# Xcode 11+'s Swift dylibs are configured in a way that does not allow them to +# load the correct libswiftFoundation.dylib file from +# libXCTestSwiftSupport.dylib. This bug only affects tests that run on fallbacks +# to the correct Swift dylibs that have been packaged with Xcode. This method +# returns the path to that fallback directory. +# See https://github.com/bazelbuild/rules_apple/issues/684 for context. +def GetSwift5FallbackLibsDir(): + """Gets the Swift5 fallback libraries directory.""" + relative_path = 'Toolchains/XcodeDefault.xctoolchain/usr/lib/swift-5.0' + swift_libs_dir = os.path.join(GetXcodeDeveloperPath(), relative_path) + swift_lib_platform_dir = os.path.join(swift_libs_dir, + ios_constants.SDK.IPHONESIMULATOR) + if os.path.exists(swift_lib_platform_dir): + return swift_lib_platform_dir + return None + + def GetSdkPlatformPath(sdk): """Gets the selected SDK platform path.""" return subprocess.check_output( - ['xcrun', '--sdk', sdk, '--show-sdk-platform-path']).strip() + ['xcrun', '--sdk', sdk, + '--show-sdk-platform-path']).decode('utf-8').strip() def GetSdkVersion(sdk): """Gets the selected SDK version.""" - return subprocess.check_output( - ['xcrun', '--sdk', sdk, '--show-sdk-version']).strip() + return subprocess.check_output(['xcrun', '--sdk', sdk, '--show-sdk-version' + ]).decode('utf-8').strip() def GetXctestToolPath(sdk): @@ -75,7 +90,8 @@ def GetXctestToolPath(sdk): def GetDarwinUserCacheDir(): """Gets the path of Darwin user cache directory.""" - return subprocess.check_output(('getconf', 'DARWIN_USER_CACHE_DIR')).rstrip() + return subprocess.check_output( + ('getconf', 'DARWIN_USER_CACHE_DIR')).decode('utf-8').rstrip() def GetXcodeEmbeddedAppDeltasDir(): diff --git a/xctestrunner/simulator_control/__init__.py b/xctestrunner/simulator_control/__init__.py new file mode 100644 index 0000000..74a2d1e --- /dev/null +++ b/xctestrunner/simulator_control/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/simulator_control/simtype_profile.py b/xctestrunner/simulator_control/simtype_profile.py similarity index 69% rename from simulator_control/simtype_profile.py rename to xctestrunner/simulator_control/simtype_profile.py index 1b51cb8..944ada4 100644 --- a/simulator_control/simtype_profile.py +++ b/xctestrunner/simulator_control/simtype_profile.py @@ -47,13 +47,12 @@ def profile_plist_obj(self): profile.plist. """ if not self._profile_plist_obj: - if xcode_info_util.GetXcodeVersionNumber() >= 900: - platform_path = xcode_info_util.GetSdkPlatformPath( - ios_constants.SDK.IPHONEOS) - else: - platform_path = xcode_info_util.GetSdkPlatformPath( - ios_constants.SDK.IPHONESIMULATOR) - if xcode_info_util.GetXcodeVersionNumber() >= 1100: + xcode_version = xcode_info_util.GetXcodeVersionNumber() + platform_path = xcode_info_util.GetSdkPlatformPath( + ios_constants.SDK.IPHONEOS) + if xcode_version >= 1630: + sim_profiles_dir = '/Library/Developer/CoreSimulator/Profiles' + elif xcode_version >= 1100: sim_profiles_dir = os.path.join( platform_path, 'Library/Developer/CoreSimulator/Profiles') else: @@ -71,14 +70,12 @@ def min_os_version(self): """Gets the min supported OS version. Returns: - string, the min supported OS version. + float, the min supported OS version. """ if not self._min_os_version: - min_os_version = self.profile_plist_obj.GetPlistField('minRuntimeVersion') - # Cut build version. E.g., cut 9.3.3 to 9.3. - if min_os_version.count('.') > 1: - min_os_version = min_os_version[:min_os_version.rfind('.')] - self._min_os_version = min_os_version + min_os_version_str = self.profile_plist_obj.GetPlistField( + 'minRuntimeVersion') + self._min_os_version = _extra_os_version(min_os_version_str) return self._min_os_version @property @@ -86,19 +83,26 @@ def max_os_version(self): """Gets the max supported OS version. Returns: - string, the max supported OS version. + float, the max supported OS version or None if it is not found. """ if not self._max_os_version: # If the profile.plist does not have maxRuntimeVersion field, it means # it supports the max OS version of current iphonesimulator platform. try: - max_os_version = self.profile_plist_obj.GetPlistField( + max_os_version_str = self.profile_plist_obj.GetPlistField( 'maxRuntimeVersion') except ios_errors.PlistError: - max_os_version = xcode_info_util.GetSdkVersion( - ios_constants.SDK.IPHONESIMULATOR) - # Cut build version. E.g., cut 9.3.3 to 9.3. - if max_os_version.count('.') > 1: - max_os_version = max_os_version[:max_os_version.rfind('.')] - self._max_os_version = max_os_version + return None + self._max_os_version = _extra_os_version(max_os_version_str) return self._max_os_version + + +def _extra_os_version(os_version_str): + """Extracts os version float value from a given string.""" + # Cut build version. E.g., cut 9.3.3 to 9.3. + if os_version_str.count('.') > 1: + os_version_str = os_version_str[:os_version_str.rfind('.')] + # We need to round the os version string in the simulator profile. E.g., + # the maxRuntimeVersion of iPhone 5 is 10.255.255 and we could create iOS 10.3 + # for iPhone 5. + return round(float(os_version_str), 1) diff --git a/simulator_control/simulator_util.py b/xctestrunner/simulator_control/simulator_util.py similarity index 85% rename from simulator_control/simulator_util.py rename to xctestrunner/simulator_control/simulator_util.py index 729e370..b2014c6 100644 --- a/simulator_control/simulator_util.py +++ b/xctestrunner/simulator_control/simulator_util.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """The utility class for simulator.""" import json @@ -38,6 +37,7 @@ _SIM_OPERATION_MAX_ATTEMPTS = 3 _SIMCTL_MAX_ATTEMPTS = 2 _SIMULATOR_CREATING_TO_SHUTDOWN_TIMEOUT_SEC = 10 +_SIMULATOR_BOOTED_TIMEOUT_SEC = 10 _SIMULATOR_SHUTDOWN_TIMEOUT_SEC = 30 _SIM_ERROR_RETRY_INTERVAL_SEC = 2 _SIM_CHECK_STATE_INTERVAL_SEC = 0.5 @@ -113,6 +113,28 @@ def device_plist_object(self): self._device_plist_object = plist_util.Plist(device_plist_path) return self._device_plist_object + def Boot(self): + """Boots the simulator as asynchronously. + + Returns: + A subprocess.Popen object of the boot process. + """ + RunSimctlCommand(['xcrun', 'simctl', 'boot', self.simulator_id]) + self.WaitUntilStateBooted() + logging.info('The simulator %s is booted.', self.simulator_id) + + def BootStatus(self): + """Monitor the simulator boot status asynchronously. + + Returns: + A subprocess.Popen object of the boot status process. + """ + return subprocess.Popen( + ['xcrun', 'simctl', 'bootstatus', self.simulator_id, '-b'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding='utf-8') + def Shutdown(self): """Shuts down the simulator.""" sim_state = self.GetSimulatorState() @@ -134,28 +156,32 @@ def Shutdown(self): self.WaitUntilStateShutdown() logging.info('Shut down simulator %s.', self.simulator_id) - def Delete(self): - """Deletes the simulator asynchronously. + def Delete(self, asynchronously=True): + """Deletes the simulator. The simulator state should be SHUTDOWN when deleting it. Otherwise, it will raise exception. + Args: + asynchronously: whether deleting the simulator asynchronously. Raises: ios_errors.SimError: The simulator's state is not SHUTDOWN. """ - # In Xcode 9+, simctl can delete Booted simulator. In prior of Xcode 9, - # we have to shutdown the simulator first before deleting it. - if xcode_info_util.GetXcodeVersionNumber() < 900: - sim_state = self.GetSimulatorState() - if sim_state != ios_constants.SimState.SHUTDOWN: - raise ios_errors.SimError( - 'Can only delete the simulator with state SHUTDOWN. The current ' - 'state of simulator %s is %s.' % (self._simulator_id, sim_state)) - logging.info('Deleting simulator %s asynchronously.', self.simulator_id) - subprocess.Popen(['xcrun', 'simctl', 'delete', self.simulator_id], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - preexec_fn=os.setpgrp) + command = ['xcrun', 'simctl', 'delete', self.simulator_id] + if asynchronously: + logging.info('Deleting simulator %s asynchronously.', self.simulator_id) + subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=os.setpgrp) + else: + try: + RunSimctlCommand(command) + logging.info('Deleted simulator %s.', self.simulator_id) + except ios_errors.SimError as e: + raise ios_errors.SimError('Failed to delete simulator %s: %s' % + (self.simulator_id, str(e))) # The delete command won't delete the simulator log directory. if os.path.exists(self.simulator_log_root_dir): shutil.rmtree(self.simulator_log_root_dir, ignore_errors=True) @@ -186,17 +212,16 @@ def FetchLogToFile(self, output_file_path, start_time=None, end_time=None): def GetAppDocumentsPath(self, app_bundle_id): """Gets the path of the app's Documents directory.""" - if xcode_info_util.GetXcodeVersionNumber() >= 830: - try: - app_data_container = RunSimctlCommand([ - 'xcrun', 'simctl', 'get_app_container', self._simulator_id, - app_bundle_id, 'data' - ]) - return os.path.join(app_data_container, 'Documents') - except ios_errors.SimError as e: - raise ios_errors.SimError( - 'Failed to get data container of the app %s in simulator %s: %s' % - (app_bundle_id, self._simulator_id, str(e))) + try: + app_data_container = RunSimctlCommand([ + 'xcrun', 'simctl', 'get_app_container', self._simulator_id, + app_bundle_id, 'data' + ]) + return os.path.join(app_data_container, 'Documents') + except ios_errors.SimError as e: + raise ios_errors.SimError( + 'Failed to get data container of the app %s in simulator %s: %s' % + (app_bundle_id, self._simulator_id, str(e))) apps_dir = os.path.join(self.simulator_root_dir, 'data/Containers/Data/Application') @@ -223,6 +248,25 @@ def IsAppInstalled(self, app_bundle_id): except ios_errors.SimError: return False + def WaitUntilStateBooted(self, timeout_sec=_SIMULATOR_BOOTED_TIMEOUT_SEC): + """Waits until the simulator state becomes BOOTED. + + Args: + timeout_sec: int, timeout of waiting simulator state for becoming BOOTED + in seconds. + + Raises: + ios_errors.SimError: when it is timeout to wait the simulator state + becomes BOOTED. + """ + start_time = time.time() + while start_time + timeout_sec >= time.time(): + time.sleep(_SIM_CHECK_STATE_INTERVAL_SEC) + if self.GetSimulatorState() == ios_constants.SimState.BOOTED: + return + raise ios_errors.SimError('Timeout to wait for simulator booted in %ss.' % + timeout_sec) + def WaitUntilStateShutdown(self, timeout_sec=_SIMULATOR_SHUTDOWN_TIMEOUT_SEC): """Waits until the simulator state becomes SHUTDOWN. @@ -236,9 +280,9 @@ def WaitUntilStateShutdown(self, timeout_sec=_SIMULATOR_SHUTDOWN_TIMEOUT_SEC): """ start_time = time.time() while start_time + timeout_sec >= time.time(): + time.sleep(_SIM_CHECK_STATE_INTERVAL_SEC) if self.GetSimulatorState() == ios_constants.SimState.SHUTDOWN: return - time.sleep(_SIM_CHECK_STATE_INTERVAL_SEC) raise ios_errors.SimError('Timeout to wait for simulator shutdown in %ss.' % timeout_sec) @@ -413,9 +457,8 @@ def GetLastSupportedIphoneSimType(os_version): os_version_float = float(os_version) for sim_type in supported_sim_types: if sim_type.startswith('iPhone'): - min_os_version_float = float( - simtype_profile.SimTypeProfile(sim_type).min_os_version) - if os_version_float >= min_os_version_float: + min_os_version = simtype_profile.SimTypeProfile(sim_type).min_os_version + if os_version_float >= min_os_version: return sim_type raise ios_errors.SimError('Can not find supported iPhone simulator type.') @@ -520,17 +563,19 @@ def GetLastSupportedSimOsVersion(os_type=ios_constants.OS.IOS, if not device_type: return supported_os_versions[-1] - simtype_max_os_version_float = float( - simtype_profile.SimTypeProfile(device_type).max_os_version) + max_os_version = simtype_profile.SimTypeProfile(device_type).max_os_version + # The supported os versions will be from latest to older after reverse(). supported_os_versions.reverse() + if not max_os_version: + return supported_os_versions[0] + for os_version in supported_os_versions: - if float(os_version) <= simtype_max_os_version_float: + if float(os_version) <= max_os_version: return os_version - if not supported_os_versions: - raise ios_errors.IllegalArgumentError( - 'The supported OS version %s can not match simulator type %s. Because ' - 'its max OS version is %s' % - (supported_os_versions, device_type, simtype_max_os_version_float)) + raise ios_errors.IllegalArgumentError( + 'The supported OS version %s can not match simulator type %s. Because ' + 'its max OS version is %s' % + (supported_os_versions, device_type, max_os_version)) def GetOsType(device_type): @@ -598,25 +643,22 @@ def _ValidateSimulatorTypeWithOsVersion(device_type, os_version): """ os_version_float = float(os_version) sim_profile = simtype_profile.SimTypeProfile(device_type) - min_os_version_float = float(sim_profile.min_os_version) - if min_os_version_float > os_version_float: - raise ios_errors.IllegalArgumentError( - 'The min OS version of %s is %s. But current OS version is %s' % - (device_type, min_os_version_float, os_version)) - max_os_version_float = float(sim_profile.max_os_version) - if max_os_version_float < os_version_float: + min_os_version = sim_profile.min_os_version + if min_os_version > os_version_float: raise ios_errors.IllegalArgumentError( - 'The max OS version of %s is %s. But current OS version is %s' % - (device_type, max_os_version_float, os_version)) + 'The min OS version of %s is %f. But current OS version is %s' % + (device_type, min_os_version, os_version)) + max_os_version = sim_profile.max_os_version + if max_os_version: + if max_os_version < os_version_float: + raise ios_errors.IllegalArgumentError( + 'The max OS version of %s is %f. But current OS version is %s' % + (device_type, max_os_version, os_version)) def QuitSimulatorApp(): """Quits the Simulator.app.""" - if xcode_info_util.GetXcodeVersionNumber() >= 700: - simulator_name = 'Simulator' - else: - simulator_name = 'iOS Simulator' - subprocess.Popen(['killall', simulator_name], + subprocess.Popen(['killall', 'Simulator'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) @@ -668,16 +710,16 @@ def RunSimctlCommand(command): """Runs simctl command.""" for i in range(_SIMCTL_MAX_ATTEMPTS): process = subprocess.Popen( - command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding='utf-8') stdout, stderr = process.communicate() - if ios_constants.CORESIMULATOR_CHANGE_ERROR in stderr: - output = stdout - else: - output = '\n'.join([stdout, stderr]) - output = output.strip() + all_output = '\n'.join([stdout, stderr]) + output = stdout.strip() if process.poll() != 0: if (i < (_SIMCTL_MAX_ATTEMPTS - 1) and - ios_constants.CORESIMULATOR_INTERRUPTED_ERROR in output): + ios_constants.CORESIMULATOR_INTERRUPTED_ERROR in all_output): continue raise ios_errors.SimError(output) return output diff --git a/xctestrunner/test_runner/__init__.py b/xctestrunner/test_runner/__init__.py new file mode 100644 index 0000000..74a2d1e --- /dev/null +++ b/xctestrunner/test_runner/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test_runner/ios_test_runner.py b/xctestrunner/test_runner/ios_test_runner.py similarity index 73% rename from test_runner/ios_test_runner.py rename to xctestrunner/test_runner/ios_test_runner.py index f2a2f81..c85800d 100644 --- a/test_runner/ios_test_runner.py +++ b/xctestrunner/test_runner/ios_test_runner.py @@ -27,7 +27,6 @@ from xctestrunner.shared import ios_constants from xctestrunner.shared import ios_errors -from xctestrunner.shared import xcode_info_util from xctestrunner.simulator_control import simulator_util from xctestrunner.test_runner import runner_exit_codes from xctestrunner.test_runner import xctest_session @@ -102,13 +101,53 @@ def _AddGeneralArguments(parser): 'test ends.') +def _AddPrepareSubParser(subparsers): + """Adds sub parser for sub command `prepare`.""" + def _Prepare(args): + """The function of sub command `prepare`.""" + sdk = _PlatformToSdk(args.platform) if args.platform else _GetSdk(args.id) + device_arch = args.arch + with xctest_session.XctestSession( + sdk=sdk, + device_arch=device_arch, + work_dir=args.work_dir, + output_dir=args.output_dir) as session: + session.Prepare( + app_under_test=args.app_under_test_path, + test_bundle=args.test_bundle_path, + xctestrun_file_path=args.xctestrun, + test_type=args.test_type, + signing_options=_GetJson(args.signing_options_json_path)) + session.SetLaunchOptions(_GetJson(args.launch_options_json_path)) + + test_parser = subparsers.add_parser( + 'prepare', + help='Prepare the working directory to run the test.') + required_arguments = test_parser.add_argument_group('Required arguments') + required_arguments.add_argument( + '--platform', + help='The platform of the device. The value can be ios_device or ' + 'ios_simulator.' + ) + required_arguments.add_argument( + '--arch', + help='The architecture of the device. The value can be x86_64, armv7,' + 'arm64, arm64e' + ) + test_parser.set_defaults(func=_Prepare) + + def _AddTestSubParser(subparsers): """Adds sub parser for sub command `test`.""" def _Test(args): """The function of sub command `test`.""" sdk = _PlatformToSdk(args.platform) if args.platform else _GetSdk(args.id) + device_arch = _GetDeviceArch(args.id, sdk) with xctest_session.XctestSession( - sdk=sdk, work_dir=args.work_dir, output_dir=args.output_dir) as session: + sdk=sdk, + device_arch=device_arch, + work_dir=args.work_dir, + output_dir=args.output_dir) as session: session.Prepare( app_under_test=args.app_under_test_path, test_bundle=args.test_bundle_path, @@ -142,67 +181,34 @@ def _RunSimulatorTest(args): """The function of running test with new simulator.""" with xctest_session.XctestSession( sdk=ios_constants.SDK.IPHONESIMULATOR, + device_arch=ios_constants.ARCH.X86_64, work_dir=args.work_dir, output_dir=args.output_dir) as session: - session.Prepare( - app_under_test=args.app_under_test_path, - test_bundle=args.test_bundle_path, - xctestrun_file_path=args.xctestrun, - test_type=args.test_type, - signing_options=_GetJson(args.signing_options_json_path)) - session.SetLaunchOptions(_GetJson(args.launch_options_json_path)) - - # In prior of Xcode 9, `xcodebuild test` will launch the Simulator.app - # process. If there is Simulator.app before running test, it will cause - # error later. - if xcode_info_util.GetXcodeVersionNumber() < 900: - simulator_util.QuitSimulatorApp() - max_attempts = 3 - reboot_sim = False - for i in range(max_attempts): - if not reboot_sim: - simulator_id, _, _, _ = simulator_util.CreateNewSimulator( - device_type=args.device_type, os_version=args.os_version, - name_prefix=args.new_simulator_name_prefix) - reboot_sim = False - - try: - # Don't use command "{Xcode_developer_dir}Applications/ \ - # Simulator.app/Contents/MacOS/Simulator" to launch the Simulator.app. - # 1) `xcodebuild test` will handle the launch Simulator. - # 2) If there are two Simulator.app processes launched by command line - # and `xcodebuild test` starts to run on one of Simulator, the another - # Simulator.app will popup 'Unable to boot device in current state: \ - # Booted' dialog and may cause potential error. - exit_code = session.RunTest(simulator_id) - if i < max_attempts - 1: - if exit_code == runner_exit_codes.EXITCODE.NEED_RECREATE_SIM: - logging.warning( - 'Will create a new simulator to retry running test.') - continue - if exit_code == runner_exit_codes.EXITCODE.NEED_REBOOT_DEVICE: - reboot_sim = True - logging.warning( - 'Will reboot the simulator to retry running test.') - continue - return exit_code - finally: - # 1. In prior of Xcode 9, `xcodebuild test` will launch the - # Simulator.app process. Quit the Simulator.app to avoid side effect. - # 2. Quit Simulator.app can also shutdown the simulator. To make sure - # the Simulator state to be SHUTDOWN, still call shutdown command - # later. - if xcode_info_util.GetXcodeVersionNumber() < 900: - simulator_util.QuitSimulatorApp() - simulator_obj = simulator_util.Simulator(simulator_id) - if reboot_sim: - simulator_obj.Shutdown() - else: - # In Xcode 9+, simctl can delete the Booted simulator. - # In prior of Xcode 9, we have to shutdown the simulator first - # before deleting it. - if xcode_info_util.GetXcodeVersionNumber() < 900: - simulator_obj.Shutdown() - simulator_obj.Delete() + simulator_id, _, os_version, _ = simulator_util.CreateNewSimulator( + device_type=args.device_type, + os_version=args.os_version, + name_prefix=args.new_simulator_name_prefix) + simulator_obj = simulator_util.Simulator(simulator_id) + hostless = args.app_under_test_path is None + try: + if not hostless: + simulator_obj.Boot() + session.Prepare( + app_under_test=args.app_under_test_path, + test_bundle=args.test_bundle_path, + xctestrun_file_path=args.xctestrun, + test_type=args.test_type, + signing_options=_GetJson(args.signing_options_json_path)) + session.SetLaunchOptions(_GetJson(args.launch_options_json_path)) + if not hostless: + try: + simulator_obj.BootStatus().wait(timeout=60) + except subprocess.TimeoutExpired: + logging.warning( + 'The simulator %s could not be booted in 60s. Will try to run ' + 'test directly.', simulator_id) + return session.RunTest(simulator_id, os_version=os_version) + finally: + simulator_obj.Delete() def _SimulatorTest(args): """The function of sub command `simulator_test`.""" @@ -245,6 +251,7 @@ def _BuildParser(): formatter_class=argparse.RawTextHelpFormatter) _AddGeneralArguments(parser) subparsers = parser.add_subparsers(help='Sub-commands help') + _AddPrepareSubParser(subparsers) _AddTestSubParser(subparsers) _AddSimulatorTestSubParser(subparsers) return parser @@ -274,23 +281,27 @@ def _PlatformToSdk(platform): def _GetSdk(device_id): """Gets the sdk of the target device with the given device_id.""" - # The command `instruments -s devices` is much slower than - # `xcrun simctl list devices`. So use `xcrun simctl list devices` to check - # IPHONESIMULATOR SDK first. - simlist_devices_output = simulator_util.RunSimctlCommand( - ['xcrun', 'simctl', 'list', 'devices']) - if device_id in simlist_devices_output: - return ios_constants.SDK.IPHONESIMULATOR - - known_devices_output = subprocess.check_output( - ['instruments', '-s', 'devices']) - for line in known_devices_output.split('\n'): - if device_id in line and '(Simulator)' not in line: - return ios_constants.SDK.IPHONEOS + devices_list_output = subprocess.check_output( + ['xcrun', 'xcdevice', 'list']).decode('utf-8') + for device_info in json.loads(devices_list_output): + if device_info['identifier'] == device_id: + return ios_constants.SDK.IPHONESIMULATOR if device_info[ + 'simulator'] else ios_constants.SDK.IPHONEOS raise ios_errors.IllegalArgumentError( 'The device with id %s can not be found. The known devices are %s.' % - (device_id, known_devices_output)) + (device_id, devices_list_output)) + + +def _GetDeviceArch(device_id, sdk): + """Gets the device architecture.""" + # It is a temporary soluton to get device architecture. Checking i386 and + # armv7/armv7s is not supported. + if sdk == ios_constants.SDK.IPHONESIMULATOR: + return ios_constants.ARCH.X86_64 + if '-' in device_id: + return ios_constants.ARCH.ARM64E + return ios_constants.ARCH.ARM64 def main(argv): diff --git a/test_runner/logic_test_util.py b/xctestrunner/test_runner/logic_test_util.py similarity index 68% rename from test_runner/logic_test_util.py rename to xctestrunner/test_runner/logic_test_util.py index 9021d8f..c464e76 100644 --- a/test_runner/logic_test_util.py +++ b/xctestrunner/test_runner/logic_test_util.py @@ -14,18 +14,24 @@ """The helper classes to run logic test.""" +import os import subprocess import sys from xctestrunner.shared import ios_constants +from xctestrunner.shared import version_util from xctestrunner.shared import xcode_info_util from xctestrunner.test_runner import runner_exit_codes _SIMCTL_ENV_VAR_PREFIX = 'SIMCTL_CHILD_' -def RunLogicTestOnSim( - sim_id, test_bundle_path, env_vars=None, args=None, tests_to_run=None): +def RunLogicTestOnSim(sim_id, + test_bundle_path, + env_vars=None, + args=None, + tests_to_run=None, + os_version=None): """Runs logic tests on the simulator. The output prints on system stdout. Args: @@ -36,6 +42,7 @@ def RunLogicTestOnSim( args: array, the additional arguments passing to test's process. tests_to_run: array, the format of each item is TestClass[/TestMethod]. If it is empty, then runs with All methods. + os_version: string, the OS version of the simulator. Returns: exit_code: A value of type runner_exit_codes.EXITCODE. @@ -48,6 +55,18 @@ def RunLogicTestOnSim( for key in env_vars: simctl_env_vars[_SIMCTL_ENV_VAR_PREFIX + key] = env_vars[key] simctl_env_vars['NSUnbufferedIO'] = 'YES' + # When running tests on iOS 12.1 or earlier simulator under Xcode 11 or later, + # it is required to add swift5 fallback libraries to environment variable. + # See https://github.com/bazelbuild/rules_apple/issues/684 for context. + if (xcode_info_util.GetXcodeVersionNumber() >= 1100 and + os_version and + version_util.GetVersionNumber(os_version) < 1220): + key = _SIMCTL_ENV_VAR_PREFIX + 'DYLD_FALLBACK_LIBRARY_PATH' + simctl_env_vars[key] = xcode_info_util.GetSwift5FallbackLibsDir() + # We need to set the DEVELOPER_DIR to ensure xcrun works correctly + developer_dir = os.environ.get('DEVELOPER_DIR') + if developer_dir: + simctl_env_vars['DEVELOPER_DIR'] = developer_dir command = [ 'xcrun', 'simctl', 'spawn', '-s', sim_id, xcode_info_util.GetXctestToolPath(ios_constants.SDK.IPHONESIMULATOR)] diff --git a/test_runner/runner_exit_codes.py b/xctestrunner/test_runner/runner_exit_codes.py similarity index 100% rename from test_runner/runner_exit_codes.py rename to xctestrunner/test_runner/runner_exit_codes.py diff --git a/test_runner/xcodebuild_test_executor.py b/xctestrunner/test_runner/xcodebuild_test_executor.py similarity index 86% rename from test_runner/xcodebuild_test_executor.py rename to xctestrunner/test_runner/xcodebuild_test_executor.py index 4665db4..db987c6 100644 --- a/test_runner/xcodebuild_test_executor.py +++ b/xctestrunner/test_runner/xcodebuild_test_executor.py @@ -21,7 +21,6 @@ import re import shutil import subprocess -import sys import threading import time @@ -49,6 +48,10 @@ _TOO_MANY_INSTANCES_ALREADY_RUNNING = ('Too many instances of this service are ' 'already running.') _LOST_CONNECTION_ERROR = 'Lost connection to testmanagerd' +_LOST_CONNECTION_TO_DTSERVICEHUB_ERROR = 'Lost connection to DTServiceHub' +_DEVICE_NO_LONGER_CONNECTED = 'This device is no longer connected' +_UNABLE_FIND_DEVICE_IDENTIFIER = 'Unable to find device with identifier' +_BUNDLE_DAMAGED = 'The bundle is damaged or missing necessary resources.' class CheckXcodebuildStuckThread(threading.Thread): @@ -129,11 +132,12 @@ def __init__(self, self._startup_timeout_sec = ( startup_timeout_sec or _XCODEBUILD_TEST_STARTUP_TIMEOUT_SEC) - def Execute(self, return_output=True): + def Execute(self, return_output=True, result_bundle_path=None): """Executes the xcodebuild test command. Args: return_output: bool, whether save output in the execution result. + result_bundle_path: string, path of the result bundle. Returns: a tuple of two fields: @@ -158,14 +162,16 @@ def Execute(self, return_output=True): test_failed = False for i in range(max_attempts): + if result_bundle_path and os.path.exists(result_bundle_path): + shutil.rmtree(result_bundle_path, ignore_errors=True) process = subprocess.Popen( self._command, env=run_env, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, encoding='ascii', errors='ignore') check_xcodebuild_stuck = CheckXcodebuildStuckThread( process, self._startup_timeout_sec) check_xcodebuild_stuck.start() - output = io.BytesIO() - for stdout_line in iter(process.stdout.readline, ''): + output = io.StringIO() + for stdout_line in process.stdout: if not test_started: # Terminates the CheckXcodebuildStuckThread when test has started # or XCTRunner.app has started. @@ -187,8 +193,7 @@ def Execute(self, return_output=True): if self._failed_signal and self._failed_signal in stdout_line: test_failed = True - sys.stdout.write(stdout_line) - sys.stdout.flush() + print(stdout_line, flush=True, end='') # If return_output is false, the output is only used for checking error # cause and deleting cached files (_DeleteTestCacheFileDirs method). if return_output or not test_started: @@ -206,14 +211,32 @@ def Execute(self, return_output=True): check_xcodebuild_stuck.Terminate() if check_xcodebuild_stuck.is_xcodebuild_stuck: + if self._sdk == ios_constants.SDK.IPHONESIMULATOR and i < max_attempts - 1: + logging.warning( + 'xcodebuild stuck on simulator (attempt %d/%d). ' + 'Will reboot simulator and retry.', + i + 1, max_attempts) + simulator = simulator_util.Simulator(self._device_id) + simulator.Shutdown() + simulator.Boot() + time.sleep(2) + continue return self._GetResultForXcodebuildStuck(output, return_output) output_str = output.getvalue() + # Don't need to retry the case for the damaged test bundle. + if _BUNDLE_DAMAGED in output_str: + return (runner_exit_codes.EXITCODE.TEST_NOT_START, + output_str if return_output else None) + if self._sdk == ios_constants.SDK.IPHONEOS: - if ((re.search(_DEVICE_TYPE_WAS_NULL_PATTERN, output_str) or - _LOST_CONNECTION_ERROR in output_str) and i < max_attempts - 1): + if (self._NeedRetryForDeviceTesting(output_str) and + i < max_attempts - 1): logging.warning( - 'Failed to launch test on the device. Will relaunch again.') + 'Failed to launch test on the device. Will relaunch again ' + 'after 5s.' + ) + time.sleep(5) continue if _TOO_MANY_INSTANCES_ALREADY_RUNNING in output_str: return (runner_exit_codes.EXITCODE.NEED_REBOOT_DEVICE, @@ -292,6 +315,14 @@ def _NeedRecreateSim(self, output_str): return True return False + def _NeedRetryForDeviceTesting(self, output_str): + """Returns true if the device testing needs retry.""" + return (re.search(_DEVICE_TYPE_WAS_NULL_PATTERN, output_str) or + _LOST_CONNECTION_ERROR in output_str or + _LOST_CONNECTION_TO_DTSERVICEHUB_ERROR in output_str or + _DEVICE_NO_LONGER_CONNECTED in output_str or + _UNABLE_FIND_DEVICE_IDENTIFIER in output_str) + def _DeleteTestCacheFileDirs(xcodebuild_test_output, sdk, test_type): """Deletes the cache files of the test session according to arguments.""" @@ -340,4 +371,5 @@ def _FetchTestCacheFileDirs(xcodebuild_test_output, max_dir_num=1): def _ReadFileTailInShell(file_path, line): """Tails the file in the last several lines.""" - return subprocess.check_output(['tail', '-%d' % line, file_path]) + return subprocess.check_output(['tail', '-%d' % line, + file_path]).decode('utf-8') diff --git a/xctestrunner/test_runner/xcresult_util.py b/xctestrunner/test_runner/xcresult_util.py new file mode 100644 index 0000000..57b3696 --- /dev/null +++ b/xctestrunner/test_runner/xcresult_util.py @@ -0,0 +1,153 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helper class for parsing xcresult under Xcode 11 or later.""" + +import json +import os +import subprocess + +from xctestrunner.shared import ios_errors +from xctestrunner.shared import xcode_info_util + + +def ExposeXcresult(xcresult_path, output_path): + """Exposes the files from xcresult. + + The files includes the diagnostics files and attachments files. + + Args: + xcresult_path: string, path of xcresult bundle. + output_path: string, path of output directory. + """ + root_result_bundle = _GetResultBundleObject(xcresult_path, bundle_id=None) + actions = root_result_bundle['actions']['_values'] + action_result = None + for action in actions: + if action['_type']['_name'] == 'ActionRecord': + action_result = action['actionResult'] + break + if action_result is None: + raise ios_errors.XcresultError( + 'Failed to get "ActionResult" from result bundle %s' % + root_result_bundle) + _ExposeDiagnostics(xcresult_path, output_path, action_result) + _ExposeAttachments(xcresult_path, output_path, action_result) + + +def _ExposeDiagnostics(xcresult_path, output_path, action_result): + """Exposes the diagnostics files from the given xcresult file.""" + if 'diagnosticsRef' not in action_result: + return + diagnostics_id = action_result['diagnosticsRef']['id']['_value'] + export_command = _MakeXcresulttoolCommand([ + 'export', '--path', xcresult_path, + '--output-path', output_path, '--type', 'directory', '--id', + diagnostics_id + ]) + subprocess.check_call(export_command) + + +def _ExposeAttachments(xcresult_path, output_path, action_result): + """Exposes the attachments files from the given xcresult file.""" + testsref_id = action_result['testsRef']['id']['_value'] + test_plan_summaries = _GetResultBundleObject( + xcresult_path, bundle_id=testsref_id) + test_plan_summary = test_plan_summaries['summaries']['_values'][0] + testable_summary = test_plan_summary['testableSummaries']['_values'][0] + # If the app under test crashes in unit test (XCTest) before loading the + # tests, the testable summary won't have tests summary. + if 'tests' not in testable_summary: + return + root_tests_summary = testable_summary['tests']['_values'][0] + failure_test_ref_ids = _GetFailureTestRefs(root_tests_summary) + for test_ref_id in failure_test_ref_ids: + test_summary_result = _GetResultBundleObject(xcresult_path, test_ref_id) + # if the test results in an `expectedFailures` entry, there might be an + # `activitySummaries` field present. + if 'activitySummaries' not in test_summary_result: + continue + + activity_summaries = test_summary_result['activitySummaries']['_values'] + for activity_summary in activity_summaries: + if 'attachments' in activity_summary: + test_identifier = test_summary_result['identifier']['_value'] + for attachment in activity_summary['attachments']['_values']: + file_name = attachment['filename']['_value'] + target_file_dir = os.path.join(output_path, 'Attachments', + test_identifier) + if not os.path.exists(target_file_dir): + os.makedirs(target_file_dir) + target_file_path = os.path.join(target_file_dir, file_name) + + payload_ref_id = attachment['payloadRef']['id']['_value'] + export_command = _MakeXcresulttoolCommand([ + 'export', '--path', xcresult_path, + '--output-path', target_file_path, '--type', 'file', '--id', + payload_ref_id + ]) + subprocess.check_call(export_command) + + +def _GetResultBundleObject(xcresult_path, bundle_id=None): + """Gets the result bundle object in json format. + + Args: + xcresult_path: string, path of xcresult bundle. + bundle_id: string, id of the result bundle object. If it is None, it is + rootID. + Returns: + A dict, result bundle object in json format. + """ + command = _MakeXcresulttoolCommand([ + 'get', '--format', 'json', '--path', xcresult_path + ]) + if bundle_id: + command.extend(['--id', bundle_id]) + return json.loads(subprocess.check_output(command).decode('utf-8')) + + +def _GetFailureTestRefs(test_summary): + """Gets a list of test summaryRef id of all failure test. + + Args: + test_summary: dict, a dict of test summary object. + Returns: + A list of failure test case's summaryRef id. + """ + failure_test_refs = [] + if 'subtests' in test_summary: + for sub_test_summary in test_summary['subtests']['_values']: + failure_test_refs.extend(_GetFailureTestRefs(sub_test_summary)) + else: + if (('testStatus' not in test_summary or + test_summary['testStatus']['_value'] != 'Success') and + 'summaryRef' in test_summary): + summary_ref_id = test_summary['summaryRef']['id']['_value'] + failure_test_refs.append(summary_ref_id) + return failure_test_refs + +def _MakeXcresulttoolCommand(args): + """Constructs xcresulttool command for selected Xcode version. + + Args: + args: array, a list of arguments to pass to xcresulttool. + Returns: + The xcresulttool command. + """ + command = ['xcrun', 'xcresulttool'] + args + xcode_version = xcode_info_util.GetXcodeVersionNumber() + if xcode_version >= 1600: + command.extend(['--legacy']) + return command diff --git a/test_runner/xctest_session.py b/xctestrunner/test_runner/xctest_session.py similarity index 80% rename from test_runner/xctest_session.py rename to xctestrunner/test_runner/xctest_session.py index f8badef..347d014 100644 --- a/test_runner/xctest_session.py +++ b/xctestrunner/test_runner/xctest_session.py @@ -24,17 +24,15 @@ from xctestrunner.shared import ios_constants from xctestrunner.shared import ios_errors from xctestrunner.shared import xcode_info_util -from xctestrunner.test_runner import dummy_project from xctestrunner.test_runner import logic_test_util -from xctestrunner.test_runner import runner_exit_codes -from xctestrunner.test_runner import test_summaries_util +from xctestrunner.test_runner import xcresult_util from xctestrunner.test_runner import xctestrun class XctestSession(object): """The class that runs XCTEST based tests.""" - def __init__(self, sdk, work_dir=None, output_dir=None): + def __init__(self, sdk, device_arch, work_dir=None, output_dir=None): """Initializes the XctestSession object. If work_dir is not provdied, will create a temp direcotry to be work_dir and @@ -43,6 +41,7 @@ def __init__(self, sdk, work_dir=None, output_dir=None): Args: sdk: ios_constants.SDK. The sdk of the target device. + device_arch: ios_constants.ARCH. The architecture of the target device. work_dir: string, the working directory contains runfiles. output_dir: string, The directory where derived data will go, including: 1) the detailed test session log which includes test output and the @@ -51,6 +50,7 @@ def __init__(self, sdk, work_dir=None, output_dir=None): specified, the directory will not be deleted after test ends.' """ self._sdk = sdk + self._device_arch = device_arch self._work_dir = work_dir self._delete_work_dir = True self._output_dir = output_dir @@ -58,8 +58,8 @@ def __init__(self, sdk, work_dir=None, output_dir=None): self._startup_timeout_sec = None self._destination_timeout_sec = None self._xctestrun_obj = None - self._dummy_project_obj = None self._prepared = False + self._keep_xcresult_data = True # The following fields are only for Logic Test. self._logic_test_bundle = None self._logic_test_env_vars = None @@ -82,7 +82,7 @@ def Prepare(self, app_under_test=None, test_bundle=None, """Prepares the test session. If xctestrun_file is not provided, will use app under test and test bundle - path to generate a new xctest file or dummy project. + path to generate a new xctest file. Args: app_under_test: string, the path of the application to be tested. It can @@ -121,11 +121,6 @@ def Prepare(self, app_under_test=None, test_bundle=None, self._delete_output_dir = True if xctestrun_file_path: - xcode_version_num = xcode_info_util.GetXcodeVersionNumber() - if xcode_version_num < 800: - raise ios_errors.IllegalArgumentError( - 'The xctestrun file is only supported in Xcode 8+. But current ' - 'Xcode version number is %s' % xcode_version_num) self._xctestrun_obj = xctestrun.XctestRun( xctestrun_file_path, test_type) else: @@ -138,37 +133,18 @@ def Prepare(self, app_under_test=None, test_bundle=None, test_bundle_dir, self._sdk, app_under_test_dir=app_under_test_dir, original_test_type=test_type) - # xctestrun can only support in Xcode 8+. - # Since xctestrun approach is more flexiable to local debug and is easy to - # support tests_to_run feature. So in Xcode 8+, use xctestrun approach to - # run XCTest and Logic Test. - if (test_type in ios_constants.SUPPORTED_TEST_TYPES and - test_type != ios_constants.TestType.LOGIC_TEST and - xcode_info_util.GetXcodeVersionNumber() >= 800): + if test_type not in ios_constants.SUPPORTED_TEST_TYPES: + raise ios_errors.IllegalArgumentError( + 'The test type %s is not supported. Supported test types are %s' % + (test_type, ios_constants.SUPPORTED_TEST_TYPES)) + + if test_type != ios_constants.TestType.LOGIC_TEST: xctestrun_factory = xctestrun.XctestRunFactory( - app_under_test_dir, test_bundle_dir, self._sdk, test_type, - signing_options, self._work_dir) + app_under_test_dir, test_bundle_dir, self._sdk, self._device_arch, + test_type, signing_options, self._work_dir) self._xctestrun_obj = xctestrun_factory.GenerateXctestrun() - elif test_type == ios_constants.TestType.XCUITEST: - raise ios_errors.IllegalArgumentError( - 'Only supports running XCUITest under Xcode 8+. ' - 'Current xcode version is %s' % - xcode_info_util.GetXcodeVersionNumber()) - elif test_type == ios_constants.TestType.XCTEST: - self._dummy_project_obj = dummy_project.DummyProject( - app_under_test_dir, - test_bundle_dir, - self._sdk, - ios_constants.TestType.XCTEST, - self._work_dir, - keychain_path=signing_options.get('keychain_path') or None) - self._dummy_project_obj.GenerateDummyProject() - elif test_type == ios_constants.TestType.LOGIC_TEST: - self._logic_test_bundle = test_bundle_dir else: - raise ios_errors.IllegalArgumentError( - 'The test type %s is not supported. Supported test types are %s' - % (test_type, ios_constants.SUPPORTED_TEST_TYPES)) + self._logic_test_bundle = test_bundle_dir self._prepared = True def SetLaunchOptions(self, launch_options): @@ -184,6 +160,7 @@ def SetLaunchOptions(self, launch_options): 'XctestSession.Prepare first.') if not launch_options: return + self._keep_xcresult_data = launch_options.get('keep_xcresult_data', True) self._startup_timeout_sec = launch_options.get('startup_timeout_sec') self._destination_timeout_sec = launch_options.get( 'destination_timeout_sec') @@ -200,25 +177,22 @@ def SetLaunchOptions(self, launch_options): if launch_options.get('uitest_auto_screenshots'): self._disable_uitest_auto_screenshots = False # By default, this SystemAttachmentLifetime field is in the generated - # xctestrun.plist. + # test.xctestrun. try: self._xctestrun_obj.DeleteXctestrunField('SystemAttachmentLifetime') except ios_errors.PlistError: pass - elif self._dummy_project_obj: - self._dummy_project_obj.SetEnvVars(launch_options.get('env_vars')) - self._dummy_project_obj.SetArgs(launch_options.get('args')) - self._dummy_project_obj.SetSkipTests(launch_options.get('skip_tests')) elif self._logic_test_bundle: self._logic_test_env_vars = launch_options.get('env_vars') self._logic_test_args = launch_options.get('args') self._logic_tests_to_run = launch_options.get('tests_to_run') - def RunTest(self, device_id): + def RunTest(self, device_id, os_version=None): """Runs test on the target device with the given device_id. Args: device_id: string, id of the device. + os_version: string, OS version of the device. Returns: A value of type runner_exit_codes.EXITCODE. @@ -232,30 +206,31 @@ def RunTest(self, device_id): 'XctestSession.Prepare first.') if self._xctestrun_obj: + result_bundle_path = os.path.join(self._output_dir, 'test.xcresult') exit_code = self._xctestrun_obj.Run(device_id, self._sdk, self._output_dir, self._startup_timeout_sec, - self._destination_timeout_sec) - for test_summaries_path in test_summaries_util.GetTestSummariesPaths( - self._output_dir): + self._destination_timeout_sec, + os_version=os_version, + result_bundle_path=result_bundle_path) + # The xcresult only contains raw data in Xcode 11 or later. + if xcode_info_util.GetXcodeVersionNumber() >= 1100: + expose_xcresult = os.path.join(self._output_dir, 'ExposeXcresult') try: - test_summaries_util.ParseTestSummaries( - test_summaries_path, - os.path.join(self._output_dir, 'Logs/Test/Attachments'), - True if self._disable_uitest_auto_screenshots else - exit_code == runner_exit_codes.EXITCODE.SUCCEEDED) - except ios_errors.PlistError as e: - logging.warning('Failed to parse test summaries %s: %s', - test_summaries_path, str(e)) + xcresult_util.ExposeXcresult(result_bundle_path, expose_xcresult) + if not self._keep_xcresult_data: + shutil.rmtree(result_bundle_path) + except subprocess.CalledProcessError as e: + logging.warning(e.output) return exit_code - elif self._dummy_project_obj: - return self._dummy_project_obj.RunXcTest(device_id, self._work_dir, - self._output_dir, - self._startup_timeout_sec) elif self._logic_test_bundle: return logic_test_util.RunLogicTestOnSim( - device_id, self._logic_test_bundle, self._logic_test_env_vars, - self._logic_test_args, self._logic_tests_to_run) + device_id, + self._logic_test_bundle, + self._logic_test_env_vars, + self._logic_test_args, + self._logic_tests_to_run, + os_version=os_version) else: raise ios_errors.XcodebuildTestError('Unexpected runtime error.') @@ -400,8 +375,10 @@ def _FinalizeTestType( def _DetectTestType(test_bundle_dir): """Detects if the test bundle is XCUITest or XCTest.""" test_bundle_exec_path = os.path.join( - test_bundle_dir, os.path.splitext(os.path.basename(test_bundle_dir))[0]) - output = subprocess.check_output(['nm', test_bundle_exec_path]) + test_bundle_dir, + os.path.splitext(os.path.basename(test_bundle_dir))[0]) + output = subprocess.check_output(['nm', + test_bundle_exec_path]).decode('utf-8') if 'XCUIApplication' in output: return ios_constants.TestType.XCUITEST else: diff --git a/test_runner/xctestrun.py b/xctestrunner/test_runner/xctestrun.py similarity index 78% rename from test_runner/xctestrun.py rename to xctestrunner/test_runner/xctestrun.py index 3d3cf55..4dbae1c 100644 --- a/test_runner/xctestrun.py +++ b/xctestrunner/test_runner/xctestrun.py @@ -23,6 +23,7 @@ from xctestrunner.shared import ios_constants from xctestrunner.shared import ios_errors from xctestrunner.shared import plist_util +from xctestrunner.shared import version_util from xctestrunner.shared import xcode_info_util from xctestrunner.test_runner import xcodebuild_test_executor @@ -54,8 +55,8 @@ def __init__(self, xctestrun_file_path, test_type=None, aut_bundle_id=None): self._xctestrun_file_path = xctestrun_file_path self._xctestrun_file_plist_obj = plist_util.Plist(xctestrun_file_path) # xctestrun file always has only key at root dict. - self._root_key = self._xctestrun_file_plist_obj.GetPlistField( - None).keys()[0] + self._root_key = list( + self._xctestrun_file_plist_obj.GetPlistField(None).keys())[0] self._test_type = test_type self._aut_bundle_id = aut_bundle_id @@ -141,7 +142,8 @@ def SetSkipTests(self, skip_tests): self.SetXctestrunField('SkipTestIdentifiers', skip_tests) def Run(self, device_id, sdk, derived_data_dir, startup_timeout_sec, - destination_timeout_sec=None): + destination_timeout_sec=None, os_version=None, + result_bundle_path=None): """Runs the test with generated xctestrun file in the specific device. Args: @@ -151,15 +153,38 @@ def Run(self, device_id, sdk, derived_data_dir, startup_timeout_sec, startup_timeout_sec: seconds until the xcodebuild command is deemed stuck. destination_timeout_sec: Wait for the given seconds while searching for the destination device. + os_version: os version of the device. + result_bundle_path: path to output a xcresult bundle to Returns: A value of type runner_exit_codes.EXITCODE. """ + # When running tests on iOS 12.1 or earlier simulator under Xcode 11 or + # later, it is required to add swift5 fallback libraries to environment + # variable. + # See https://github.com/bazelbuild/rules_apple/issues/684 for context. + xcode_version = xcode_info_util.GetXcodeVersionNumber() + if (xcode_version >= 1100 and + sdk == ios_constants.SDK.IPHONESIMULATOR and os_version and + version_util.GetVersionNumber(os_version) < 1220): + new_env_var = { + 'DYLD_FALLBACK_LIBRARY_PATH': + xcode_info_util.GetSwift5FallbackLibsDir() + } + self.SetTestEnvVars(new_env_var) logging.info('Running test-without-building with device %s', device_id) command = ['xcodebuild', 'test-without-building', '-xctestrun', self._xctestrun_file_path, '-destination', 'id=%s' % device_id, '-derivedDataPath', derived_data_dir] + + if xcode_version >= 1100 and result_bundle_path: + shutil.rmtree(result_bundle_path, ignore_errors=True) + command.extend(['-resultBundlePath', result_bundle_path]) + + if xcode_version >= 1410: + command.extend(['-collect-test-diagnostics=never']) + if destination_timeout_sec: command.extend(['-destination-timeout', str(destination_timeout_sec)]) exit_code, _ = xcodebuild_test_executor.XcodebuildTestExecutor( @@ -171,7 +196,7 @@ def Run(self, device_id, sdk, derived_data_dir, startup_timeout_sec, device_id=device_id, app_bundle_id=self._aut_bundle_id, startup_timeout_sec=startup_timeout_sec).Execute( - return_output=False) + return_output=False, result_bundle_path=result_bundle_path) return exit_code @property @@ -251,6 +276,7 @@ class XctestRunFactory(object): def __init__(self, app_under_test_dir, test_bundle_dir, sdk=ios_constants.SDK.IPHONESIMULATOR, + device_arch=ios_constants.ARCH.X86_64, test_type=ios_constants.TestType.XCUITEST, signing_options=None, work_dir=None): """Initializes the XctestRun object. @@ -263,6 +289,7 @@ def __init__(self, app_under_test_dir, test_bundle_dir, test_bundle_dir: string, path of the test bundle. sdk: string, SDKRoot of the test. See supported SDKs in module xctestrunner.shared.ios_constants. + device_arch: ios_constants.ARCH. The architecture of the target device. test_type: string, test type of the test bundle. See supported test types in module xctestrunner.shared.ios_constants. signing_options: dict, the signing app options. See @@ -276,6 +303,7 @@ def __init__(self, app_under_test_dir, test_bundle_dir, self._test_bundle_dir = test_bundle_dir self._test_name = os.path.splitext(os.path.basename(test_bundle_dir))[0] self._sdk = sdk + self._device_arch = device_arch self._test_type = test_type if self._sdk == ios_constants.SDK.IPHONEOS: self._on_device = True @@ -314,7 +342,7 @@ def GenerateXctestrun(self): return self._xctestrun_obj if self._work_dir: self._test_root_dir = os.path.join(self._work_dir, 'TEST_ROOT') - xctestrun_file_path = os.path.join(self._test_root_dir, 'xctestrun.plist') + xctestrun_file_path = os.path.join(self._test_root_dir, 'test.xctestrun') if os.path.exists(xctestrun_file_path): logging.info('Skips generating xctestrun file which is generated.') self._xctestrun_obj = XctestRun(xctestrun_file_path) @@ -342,7 +370,7 @@ def GenerateXctestrun(self): elif self._test_type == ios_constants.TestType.LOGIC_TEST: self._GenerateTestRootForLogicTest() - xctestrun_file_path = os.path.join(self._test_root_dir, 'xctestrun.plist') + xctestrun_file_path = os.path.join(self._test_root_dir, 'test.xctestrun') plist_util.Plist(xctestrun_file_path).SetPlistField('Runner', self._xctestrun_dict) @@ -390,8 +418,8 @@ def _ValidateArguments(self): def _GenerateTestRootForXcuitest(self): """Generates the test root for XCUITest. - The approach constructs xctestrun.plist and uitest runner app from Xcode. - Then copies app under test, test bundle, xctestrun.plist and uitest + The approach constructs test.xctestrun and uitest runner app from Xcode. + Then copies app under test, test bundle, test.xctestrun and uitest runner app to test root directory. """ platform_path = xcode_info_util.GetSdkPlatformPath(self._sdk) @@ -449,13 +477,11 @@ def _GenerateTestRootForXcuitest(self): _CopyAndSignFramework( os.path.join(platform_library_path, 'Frameworks/XCTest.framework'), runner_app_frameworks_dir, test_bundle_signing_identity) - xcode_version_num = xcode_info_util.GetXcodeVersionNumber() - if xcode_version_num >= 900: - _CopyAndSignFramework( - os.path.join(platform_library_path, - 'PrivateFrameworks/XCTAutomationSupport.framework'), - runner_app_frameworks_dir, test_bundle_signing_identity) - if xcode_version_num >= 1100: + _CopyAndSignFramework( + os.path.join(platform_library_path, + 'PrivateFrameworks/XCTAutomationSupport.framework'), + runner_app_frameworks_dir, test_bundle_signing_identity) + if xcode_info_util.GetXcodeVersionNumber() >= 1100: _CopyAndSignLibFile( os.path.join(platform_path, _LIB_XCTEST_SWIFT_RELATIVE_PATH), runner_app_frameworks_dir, test_bundle_signing_identity) @@ -463,12 +489,29 @@ def _GenerateTestRootForXcuitest(self): uitest_runner_app, entitlements_plist_path=entitlements_plist_path, identity=test_bundle_signing_identity) - + if xcode_info_util.GetXcodeVersionNumber() >= 1300: + _CopyAndSignFramework( + os.path.join(platform_library_path, + 'PrivateFrameworks/XCUIAutomation.framework'), + runner_app_frameworks_dir, test_bundle_signing_identity) + _CopyAndSignFramework( + os.path.join(platform_library_path, + 'PrivateFrameworks/XCTestCore.framework'), + runner_app_frameworks_dir, test_bundle_signing_identity) + _CopyAndSignFramework( + os.path.join(platform_library_path, + 'PrivateFrameworks/XCUnit.framework'), + runner_app_frameworks_dir, test_bundle_signing_identity) + if xcode_info_util.GetXcodeVersionNumber() >= 1430: + _CopyAndSignFramework( + os.path.join(platform_library_path, + 'PrivateFrameworks/XCTestSupport.framework'), + runner_app_frameworks_dir, test_bundle_signing_identity) bundle_util.CodesignBundle(self._test_bundle_dir) bundle_util.CodesignBundle(self._app_under_test_dir) platform_name = 'iPhoneOS' if self._on_device else 'iPhoneSimulator' - developer_path = '__PLATFORMS__/%s.platform/Developer/' % platform_name + developer_path = '__PLATFORMS__/%s.platform/Developer' % platform_name test_envs = { 'DYLD_FRAMEWORK_PATH': '__TESTROOT__:{developer}/Library/Frameworks:' '{developer}/Library/PrivateFrameworks'.format( @@ -476,6 +519,7 @@ def _GenerateTestRootForXcuitest(self): 'DYLD_LIBRARY_PATH': '__TESTROOT__:%s/usr/lib' % developer_path } self._xctestrun_dict = { + 'ProductModuleName': self._test_name.replace("-", "_"), 'IsUITestBundle': True, 'SystemAttachmentLifetime': 'keepNever', 'TestBundlePath': self._test_bundle_dir, @@ -506,10 +550,32 @@ def _GetUitestRunnerAppFromXcode(self, platform_library_path): uitest_runner_app_name = '%s-Runner' % test_bundle_name uitest_runner_app = os.path.join(self._test_root_dir, uitest_runner_app_name + '.app') + if os.path.exists(uitest_runner_app): + shutil.rmtree(uitest_runner_app) shutil.copytree(xctrunner_app, uitest_runner_app) + uitest_runner_exec = os.path.join(uitest_runner_app, uitest_runner_app_name) shutil.move( - os.path.join(uitest_runner_app, 'XCTRunner'), - os.path.join(uitest_runner_app, uitest_runner_app_name)) + os.path.join(uitest_runner_app, 'XCTRunner'), uitest_runner_exec) + # XCTRunner is multi-archs. When launching XCTRunner on arm64e device, it + # will be launched as arm64e process by default. If the test bundle is arm64 + # bundle, the XCTRunner which hosts the test bundle will fail to be + # launched. So removing the arm64e arch from XCTRunner can resolve this + # case. + test_executable = os.path.join(self._test_bundle_dir, test_bundle_name) + if self._device_arch == ios_constants.ARCH.ARM64E: + test_archs = bundle_util.GetFileArchTypes(test_executable) + if ios_constants.ARCH.ARM64E not in test_archs: + bundle_util.RemoveArchType(uitest_runner_exec, + ios_constants.ARCH.ARM64E) + # XCTRunner is multi-archs. When launching XCTRunner on Apple silicon + # simulator, it will be launched as arm64 process by default. If the test + # bundle is still x86_64, the XCTRunner which hosts the test bundle will + # fail to be launched. So removing the arm64 arch from XCTRunner can + # resolve this case. + elif not self._on_device: + test_archs = bundle_util.GetFileArchTypes(test_executable) + if ios_constants.ARCH.X86_64 in test_archs: + bundle_util.RemoveArchType(uitest_runner_exec, ios_constants.ARCH.ARM64) runner_app_info_plist_path = os.path.join(uitest_runner_app, 'Info.plist') info_plist = plist_util.Plist(runner_app_info_plist_path) @@ -539,8 +605,8 @@ def _PrepareUitestInRunerApp(self, uitest_runner_app): def _GenerateTestRootForXctest(self): """Generates the test root for XCTest. - The approach constructs xctestrun.plist from Xcode. Then copies app under - test, test bundle and xctestrun.plist to test root directory. + The approach constructs test.xctestrun from Xcode. Then copies app under + test, test bundle and test.xctestrun to test root directory. """ app_under_test_plugins_dir = os.path.join( self._app_under_test_dir, 'PlugIns') @@ -569,20 +635,11 @@ def _GenerateTestRootForXctest(self): os.path.join(platform_path, 'Developer/Library/Frameworks/XCTest.framework'), app_under_test_frameworks_dir, app_under_test_signing_identity) - xcode_version_num = xcode_info_util.GetXcodeVersionNumber() - if xcode_version_num < 1000: - bundle_injection_lib = os.path.join( - platform_path, 'Developer/Library/PrivateFrameworks/' - 'IDEBundleInjection.framework') - _CopyAndSignFramework(bundle_injection_lib, - app_under_test_frameworks_dir, - app_under_test_signing_identity) - else: - bundle_injection_lib = os.path.join( - platform_path, 'Developer/usr/lib/libXCTestBundleInject.dylib') - _CopyAndSignLibFile(bundle_injection_lib, app_under_test_frameworks_dir, - app_under_test_signing_identity) - if xcode_version_num >= 1100: + bundle_injection_lib = os.path.join( + platform_path, 'Developer/usr/lib/libXCTestBundleInject.dylib') + _CopyAndSignLibFile(bundle_injection_lib, app_under_test_frameworks_dir, + app_under_test_signing_identity) + if xcode_info_util.GetXcodeVersionNumber() >= 1100: _CopyAndSignFramework( os.path.join( platform_path, 'Developer/Library/PrivateFrameworks/' @@ -591,17 +648,48 @@ def _GenerateTestRootForXctest(self): _CopyAndSignLibFile( os.path.join(platform_path, _LIB_XCTEST_SWIFT_RELATIVE_PATH), app_under_test_frameworks_dir, app_under_test_signing_identity) + if xcode_info_util.GetXcodeVersionNumber() >= 1300: + if xcode_info_util.GetXcodeVersionNumber() >= 1640: + _CopyAndSignFramework( + os.path.join( + platform_path, 'Developer/Library/Frameworks/' + 'XCUIAutomation.framework'), + app_under_test_frameworks_dir, app_under_test_signing_identity) + else: + _CopyAndSignFramework( + os.path.join( + platform_path, 'Developer/Library/PrivateFrameworks/' + 'XCUIAutomation.framework'), + app_under_test_frameworks_dir, app_under_test_signing_identity) + _CopyAndSignFramework( + os.path.join( + platform_path, 'Developer/Library/PrivateFrameworks/' + 'XCTestCore.framework'), + app_under_test_frameworks_dir, app_under_test_signing_identity) + _CopyAndSignFramework( + os.path.join( + platform_path, 'Developer/Library/PrivateFrameworks/' + 'XCUnit.framework'), + app_under_test_frameworks_dir, app_under_test_signing_identity) + if xcode_info_util.GetXcodeVersionNumber() >= 1430: + _CopyAndSignFramework( + os.path.join( + platform_path, + 'Developer/Library/PrivateFrameworks/XCTestSupport.framework', + ), + app_under_test_frameworks_dir, + app_under_test_signing_identity, + ) bundle_util.CodesignBundle(self._test_bundle_dir) bundle_util.CodesignBundle(self._app_under_test_dir) app_under_test_name = os.path.splitext( os.path.basename(self._app_under_test_dir))[0] platform_name = 'iPhoneOS' if self._on_device else 'iPhoneSimulator' - developer_path = '__PLATFORMS__/%s.platform/Developer/' % platform_name - if xcode_info_util.GetXcodeVersionNumber() < 1000: - dyld_insert_libs = ('%s/Library/PrivateFrameworks/' - 'IDEBundleInjection.framework/IDEBundleInjection' % - developer_path) + developer_path = '__PLATFORMS__/%s.platform/Developer' % platform_name + + if self._on_device: + dyld_insert_libs = '__TESTHOST__/Frameworks/libXCTestBundleInject.dylib' else: dyld_insert_libs = ('%s/usr/lib/libXCTestBundleInject.dylib' % developer_path) @@ -614,6 +702,7 @@ def _GenerateTestRootForXctest(self): 'DYLD_LIBRARY_PATH': '__TESTROOT__:%s/usr/lib:' % developer_path } self._xctestrun_dict = { + 'ProductModuleName': self._test_name.replace("-", "_"), 'TestHostPath': self._app_under_test_dir, 'TestBundlePath': self._test_bundle_dir, 'IsAppHostedTestBundle': True, @@ -623,8 +712,8 @@ def _GenerateTestRootForXctest(self): def _GenerateTestRootForLogicTest(self): """Generates the test root for Logic test. - The approach constructs xctestrun.plist from Xcode. Then copies test bundle - and xctestrun.plist to test root directory. + The approach constructs test.xctestrun from Xcode. Then copies test bundle + and test.xctestrun to test root directory. """ dyld_framework_path = os.path.join( xcode_info_util.GetSdkPlatformPath(self._sdk), @@ -634,6 +723,7 @@ def _GenerateTestRootForLogicTest(self): 'DYLD_LIBRARY_PATH': dyld_framework_path } self._xctestrun_dict = { + 'ProductModuleName': self._test_name.replace("-", "_"), 'TestBundlePath': self._test_bundle_dir, 'TestHostPath': xcode_info_util.GetXctestToolPath(self._sdk), 'TestingEnvironmentVariables': test_envs,