diff --git a/swift/internal/derived_files.bzl b/swift/internal/derived_files.bzl index edbd21ffb..4d7537c7e 100644 --- a/swift/internal/derived_files.bzl +++ b/swift/internal/derived_files.bzl @@ -438,6 +438,18 @@ def _intermediate_swift_const_values_file(actions, target_name, src): paths.join(dirname, "{}.swiftconstvalues".format(basename)), ) +def _xctest_bundle(actions, target_name): + """Declares a directory for the `.xctest` bundle of a Darwin `swift_test`. + + Args: + actions: The context's actions object. + target_name: The name of the target being built. + + Returns: + The declared `File`. + """ + return actions.declare_directory("{}.xctest".format(target_name)) + def _xctest_runner_script(actions, target_name): """Declares a file for the script that runs an `.xctest` bundle on Darwin. @@ -474,6 +486,7 @@ derived_files = struct( vfsoverlay = _vfsoverlay, whole_module_object_file = _whole_module_object_file, swift_const_values_file = _swift_const_values_file, + xctest_bundle = _xctest_bundle, xctest_runner_script = _xctest_runner_script, generated_header = _declare_validated_generated_header, ) diff --git a/swift/internal/swift_test.bzl b/swift/internal/swift_test.bzl index 248804464..9d5de46e2 100644 --- a/swift/internal/swift_test.bzl +++ b/swift/internal/swift_test.bzl @@ -64,15 +64,55 @@ def _maybe_parse_as_library_copts(srcs): srcs[0].basename != "main.swift" return ["-parse-as-library"] if use_parse_as_library else [] -def _create_xctest_runner(name, actions, executable, xctest_runner_template): +def _create_xctest_bundle(name, actions, binary): + """Creates an `.xctest` bundle that contains the given binary. + + Args: + name: The name of the target being built, which will be used as the + basename of the bundle (followed by the .xctest bundle extension). + actions: The context's actions object. + binary: The binary that will be copied into the test bundle. + + Returns: + A `File` (tree artifact) representing the `.xctest` bundle. + """ + xctest_bundle = derived_files.xctest_bundle( + actions = actions, + target_name = name, + ) + + args = actions.args() + args.add(xctest_bundle.path) + args.add(binary) + + # When XCTest loads this bundle, it will create an instance of this class + # which will register the observer that writes the XML output. + plist = '{ NSPrincipalClass = "BazelXMLTestObserverRegistration"; }' + + actions.run_shell( + arguments = [args], + command = ( + 'mkdir -p "$1/Contents/MacOS" && ' + + 'cp "$2" "$1/Contents/MacOS" && ' + + 'echo \'{}\' > "$1/Contents/Info.plist"'.format(plist) + ), + inputs = [binary], + mnemonic = "SwiftCreateTestBundle", + outputs = [xctest_bundle], + progress_message = "Creating test bundle for {}".format(name), + ) + + return xctest_bundle + +def _create_xctest_runner(name, actions, bundle, xctest_runner_template): """Creates a script that will launch `xctest` with the given test bundle. Args: name: The name of the target being built, which will be used as the basename of the test runner script. actions: The context's actions object. - executable: The `File` representing the executable inside the `.xctest` - bundle that should be executed. + bundle: The `File` representing the `.xctest` bundle that should be + executed. xctest_runner_template: The `File` that will be used as a template to generate the test runner shell script. @@ -90,7 +130,7 @@ def _create_xctest_runner(name, actions, executable, xctest_runner_template): output = xctest_runner, template = xctest_runner_template, substitutions = { - "%executable%": executable.short_path, + "%bundle%": bundle.short_path, }, ) @@ -297,7 +337,7 @@ def _swift_test_impl(ctx): # This is already collected from `linking_context`. compilation_outputs = None, deps = ctx.attr.deps + extra_link_deps, - name = "{0}.xctest/Contents/MacOS/{0}".format(ctx.label.name) if is_bundled else ctx.label.name, + name = ctx.label.name, output_type = "executable", owner = ctx.label, stamp = ctx.attr.stamp, @@ -313,13 +353,18 @@ def _swift_test_impl(ctx): # script that launches it via `xctest`. Otherwise, just use the binary # itself as the executable to launch. if is_bundled: + xctest_bundle = _create_xctest_bundle( + name = ctx.label.name, + actions = ctx.actions, + binary = linking_outputs.executable, + ) xctest_runner = _create_xctest_runner( name = ctx.label.name, actions = ctx.actions, - executable = linking_outputs.executable, + bundle = xctest_bundle, xctest_runner_template = ctx.file._xctest_runner_template, ) - additional_test_outputs = [linking_outputs.executable] + additional_test_outputs = [xctest_bundle] executable = xctest_runner else: additional_test_outputs = [] diff --git a/tools/test_discoverer/TestPrinter.swift b/tools/test_discoverer/TestPrinter.swift index e9bbb5148..03186c075 100644 --- a/tools/test_discoverer/TestPrinter.swift +++ b/tools/test_discoverer/TestPrinter.swift @@ -134,6 +134,7 @@ struct TestPrinter { /// Prints the main test runner to a Swift source file. func printTestRunner(toFileAt url: URL) { var contents = """ + import BazelTestObservation import XCTest @main @@ -153,6 +154,9 @@ struct TestPrinter { } contents += """ + if let xmlObserver = BazelXMLTestObserver.default { + XCTestObservationCenter.shared.addTestObserver(xmlObserver) + } XCTMain(tests) } } diff --git a/tools/test_observer/BUILD b/tools/test_observer/BUILD new file mode 100644 index 000000000..e7b519942 --- /dev/null +++ b/tools/test_observer/BUILD @@ -0,0 +1,12 @@ +load("//swift:swift.bzl", "swift_library") + +swift_library( + name = "test_observer", + srcs = [ + "BazelXMLTestObserver.swift", + "BazelXMLTestObserverRegistration.swift", + "StringInterpolation+XMLEscaping.swift", + ], + module_name = "BazelTestObservation", + visibility = ["//visibility:public"], +) diff --git a/tools/test_observer/BazelXMLTestObserver.swift b/tools/test_observer/BazelXMLTestObserver.swift new file mode 100644 index 000000000..7995382e1 --- /dev/null +++ b/tools/test_observer/BazelXMLTestObserver.swift @@ -0,0 +1,210 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import XCTest + +/// An XCTest observer that generates an XML file in the format described by the +/// [JUnit test result schema](https://windyroad.com.au/dl/Open%20Source/JUnit.xsd). +public final class BazelXMLTestObserver: NSObject { + /// The file handle to which the XML content will be written. + private let fileHandle: FileHandle + + /// The current indentation to print before each line, as UTF-8 code units. + private var indentation: Data + + /// The default XML-generating XCTest observer, which determines the output file based on the + /// value of the `XML_OUTPUT_FILE` environment variable. + /// + /// If the `XML_OUTPUT_FILE` environment variable is not set or the file at that path could not be + /// created and opened for writing, the value of this property will be nil. + public static let `default`: BazelXMLTestObserver? = { + guard + let outputPath = ProcessInfo.processInfo.environment["XML_OUTPUT_FILE"], + FileManager.default.createFile(atPath: outputPath, contents: nil, attributes: nil), + let fileHandle = FileHandle(forWritingAtPath: outputPath) + else { + return nil + } + return .init(fileHandle: fileHandle) + }() + + /// Creates a new XML-generating XCTest observer that writes its content to the given file handle. + private init(fileHandle: FileHandle) { + self.fileHandle = fileHandle + self.indentation = Data() + } + + /// Writes the given string to the observer's file handle. + private func writeLine(_ string: S) { + if !indentation.isEmpty { + fileHandle.write(indentation) + } + fileHandle.write(string.data(using: .utf8)!) // Conversion to UTF-8 cannot fail. + fileHandle.write(Data([UInt8(ascii: "\n")])) + } + + /// Increases the current indentation level by two spaces. + private func indent() { + indentation.append(contentsOf: [UInt8(ascii: " "), UInt8(ascii: " ")]) + } + + /// Reduces the current indentation level by two spaces. + private func dedent() { + indentation.removeLast(2) + } + + /// Canonicalizes the name of the test case for printing into the XML file. + /// + /// The canonical name of the test is `TestClass.testMethod` (i.e., Swift-style syntax). When + /// running tests under the Objective-C runtime, the test cases will have Objective-C-style names + /// (i.e., `-[TestClass testMethod]`), so this method converts those to the desired syntax. + /// + /// Any test name that does not match one of those two syntaxes is returned unchanged. + private func canonicalizedName(of testCase: XCTestCase) -> String { + let name = testCase.name + guard name.hasPrefix("-[") && name.hasSuffix("]") else { + return name + } + + let trimmedName = name.dropFirst(2).dropLast() + guard let spaceIndex = trimmedName.lastIndex(of: " ") else { + return String(trimmedName) + } + + return "\(trimmedName[.."#) + writeLine("") + indent() + } + + public func testBundleDidFinish(_ testBundle: Bundle) { + dedent() + writeLine("") + } + + public func testSuiteWillStart(_ testSuite: XCTestSuite) { + writeLine( + #""#) + indent() + } + + public func testSuiteDidFinish(_ testSuite: XCTestSuite) { + dedent() + writeLine("") + } + + public func testCaseWillStart(_ testCase: XCTestCase) { + writeLine( + #""#) + indent() + } + + public func testCaseDidFinish(_ testCase: XCTestCase) { + dedent() + writeLine("") + } + + // On platforms with the Objective-C runtime, we use the richer `XCTIssue`-based APIs. Anywhere + // else, we're building with the open-source version of XCTest which has only the older + // `didFailWithDescription` API. + #if canImport(ObjectiveC) + public func testCase(_ testCase: XCTestCase, didRecord issue: XCTIssue) { + let tag: String + switch issue.type { + case .assertionFailure, .performanceRegression, .unmatchedExpectedFailure: + tag = "failure" + case .system, .thrownError, .uncaughtException: + tag = "error" + @unknown default: + tag = "failure" + } + + writeLine(#"<\#(tag) message="\#(xmlEscaping: issue.compactDescription)"/>"#) + } + #else + public func testCase( + _ testCase: XCTestCase, + didFailWithDescription description: String, + inFile filePath: String?, + atLine lineNumber: Int + ) { + let tag = description.hasPrefix(#"threw error ""#) ? "error" : "failure" + writeLine(#"<\#(tag) message="\#(xmlEscaping: description)"/>"#) + } + #endif +} + +// Hacks ahead! XCTest does not declare the methods that it uses to notify observers of skipped +// tests as part of the public `XCTestObservation` protocol. Instead, they are only available on +// various framework-internal protocols that XCTest checks for conformance against at runtime. +// +// On Darwin platforms, thanks to the Objective-C runtime, we can declare protocols with the same +// names in our library and implement those methods, and XCTest will call them so that we can log +// the skipped tests in our output. Note that we have to re-specify the protocol name in the `@objc` +// attribute to remove the module name for the runtime. +// +// On non-Darwin platforms, we don't have an escape hatch because XCTest is implemented in pure +// Swift and we can't play the same runtime games, so skipped tests simply get tracked as "passing" +// there. +#if canImport(ObjectiveC) + /// Declares the observation method that is called by XCTest in Xcode 12.5 when a test case is + /// skipped. + @objc(_XCTestObservationInternal) + protocol _XCTestObservationInternal { + func testCase( + _ testCase: XCTestCase, + wasSkippedWithDescription description: String, + sourceCodeContext: XCTSourceCodeContext?) + } + + extension BazelXMLTestObserver: _XCTestObservationInternal { + public func testCase( + _ testCase: XCTestCase, + wasSkippedWithDescription description: String, + sourceCodeContext: XCTSourceCodeContext? + ) { + self.testCase( + testCase, + didRecordSkipWithDescription: description, + sourceCodeContext: sourceCodeContext) + } + } + + /// Declares the observation method that is called by XCTest in Xcode 13 and later when a test + /// case is skipped. + @objc(_XCTestObservationPrivate) + protocol _XCTestObservationPrivate { + func testCase( + _ testCase: XCTestCase, + didRecordSkipWithDescription description: String, + sourceCodeContext: XCTSourceCodeContext?) + } + + extension BazelXMLTestObserver: _XCTestObservationPrivate { + public func testCase( + _ testCase: XCTestCase, + didRecordSkipWithDescription description: String, + sourceCodeContext: XCTSourceCodeContext? + ) { + writeLine(#""#) + } + } +#endif diff --git a/tools/test_observer/BazelXMLTestObserverRegistration.swift b/tools/test_observer/BazelXMLTestObserverRegistration.swift new file mode 100644 index 000000000..851b8806d --- /dev/null +++ b/tools/test_observer/BazelXMLTestObserverRegistration.swift @@ -0,0 +1,31 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if canImport(ObjectiveC) + import Foundation + import XCTest + + /// The principal class in an XCTest bundle on Darwin-based platforms, which registers the + /// XML-generating observer with the XCTest observation center when the bundle is loaded. + @objc(BazelXMLTestObserverRegistration) + public final class BazelXMLTestObserverRegistration: NSObject { + @objc public override init() { + super.init() + + if let observer = BazelXMLTestObserver.default { + XCTestObservationCenter.shared.addTestObserver(observer) + } + } + } +#endif diff --git a/tools/test_observer/StringInterpolation+XMLEscaping.swift b/tools/test_observer/StringInterpolation+XMLEscaping.swift new file mode 100644 index 000000000..47fe1d198 --- /dev/null +++ b/tools/test_observer/StringInterpolation+XMLEscaping.swift @@ -0,0 +1,38 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +extension String.StringInterpolation { + /// Appends the given string to a string interpolation, escaping any characters with special XML + /// meanings. + mutating func appendInterpolation(xmlEscaping string: S) { + var remainder = string[...] + while let escapeIndex = remainder.firstIndex(where: { xmlEscapeMapping[$0] != nil }) { + appendLiteral(String(remainder[..": ">", +] diff --git a/tools/xctest_runner/xctest_runner.sh.template b/tools/xctest_runner/xctest_runner.sh.template index dc0e529b4..6453196bb 100644 --- a/tools/xctest_runner/xctest_runner.sh.template +++ b/tools/xctest_runner/xctest_runner.sh.template @@ -26,9 +26,10 @@ trap 'rm -rf "$tmp_dir"' EXIT readonly profraw="$tmp_dir/coverage.profraw" readonly testlog="$tmp_dir/test.log" -# Foo.xctest/Contents/MacOS/Foo -> Foo.xctest -executable_path="%executable%" -bundle_path="$(dirname "$(dirname "$(dirname "$executable_path")")")" +# some/path/Foo.xctest -> some/path/Foo.xctest/Contents/MacOS/Foo +bundle_path="%bundle%" +basename="${bundle_path##*/}" +executable_path="$bundle_path/Contents/MacOS/${basename%.xctest}" sanitizer_dyld_env="" if output=$(otool -L "$executable_path" | grep @rpath/libclang_rt | xargs | cut -d " " -f 1); then