Add an XCTest observer to swift_test targets that generates a JUnit…

…-style XML log at the path in the `XML_OUTPUT_PATH` environment variable defined by Bazel.

Due to differences between the open-source XCTest and Xcode's XCTest, only Darwin-based platforms can distinguish "skipped" tests from "passing" tests at this time (and even on that platform, it can only do so by referencing private APIs).

PiperOrigin-RevId: 437304646

swift/internal/swift_test.bzl

@@ -60,11 +60,16 @@ def _create_xctest_bundle(name, actions, binary):
     args.add(xctest_bundle.path)
     args.add(binary)
 
+    # When XCTest loads this bundle, it will create an instance of this class
+    # which will register the observer that writes the XML output.
+    plist = '{ NSPrincipalClass = "BazelXMLTestObserverRegistration"; }'
+
     actions.run_shell(
         arguments = [args],
         command = (
             'mkdir -p "$1/Contents/MacOS" && ' +
-            'cp "$2" "$1/Contents/MacOS"'
+            'cp "$2" "$1/Contents/MacOS" && ' +
+            'echo \'{}\' > "$1/Contents/Info.plist"'.format(plist)
         ),
         inputs = [binary],
         mnemonic = "SwiftCreateTestBundle",
@@ -200,6 +205,7 @@ def _swift_test_impl(ctx):
 
     srcs = ctx.files.srcs
     extra_copts = []
+    extra_deps = []
 
     # If no sources were provided and we're not using `.xctest` bundling, assume
     # that we need to discover tests using symbol graphs.
@@ -217,6 +223,9 @@ def _swift_test_impl(ctx):
 
         # The generated test runner uses `@main`.
         extra_copts = ["-parse-as-library"]
+        extra_deps = [ctx.attr._test_observer]
+    elif is_bundled:
+        extra_deps = [ctx.attr._test_observer]
 
     if srcs:
         module_name = ctx.attr.module_name
@@ -226,7 +235,9 @@ def _swift_test_impl(ctx):
         _, compilation_outputs = swift_common.compile(
             actions = ctx.actions,
             additional_inputs = ctx.files.swiftc_inputs,
-            compilation_contexts = get_compilation_contexts(ctx.attr.deps),
+            compilation_contexts = get_compilation_contexts(
+                ctx.attr.deps + extra_deps,
+            ),
             copts = expand_locations(
                 ctx,
                 ctx.attr.copts,
@@ -236,7 +247,7 @@ def _swift_test_impl(ctx):
             feature_configuration = feature_configuration,
             module_name = module_name,
             srcs = srcs,
-            swift_infos = get_providers(ctx.attr.deps, SwiftInfo),
+            swift_infos = get_providers(ctx.attr.deps + extra_deps, SwiftInfo),
             swift_toolchain = swift_toolchain,
             target_name = ctx.label.name,
         )
@@ -253,7 +264,7 @@ def _swift_test_impl(ctx):
         additional_linking_contexts = [malloc_linking_context(ctx)],
         cc_feature_configuration = cc_feature_configuration,
         compilation_outputs = compilation_outputs,
-        deps = ctx.attr.deps,
+        deps = ctx.attr.deps + extra_deps,
         grep_includes = ctx.file._grep_includes,
         name = ctx.label.name,
         output_type = "executable",
@@ -336,6 +347,11 @@ swift_test = rule(
                 ),
                 executable = True,
             ),
+            "_test_observer": attr.label(
+                default = Label(
+                    "@build_bazel_rules_swift//tools/test_observer",
+                ),
+            ),
             "_xctest_runner_template": attr.label(
                 allow_single_file = True,
                 default = Label(

tools/test_discoverer/TestPrinter.swift

@@ -134,6 +134,7 @@ struct TestPrinter {
   /// Prints the main test runner to a Swift source file.
   func printTestRunner(toFileAt url: URL) {
     var contents = """
+      import BazelTestObservation
       import XCTest
 
       @main
@@ -153,6 +154,9 @@ struct TestPrinter {
     }
 
     contents += """
+          if let xmlObserver = BazelXMLTestObserver.default {
+            XCTestObservationCenter.shared.addTestObserver(xmlObserver)
+          }
           XCTMain(tests)
         }
       }

tools/test_observer/BUILD

@@ -0,0 +1,12 @@
+load("//swift:swift.bzl", "swift_library")
+
+swift_library(
+    name = "test_observer",
+    srcs = [
+        "BazelXMLTestObserver.swift",
+        "BazelXMLTestObserverRegistration.swift",
+        "StringInterpolation+XMLEscaping.swift",
+    ],
+    module_name = "BazelTestObservation",
+    visibility = ["//visibility:public"],
+)

tools/test_observer/BazelXMLTestObserver.swift

@@ -0,0 +1,210 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+import XCTest
+
+/// An XCTest observer that generates an XML file in the format described by the
+/// [JUnit test result schema](https://windyroad.com.au/dl/Open%20Source/JUnit.xsd).
+public final class BazelXMLTestObserver: NSObject {
+  /// The file handle to which the XML content will be written.
+  private let fileHandle: FileHandle
+
+  /// The current indentation to print before each line, as UTF-8 code units.
+  private var indentation: Data
+
+  /// The default XML-generating XCTest observer, which determines the output file based on the
+  /// value of the `XML_OUTPUT_FILE` environment variable.
+  ///
+  /// If the `XML_OUTPUT_FILE` environment variable is not set or the file at that path could not be
+  /// created and opened for writing, the value of this property will be nil.
+  public static let `default`: BazelXMLTestObserver? = {
+    guard
+      let outputPath = ProcessInfo.processInfo.environment["XML_OUTPUT_FILE"],
+      FileManager.default.createFile(atPath: outputPath, contents: nil, attributes: nil),
+      let fileHandle = FileHandle(forWritingAtPath: outputPath)
+    else {
+      return nil
+    }
+    return .init(fileHandle: fileHandle)
+  }()
+
+  /// Creates a new XML-generating XCTest observer that writes its content to the given file handle.
+  private init(fileHandle: FileHandle) {
+    self.fileHandle = fileHandle
+    self.indentation = Data()
+  }
+
+  /// Writes the given string to the observer's file handle.
+  private func writeLine<S: StringProtocol>(_ string: S) {
+    if !indentation.isEmpty {
+      fileHandle.write(indentation)
+    }
+    fileHandle.write(string.data(using: .utf8)!)  // Conversion to UTF-8 cannot fail.
+    fileHandle.write(Data([UInt8(ascii: "\n")]))
+  }
+
+  /// Increases the current indentation level by two spaces.
+  private func indent() {
+    indentation.append(contentsOf: [UInt8(ascii: " "), UInt8(ascii: " ")])
+  }
+
+  /// Reduces the current indentation level by two spaces.
+  private func dedent() {
+    indentation.removeLast(2)
+  }
+
+  /// Canonicalizes the name of the test case for printing into the XML file.
+  ///
+  /// The canonical name of the test is `TestClass.testMethod` (i.e., Swift-style syntax). When
+  /// running tests under the Objective-C runtime, the test cases will have Objective-C-style names
+  /// (i.e., `-[TestClass testMethod]`), so this method converts those to the desired syntax.
+  ///
+  /// Any test name that does not match one of those two syntaxes is returned unchanged.
+  private func canonicalizedName(of testCase: XCTestCase) -> String {
+    let name = testCase.name
+    guard name.hasPrefix("-[") && name.hasSuffix("]") else {
+      return name
+    }
+
+    let trimmedName = name.dropFirst(2).dropLast()
+    guard let spaceIndex = trimmedName.lastIndex(of: " ") else {
+      return String(trimmedName)
+    }
+
+    return "\(trimmedName[..<spaceIndex]).\(trimmedName[trimmedName.index(after: spaceIndex)...])"
+  }
+}
+
+extension BazelXMLTestObserver: XCTestObservation {
+  public func testBundleWillStart(_ testBundle: Bundle) {
+    writeLine(#"<?xml version="1.0" encoding="utf-8"?>"#)
+    writeLine("<testsuites>")
+    indent()
+  }
+
+  public func testBundleDidFinish(_ testBundle: Bundle) {
+    dedent()
+    writeLine("</testsuites>")
+  }
+
+  public func testSuiteWillStart(_ testSuite: XCTestSuite) {
+    writeLine(
+      #"<testsuite name="\#(xmlEscaping: testSuite.name)" tests="\#(testSuite.testCaseCount)">"#)
+    indent()
+  }
+
+  public func testSuiteDidFinish(_ testSuite: XCTestSuite) {
+    dedent()
+    writeLine("</testsuite>")
+  }
+
+  public func testCaseWillStart(_ testCase: XCTestCase) {
+    writeLine(
+      #"<testcase name="\#(xmlEscaping: canonicalizedName(of: testCase))" status="run" "#
+      + #"result="completed">"#)
+    indent()
+  }
+
+  public func testCaseDidFinish(_ testCase: XCTestCase) {
+    dedent()
+    writeLine("</testcase>")
+  }
+
+  // On platforms with the Objective-C runtime, we use the richer `XCTIssue`-based APIs. Anywhere
+  // else, we're building with the open-source version of XCTest which has only the older
+  // `didFailWithDescription` API.
+  #if canImport(ObjectiveC)
+    public func testCase(_ testCase: XCTestCase, didRecord issue: XCTIssue) {
+      let tag: String
+      switch issue.type {
+      case .assertionFailure, .performanceRegression, .unmatchedExpectedFailure:
+        tag = "failure"
+      case .system, .thrownError, .uncaughtException:
+        tag = "error"
+      @unknown default:
+        tag = "failure"
+      }
+
+      writeLine(#"<\#(tag) message="\#(xmlEscaping: issue.compactDescription)"/>"#)
+    }
+  #else
+    public func testCase(
+      _ testCase: XCTestCase,
+      didFailWithDescription description: String,
+      inFile filePath: String?,
+      atLine lineNumber: Int
+    ) {
+      let tag = description.hasPrefix(#"threw error ""#) ? "error" : "failure"
+      writeLine(#"<\#(tag) message="\#(xmlEscaping: description)"/>"#)
+    }
+  #endif
+}
+
+// Hacks ahead! XCTest does not declare the methods that it uses to notify observers of skipped
+// tests as part of the public `XCTestObservation` protocol. Instead, they are only available on
+// various framework-internal protocols that XCTest checks for conformance against at runtime.
+//
+// On Darwin platforms, thanks to the Objective-C runtime, we can declare protocols with the same
+// names in our library and implement those methods, and XCTest will call them so that we can log
+// the skipped tests in our output. Note that we have to re-specify the protocol name in the `@objc`
+// attribute to remove the module name for the runtime.
+//
+// On non-Darwin platforms, we don't have an escape hatch because XCTest is implemented in pure
+// Swift and we can't play the same runtime games, so skipped tests simply get tracked as "passing"
+// there.
+#if canImport(ObjectiveC)
+  /// Declares the observation method that is called by XCTest in Xcode 12.5 when a test case is
+  /// skipped.
+  @objc(_XCTestObservationInternal)
+  protocol _XCTestObservationInternal {
+    func testCase(
+      _ testCase: XCTestCase,
+      wasSkippedWithDescription description: String,
+      sourceCodeContext: XCTSourceCodeContext?)
+  }
+
+  extension BazelXMLTestObserver: _XCTestObservationInternal {
+    public func testCase(
+      _ testCase: XCTestCase,
+      wasSkippedWithDescription description: String,
+      sourceCodeContext: XCTSourceCodeContext?
+    ) {
+      self.testCase(
+        testCase,
+        didRecordSkipWithDescription: description,
+        sourceCodeContext: sourceCodeContext)
+    }
+  }
+
+  /// Declares the observation method that is called by XCTest in Xcode 13 and later when a test
+  /// case is skipped.
+  @objc(_XCTestObservationPrivate)
+  protocol _XCTestObservationPrivate {
+    func testCase(
+      _ testCase: XCTestCase,
+      didRecordSkipWithDescription description: String,
+      sourceCodeContext: XCTSourceCodeContext?)
+  }
+
+  extension BazelXMLTestObserver: _XCTestObservationPrivate {
+    public func testCase(
+      _ testCase: XCTestCase,
+      didRecordSkipWithDescription description: String,
+      sourceCodeContext: XCTSourceCodeContext?
+    ) {
+      writeLine(#"<skipped message="\#(xmlEscaping: description)"/>"#)
+    }
+  }
+#endif

tools/test_observer/BazelXMLTestObserverRegistration.swift

@@ -0,0 +1,31 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#if canImport(ObjectiveC)
+  import Foundation
+  import XCTest
+
+  /// The principal class in an XCTest bundle on Darwin-based platforms, which registers the
+  /// XML-generating observer with the XCTest observation center when the bundle is loaded.
+  @objc(BazelXMLTestObserverRegistration)
+  public final class BazelXMLTestObserverRegistration: NSObject {
+    @objc public override init() {
+      super.init()
+
+      if let observer = BazelXMLTestObserver.default {
+        XCTestObservationCenter.shared.addTestObserver(observer)
+      }
+    }
+  }
+#endif

tools/test_observer/StringInterpolation+XMLEscaping.swift

@@ -0,0 +1,38 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+extension String.StringInterpolation {
+  /// Appends the given string to a string interpolation, escaping any characters with special XML
+  /// meanings.
+  mutating func appendInterpolation<S: StringProtocol>(xmlEscaping string: S) {
+    var remainder = string[...]
+    while let escapeIndex = remainder.firstIndex(where: { xmlEscapeMapping[$0] != nil }) {
+      appendLiteral(String(remainder[..<escapeIndex]))
+      appendLiteral(xmlEscapeMapping[remainder[escapeIndex]]!)
+      remainder = remainder[remainder.index(after: escapeIndex)...]
+    }
+    if !remainder.isEmpty {
+      appendLiteral(String(remainder))
+    }
+  }
+}
+
+/// The mapping from characters with special meanings in XML to their escaped form.
+private let xmlEscapeMapping: [Character: String] = [
+  "\"": "&quot;",
+  "'": "&apos;",
+  "&": "&amp;",
+  "<": "&lt;",
+  ">": "&gt;",
+]