From 4da4061b0a37c9e26297ce8387d9151344204ec6 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 3 Nov 2025 21:09:45 -0500 Subject: [PATCH 01/11] Add an upper bound on the number of test cases we run in parallel. This PR adds an upper bound, `NCORES * 2`, on the number of test cases we run in parallel. Depending on the exact nature of your tests, this can significantly reduce the maximum amount of dirty memory needed, but does not generally impact execution time. You can configure a different upper bound by setting the `"SWT_EXPERIMENTAL_MAXIMUM_PARALLELIZATION_WIDTH"` environment variable (look, naming is hard okay?) --- .../Testing/ABI/EntryPoints/EntryPoint.swift | 13 ++++ Sources/Testing/CMakeLists.txt | 1 + Sources/Testing/Running/Configuration.swift | 53 +++++++++++++++ Sources/Testing/Running/Runner.swift | 6 +- Sources/Testing/Support/Serializer.swift | 65 +++++++++++++++++++ 5 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 Sources/Testing/Support/Serializer.swift diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index f4f1a751c..0b824394c 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -210,6 +210,9 @@ public struct __CommandLineArguments_v0: Sendable { /// The value of the `--parallel` or `--no-parallel` argument. public var parallel: Bool? + /// The maximum number of test tasks to run in parallel. + public var experimentalMaximumParallelizationWidth: Int? + /// The value of the `--symbolicate-backtraces` argument. public var symbolicateBacktraces: String? @@ -336,6 +339,7 @@ extension __CommandLineArguments_v0: Codable { enum CodingKeys: String, CodingKey { case listTests case parallel + case experimentalMaximumParallelizationWidth case symbolicateBacktraces case verbose case veryVerbose @@ -485,6 +489,11 @@ func parseCommandLineArguments(from args: [String]) throws -> __CommandLineArgum if args.contains("--no-parallel") { result.parallel = false } + if let maximumParallelizationWidth = args.argumentValue(forLabel: "--experimental-maximum-parallelization-width").flatMap(Int.init) + ?? Environment.variable(named: "SWT_EXPERIMENTAL_MAXIMUM_PARALLELIZATION_WIDTH").flatMap(Int.init) { + // TODO: decide if we want to repurpose --num-workers for this use case? + result.experimentalMaximumParallelizationWidth = maximumParallelizationWidth + } // Whether or not to symbolicate backtraces in the event stream. if let symbolicateBacktraces = args.argumentValue(forLabel: "--symbolicate-backtraces") { @@ -546,6 +555,10 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr // Parallelization (on by default) configuration.isParallelizationEnabled = args.parallel ?? true + if let maximumParallelizationWidth = args.experimentalMaximumParallelizationWidth { + try! FileHandle.stderr.write("MAX WIDTH: \(maximumParallelizationWidth)\n") + configuration.maximumParallelizationWidth = maximumParallelizationWidth + } // Whether or not to symbolicate backtraces in the event stream. if let symbolicateBacktraces = args.symbolicateBacktraces { diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index bebf05eb9..ebcfe4e21 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -90,6 +90,7 @@ add_library(Testing Support/Graph.swift Support/JSON.swift Support/Locked.swift + Support/Serializer.swift Support/VersionNumber.swift Support/Versions.swift Discovery+Macro.swift diff --git a/Sources/Testing/Running/Configuration.swift b/Sources/Testing/Running/Configuration.swift index 12b8827de..754dda02b 100644 --- a/Sources/Testing/Running/Configuration.swift +++ b/Sources/Testing/Running/Configuration.swift @@ -8,6 +8,8 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // +private import _TestingInternals + /// A type containing settings for preparing and running tests. @_spi(ForToolsIntegrationOnly) public struct Configuration: Sendable { @@ -20,6 +22,57 @@ public struct Configuration: Sendable { /// Whether or not to parallelize the execution of tests and test cases. public var isParallelizationEnabled: Bool = true + /// The number of CPU cores on the current system, or `nil` if that + /// information is not available. + private static var _cpuCoreCount: Int? { +#if SWT_TARGET_OS_APPLE + var result: Int32 = -1 + var mib: [Int32] = [CTL_HW, HW_NCPU] + var resultByteCount = MemoryLayout.stride + guard 0 == sysctl(&mib, UInt32(mib.count), &result, &resultByteCount, nil, 0) else { + return nil + } + return Int(result) +#elseif os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) + return Int(sysconf(Int32(_SC_NPROCESSORS_CONF))) +#elseif os(Windows) + var siInfo = SYSTEM_INFO() + GetSystemInfo(&siInfo) + return Int(siInfo.dwNumberOfProcessors) +#else + return nil +#endif + + } + + /// The maximum width of parallelization. + /// + /// The value of this property determines how many tests (or rather, test + /// cases) will run in parallel. The default value of this property is equal + /// to twice the number of CPU cores reported by the operating system, or + /// `Int.max` if that value is not available. + @_spi(Experimental) + public var maximumParallelizationWidth: Int { + get { + serializer?.maximumWidth ?? .max + } + set { + if newValue < .max { + serializer = Serializer(maximumWidth: maximumParallelizationWidth) + } else { + serializer = nil + } + } + } + + /// The serializer that backs ``maximumParallelizationWidth``. + /// + /// - Note: This serializer is ignored if ``isParallelizationEnabled`` is + /// `false`. + var serializer: Serializer? = Self._cpuCoreCount.flatMap { cpuCoreCount in + Serializer(maximumWidth: cpuCoreCount * 2) + } + /// How to symbolicate backtraces captured during a test run. /// /// If the value of this property is not `nil`, symbolication will be diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index 1cedf6182..29170550a 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -359,7 +359,11 @@ extension Runner { } await _forEach(in: testCases.enumerated(), namingTasksWith: taskNamer) { _, testCase in - await _runTestCase(testCase, within: step) + if _configuration.isParallelizationEnabled, let serializer = _configuration.serializer { + await serializer.run { await _runTestCase(testCase, within: step) } + } else { + await _runTestCase(testCase, within: step) + } } } diff --git a/Sources/Testing/Support/Serializer.swift b/Sources/Testing/Support/Serializer.swift new file mode 100644 index 000000000..b8c2cacf4 --- /dev/null +++ b/Sources/Testing/Support/Serializer.swift @@ -0,0 +1,65 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024–2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +/// A type whose instances can run a series of work items in strict order. +/// +/// When a work item is scheduled on an instance of this type, it runs after any +/// previously-scheduled work items. If it suspends, subsequently-scheduled work +/// items do not start running; they must wait until the suspended work item +/// either returns or throws an error. +final actor Serializer { + /// The maximum number of work items that may run concurrently. + nonisolated let maximumWidth: Int + + /// The number of scheduled work items, including any currently running. + private var _currentWidth = 0 + + /// Continuations for any scheduled work items that haven't started yet. + private var _continuations = [CheckedContinuation]() + + init(maximumWidth: Int = 1) { + self.maximumWidth = maximumWidth + } + + /// Run a work item serially after any previously-scheduled work items. + /// + /// - Parameters: + /// - workItem: A closure to run. + /// + /// - Returns: Whatever is returned from `workItem`. + /// + /// - Throws: Whatever is thrown by `workItem`. + func run(_ workItem: @Sendable @isolated(any) () async throws -> R) async rethrows -> R where R: Sendable { + _currentWidth += 1 + defer { + // Resume the next scheduled closure. + if !_continuations.isEmpty { + let continuation = _continuations.removeFirst() + continuation.resume() + } + + _currentWidth -= 1 + } + + await withCheckedContinuation { continuation in + if _currentWidth <= maximumWidth { + // Nothing else was scheduled, so we can resume immediately. + continuation.resume() + } else { + // Something was scheduled, so add the continuation to the + // list. When it resumes, we can run. + _continuations.append(continuation) + } + } + + return try await workItem() + } +} + From 85822e321701d008905f18a0ef52b5289f6db486 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 3 Nov 2025 21:32:20 -0500 Subject: [PATCH 02/11] Don't need a separate Darwin implementation --- Sources/Testing/Running/Configuration.swift | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/Sources/Testing/Running/Configuration.swift b/Sources/Testing/Running/Configuration.swift index 754dda02b..fa2d6f01a 100644 --- a/Sources/Testing/Running/Configuration.swift +++ b/Sources/Testing/Running/Configuration.swift @@ -25,21 +25,16 @@ public struct Configuration: Sendable { /// The number of CPU cores on the current system, or `nil` if that /// information is not available. private static var _cpuCoreCount: Int? { -#if SWT_TARGET_OS_APPLE - var result: Int32 = -1 - var mib: [Int32] = [CTL_HW, HW_NCPU] - var resultByteCount = MemoryLayout.stride - guard 0 == sysctl(&mib, UInt32(mib.count), &result, &resultByteCount, nil, 0) else { - return nil - } - return Int(result) -#elseif os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) return Int(sysconf(Int32(_SC_NPROCESSORS_CONF))) #elseif os(Windows) var siInfo = SYSTEM_INFO() GetSystemInfo(&siInfo) return Int(siInfo.dwNumberOfProcessors) +#elseif os(WASI) + return 1 #else +#warning("Platform-specific implementation missing: CPU core count unavailable") return nil #endif From 23224a2bc1b82ab386658919ff816e46c6de4a9e Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 3 Nov 2025 21:37:55 -0500 Subject: [PATCH 03/11] Guard against (hypothetical) bad CPU core counts from the OS --- Sources/Testing/Running/Configuration.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Testing/Running/Configuration.swift b/Sources/Testing/Running/Configuration.swift index fa2d6f01a..bab0ba2fe 100644 --- a/Sources/Testing/Running/Configuration.swift +++ b/Sources/Testing/Running/Configuration.swift @@ -65,7 +65,7 @@ public struct Configuration: Sendable { /// - Note: This serializer is ignored if ``isParallelizationEnabled`` is /// `false`. var serializer: Serializer? = Self._cpuCoreCount.flatMap { cpuCoreCount in - Serializer(maximumWidth: cpuCoreCount * 2) + Serializer(maximumWidth: max(1, cpuCoreCount) * 2) } /// How to symbolicate backtraces captured during a test run. From 5f9bbc52ef1c0913b25e2c6d40e64cf0846732c2 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 3 Nov 2025 21:45:09 -0500 Subject: [PATCH 04/11] Don't bother making the serializer an optional --- Sources/Testing/Running/Configuration.swift | 15 ++++++--------- Sources/Testing/Running/Runner.swift | 4 ++-- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/Sources/Testing/Running/Configuration.swift b/Sources/Testing/Running/Configuration.swift index bab0ba2fe..ff2c7d021 100644 --- a/Sources/Testing/Running/Configuration.swift +++ b/Sources/Testing/Running/Configuration.swift @@ -49,14 +49,10 @@ public struct Configuration: Sendable { @_spi(Experimental) public var maximumParallelizationWidth: Int { get { - serializer?.maximumWidth ?? .max + serializer.maximumWidth } set { - if newValue < .max { - serializer = Serializer(maximumWidth: maximumParallelizationWidth) - } else { - serializer = nil - } + serializer = Serializer(maximumWidth: newValue) } } @@ -64,9 +60,10 @@ public struct Configuration: Sendable { /// /// - Note: This serializer is ignored if ``isParallelizationEnabled`` is /// `false`. - var serializer: Serializer? = Self._cpuCoreCount.flatMap { cpuCoreCount in - Serializer(maximumWidth: max(1, cpuCoreCount) * 2) - } + var serializer: Serializer = { + let cpuCoreCount = Self._cpuCoreCount.map { max(1, $0) * 2 } ?? .max + return Serializer(maximumWidth: cpuCoreCount) + }() /// How to symbolicate backtraces captured during a test run. /// diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index 29170550a..4907040d3 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -359,8 +359,8 @@ extension Runner { } await _forEach(in: testCases.enumerated(), namingTasksWith: taskNamer) { _, testCase in - if _configuration.isParallelizationEnabled, let serializer = _configuration.serializer { - await serializer.run { await _runTestCase(testCase, within: step) } + if _configuration.isParallelizationEnabled { + await _configuration.serializer.run { await _runTestCase(testCase, within: step) } } else { await _runTestCase(testCase, within: step) } From 2b551e6ee6c2f00fdd595681225423fd3119dd1e Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 3 Nov 2025 21:48:54 -0500 Subject: [PATCH 05/11] Hit the task local less frequently --- Sources/Testing/Running/Runner.swift | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index 4907040d3..3abef29dc 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -344,8 +344,10 @@ extension Runner { /// If parallelization is supported and enabled, the generated test cases will /// be run in parallel using a task group. private static func _runTestCases(_ testCases: some Sequence, within step: Plan.Step) async { + let configuration = _configuration + // Apply the configuration's test case filter. - let testCaseFilter = _configuration.testCaseFilter + let testCaseFilter = configuration.testCaseFilter let testCases = testCases.lazy.filter { testCase in testCaseFilter(testCase, step.test) } @@ -359,8 +361,8 @@ extension Runner { } await _forEach(in: testCases.enumerated(), namingTasksWith: taskNamer) { _, testCase in - if _configuration.isParallelizationEnabled { - await _configuration.serializer.run { await _runTestCase(testCase, within: step) } + if configuration.isParallelizationEnabled { + await configuration.serializer.run { await _runTestCase(testCase, within: step) } } else { await _runTestCase(testCase, within: step) } From 5e47794b7c005ea0469490312d64b55d87f4ce09 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Fri, 7 Nov 2025 15:55:06 -0500 Subject: [PATCH 06/11] Incorporate feedback --- Sources/Testing/Running/Configuration.swift | 39 +++----------------- Sources/Testing/Running/Runner.swift | 40 +++++++++++++++------ Sources/Testing/Support/Serializer.swift | 23 +++++++++++- 3 files changed, 55 insertions(+), 47 deletions(-) diff --git a/Sources/Testing/Running/Configuration.swift b/Sources/Testing/Running/Configuration.swift index ff2c7d021..61dc2dd37 100644 --- a/Sources/Testing/Running/Configuration.swift +++ b/Sources/Testing/Running/Configuration.swift @@ -22,48 +22,17 @@ public struct Configuration: Sendable { /// Whether or not to parallelize the execution of tests and test cases. public var isParallelizationEnabled: Bool = true - /// The number of CPU cores on the current system, or `nil` if that - /// information is not available. - private static var _cpuCoreCount: Int? { -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) - return Int(sysconf(Int32(_SC_NPROCESSORS_CONF))) -#elseif os(Windows) - var siInfo = SYSTEM_INFO() - GetSystemInfo(&siInfo) - return Int(siInfo.dwNumberOfProcessors) -#elseif os(WASI) - return 1 -#else -#warning("Platform-specific implementation missing: CPU core count unavailable") - return nil -#endif - - } - /// The maximum width of parallelization. /// /// The value of this property determines how many tests (or rather, test /// cases) will run in parallel. The default value of this property is equal /// to twice the number of CPU cores reported by the operating system, or /// `Int.max` if that value is not available. - @_spi(Experimental) - public var maximumParallelizationWidth: Int { - get { - serializer.maximumWidth - } - set { - serializer = Serializer(maximumWidth: newValue) - } - } - - /// The serializer that backs ``maximumParallelizationWidth``. /// - /// - Note: This serializer is ignored if ``isParallelizationEnabled`` is - /// `false`. - var serializer: Serializer = { - let cpuCoreCount = Self._cpuCoreCount.map { max(1, $0) * 2 } ?? .max - return Serializer(maximumWidth: cpuCoreCount) - }() + /// If the value of ``isParallelizationEnabled`` is `false`, this property has + /// no effect. + @_spi(Experimental) + public var maximumParallelizationWidth: Int = cpuCoreCount.map { max(1, $0) * 2 } ?? .max /// How to symbolicate backtraces captured during a test run. /// diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index 3abef29dc..f67d2446f 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -66,6 +66,12 @@ extension Runner { .current ?? .init() } + /// Context to apply to a test run. + struct Context: Sendable { + /// A serializer used to reduce parallelism among test cases. + var testCaseSerializer: Serializer + } + /// Apply the custom scope for any test scope providers of the traits /// associated with a specified test by calling their /// ``TestScoping/provideScope(for:testCase:performing:)`` function. @@ -179,6 +185,7 @@ extension Runner { /// /// - Parameters: /// - stepGraph: The subgraph whose root value, a step, is to be run. + /// - context: Context for the test run. /// /// - Throws: Whatever is thrown from the test body. Thrown errors are /// normally reported as test failures. @@ -193,7 +200,7 @@ extension Runner { /// ## See Also /// /// - ``Runner/run()`` - private static func _runStep(atRootOf stepGraph: Graph) async throws { + private static func _runStep(atRootOf stepGraph: Graph, context: Context) async throws { // Whether to send a `.testEnded` event at the end of running this step. // Some steps' actions may not require a final event to be sent — for // example, a skip event only sends `.testSkipped`. @@ -250,18 +257,18 @@ extension Runner { try await _applyScopingTraits(for: step.test, testCase: nil) { // Run the test function at this step (if one is present.) if let testCases = step.test.testCases { - await _runTestCases(testCases, within: step) + await _runTestCases(testCases, within: step, context: context) } // Run the children of this test (i.e. the tests in this suite.) - try await _runChildren(of: stepGraph) + try await _runChildren(of: stepGraph, context: context) } } } } else { // There is no test at this node in the graph, so just skip down to the // child nodes. - try await _runChildren(of: stepGraph) + try await _runChildren(of: stepGraph, context: context) } } @@ -286,10 +293,11 @@ extension Runner { /// - Parameters: /// - stepGraph: The subgraph whose root value, a step, will be used to /// find children to run. + /// - context: Context for the test run. /// /// - Throws: Whatever is thrown from the test body. Thrown errors are /// normally reported as test failures. - private static func _runChildren(of stepGraph: Graph) async throws { + private static func _runChildren(of stepGraph: Graph, context: Context) async throws { let childGraphs = if _configuration.isParallelizationEnabled { // Explicitly shuffle the steps to help detect accidental dependencies // between tests due to their ordering. @@ -331,7 +339,7 @@ extension Runner { // Run the child nodes. try await _forEach(in: childGraphs.lazy.map(\.value), namingTasksWith: taskNamer) { childGraph in - try await _runStep(atRootOf: childGraph) + try await _runStep(atRootOf: childGraph, context: context) } } @@ -340,10 +348,11 @@ extension Runner { /// - Parameters: /// - testCases: The test cases to be run. /// - step: The runner plan step associated with this test case. + /// - context: Context for the test run. /// /// If parallelization is supported and enabled, the generated test cases will /// be run in parallel using a task group. - private static func _runTestCases(_ testCases: some Sequence, within step: Plan.Step) async { + private static func _runTestCases(_ testCases: some Sequence, within step: Plan.Step, context: Context) async { let configuration = _configuration // Apply the configuration's test case filter. @@ -362,9 +371,9 @@ extension Runner { await _forEach(in: testCases.enumerated(), namingTasksWith: taskNamer) { _, testCase in if configuration.isParallelizationEnabled { - await configuration.serializer.run { await _runTestCase(testCase, within: step) } + await context.testCaseSerializer.run { await _runTestCase(testCase, within: step, context: context) } } else { - await _runTestCase(testCase, within: step) + await _runTestCase(testCase, within: step, context: context) } } } @@ -374,10 +383,11 @@ extension Runner { /// - Parameters: /// - testCase: The test case to run. /// - step: The runner plan step associated with this test case. + /// - context: Context for the test run. /// /// This function sets ``Test/Case/current``, then invokes the test case's /// body closure. - private static func _runTestCase(_ testCase: Test.Case, within step: Plan.Step) async { + private static func _runTestCase(_ testCase: Test.Case, within step: Plan.Step, context: Context) async { let configuration = _configuration Event.post(.testCaseStarted, for: (step.test, testCase), configuration: configuration) @@ -434,6 +444,14 @@ extension Runner { eventHandler(event, context) } + // Context to pass into the test run. We intentionally don't pass the Runner + // itself (implicitly as `self` nor as an argument) because we don't want to + // accidentally depend on e.g. the `configuration` property rather than the + // current configuration. + let context = Context( + testCaseSerializer: Serializer(maximumWidth: runner.configuration.maximumParallelizationWidth) + ) + await Configuration.withCurrent(runner.configuration) { // Post an event for every test in the test plan being run. These events // are turned into JSON objects if JSON output is enabled. @@ -460,7 +478,7 @@ extension Runner { taskAction = "running iteration #\(iterationIndex + 1)" } _ = taskGroup.addTaskUnlessCancelled(name: decorateTaskName("test run", withAction: taskAction)) { - try? await _runStep(atRootOf: runner.plan.stepGraph) + try? await _runStep(atRootOf: runner.plan.stepGraph, context: context) } await taskGroup.waitForAll() } diff --git a/Sources/Testing/Support/Serializer.swift b/Sources/Testing/Support/Serializer.swift index b8c2cacf4..80ed187b0 100644 --- a/Sources/Testing/Support/Serializer.swift +++ b/Sources/Testing/Support/Serializer.swift @@ -8,12 +8,33 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // +private import _TestingInternals + +/// The number of CPU cores on the current system, or `nil` if that +/// information is not available. +var cpuCoreCount: Int? { +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) + return Int(sysconf(Int32(_SC_NPROCESSORS_CONF))) +#elseif os(Windows) + var siInfo = SYSTEM_INFO() + GetSystemInfo(&siInfo) + return Int(siInfo.dwNumberOfProcessors) +#elseif os(WASI) + return 1 +#else +#warning("Platform-specific implementation missing: CPU core count unavailable") + return nil +#endif +} + /// A type whose instances can run a series of work items in strict order. /// /// When a work item is scheduled on an instance of this type, it runs after any /// previously-scheduled work items. If it suspends, subsequently-scheduled work /// items do not start running; they must wait until the suspended work item /// either returns or throws an error. +/// +/// This type is not part of the public interface of the testing library. final actor Serializer { /// The maximum number of work items that may run concurrently. nonisolated let maximumWidth: Int @@ -36,7 +57,7 @@ final actor Serializer { /// - Returns: Whatever is returned from `workItem`. /// /// - Throws: Whatever is thrown by `workItem`. - func run(_ workItem: @Sendable @isolated(any) () async throws -> R) async rethrows -> R where R: Sendable { + func run(_ workItem: @isolated(any) @Sendable () async throws -> R) async rethrows -> R where R: Sendable { _currentWidth += 1 defer { // Resume the next scheduled closure. From 56a354ec19c80f82ce0f27ea999194864195a8c9 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 10 Nov 2025 16:37:52 -0500 Subject: [PATCH 07/11] Add width checks --- Sources/Testing/ABI/EntryPoints/EntryPoint.swift | 4 +++- Sources/Testing/Support/Serializer.swift | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index 0b824394c..734a1bf74 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -556,7 +556,9 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr // Parallelization (on by default) configuration.isParallelizationEnabled = args.parallel ?? true if let maximumParallelizationWidth = args.experimentalMaximumParallelizationWidth { - try! FileHandle.stderr.write("MAX WIDTH: \(maximumParallelizationWidth)\n") + if maximumParallelizationWidth < 1 { + throw _EntryPointError.invalidArgument("--experimental-maximum-parallelization-width", value: String(describing: maximumParallelizationWidth)) + } configuration.maximumParallelizationWidth = maximumParallelizationWidth } diff --git a/Sources/Testing/Support/Serializer.swift b/Sources/Testing/Support/Serializer.swift index 80ed187b0..60632229c 100644 --- a/Sources/Testing/Support/Serializer.swift +++ b/Sources/Testing/Support/Serializer.swift @@ -46,6 +46,7 @@ final actor Serializer { private var _continuations = [CheckedContinuation]() init(maximumWidth: Int = 1) { + precondition(maximumWidth >= 1, "Invalid serializer width \(maximumWidth).") self.maximumWidth = maximumWidth } From dc662080c899362f5b0afd5838ba37a142a749c7 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 10 Nov 2025 17:07:19 -0500 Subject: [PATCH 08/11] Don't use a serializer if the user doesn't explicitly opt in, add some test coverage --- Sources/Testing/Running/Configuration.swift | 24 ++++++++++++---- Sources/Testing/Running/Runner.swift | 31 +++++++++++++-------- Sources/Testing/Support/Serializer.swift | 6 ++++ Tests/TestingTests/SwiftPMTests.swift | 19 +++++++++++++ 4 files changed, 64 insertions(+), 16 deletions(-) diff --git a/Sources/Testing/Running/Configuration.swift b/Sources/Testing/Running/Configuration.swift index 61dc2dd37..ebd9fca07 100644 --- a/Sources/Testing/Running/Configuration.swift +++ b/Sources/Testing/Running/Configuration.swift @@ -20,19 +20,33 @@ public struct Configuration: Sendable { // MARK: - Parallelization /// Whether or not to parallelize the execution of tests and test cases. - public var isParallelizationEnabled: Bool = true + /// + /// - Note: Setting the value of this property implicitly sets the value of + /// the experimental ``maximumParallelizationWidth`` property. + public var isParallelizationEnabled: Bool { + get { + maximumParallelizationWidth > 1 + } + set { + maximumParallelizationWidth = newValue ? defaultParallelizationWidth : 1 + } + } /// The maximum width of parallelization. /// /// The value of this property determines how many tests (or rather, test - /// cases) will run in parallel. The default value of this property is equal - /// to twice the number of CPU cores reported by the operating system, or - /// `Int.max` if that value is not available. + /// cases) will run in parallel. + /// + /// @Comment { + /// The default value of this property is equal to twice the number of CPU + /// cores reported by the operating system, or `Int.max` if that value is + /// not available. + /// } /// /// If the value of ``isParallelizationEnabled`` is `false`, this property has /// no effect. @_spi(Experimental) - public var maximumParallelizationWidth: Int = cpuCoreCount.map { max(1, $0) * 2 } ?? .max + public var maximumParallelizationWidth: Int = defaultParallelizationWidth /// How to symbolicate backtraces captured during a test run. /// diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index ccd70eab4..911a68316 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -67,9 +67,9 @@ extension Runner { } /// Context to apply to a test run. - struct Context: Sendable { + private struct _Context: Sendable { /// A serializer used to reduce parallelism among test cases. - var testCaseSerializer: Serializer + var testCaseSerializer: Serializer? } /// Apply the custom scope for any test scope providers of the traits @@ -200,7 +200,7 @@ extension Runner { /// ## See Also /// /// - ``Runner/run()`` - private static func _runStep(atRootOf stepGraph: Graph, context: Context) async throws { + private static func _runStep(atRootOf stepGraph: Graph, context: _Context) async throws { // Whether to send a `.testEnded` event at the end of running this step. // Some steps' actions may not require a final event to be sent — for // example, a skip event only sends `.testSkipped`. @@ -297,7 +297,7 @@ extension Runner { /// /// - Throws: Whatever is thrown from the test body. Thrown errors are /// normally reported as test failures. - private static func _runChildren(of stepGraph: Graph, context: Context) async throws { + private static func _runChildren(of stepGraph: Graph, context: _Context) async throws { let childGraphs = if _configuration.isParallelizationEnabled { // Explicitly shuffle the steps to help detect accidental dependencies // between tests due to their ordering. @@ -352,7 +352,7 @@ extension Runner { /// /// If parallelization is supported and enabled, the generated test cases will /// be run in parallel using a task group. - private static func _runTestCases(_ testCases: some Sequence, within step: Plan.Step, context: Context) async { + private static func _runTestCases(_ testCases: some Sequence, within step: Plan.Step, context: _Context) async { let configuration = _configuration // Apply the configuration's test case filter. @@ -370,8 +370,10 @@ extension Runner { } await _forEach(in: testCases.enumerated(), namingTasksWith: taskNamer) { _, testCase in - if configuration.isParallelizationEnabled { - await context.testCaseSerializer.run { await _runTestCase(testCase, within: step, context: context) } + if let testCaseSerializer = context.testCaseSerializer { + // Note that if .serialized is applied to an inner scope, we still use + // this serializer (if set) so that we don't overcommit. + await testCaseSerializer.run { await _runTestCase(testCase, within: step, context: context) } } else { await _runTestCase(testCase, within: step, context: context) } @@ -387,7 +389,7 @@ extension Runner { /// /// This function sets ``Test/Case/current``, then invokes the test case's /// body closure. - private static func _runTestCase(_ testCase: Test.Case, within step: Plan.Step, context: Context) async { + private static func _runTestCase(_ testCase: Test.Case, within step: Plan.Step, context: _Context) async { let configuration = _configuration Event.post(.testCaseStarted, for: (step.test, testCase), configuration: configuration) @@ -451,9 +453,16 @@ extension Runner { // itself (implicitly as `self` nor as an argument) because we don't want to // accidentally depend on e.g. the `configuration` property rather than the // current configuration. - let context = Context( - testCaseSerializer: Serializer(maximumWidth: runner.configuration.maximumParallelizationWidth) - ) + let context: _Context = { + var context = _Context() + + let maximumParallelizationWidth = runner.configuration.maximumParallelizationWidth + if maximumParallelizationWidth > 1 && maximumParallelizationWidth < .max { + context.testCaseSerializer = Serializer(maximumWidth: runner.configuration.maximumParallelizationWidth) + } + + return context + }() await Configuration.withCurrent(runner.configuration) { // Post an event for every test in the test plan being run. These events diff --git a/Sources/Testing/Support/Serializer.swift b/Sources/Testing/Support/Serializer.swift index 60632229c..96adfca7c 100644 --- a/Sources/Testing/Support/Serializer.swift +++ b/Sources/Testing/Support/Serializer.swift @@ -27,6 +27,12 @@ var cpuCoreCount: Int? { #endif } +/// The default parallelization width when parallelized testing is enabled. +var defaultParallelizationWidth: Int { + // cpuCoreCount.map { max(1, $0) * 2 } ?? .max + .max +} + /// A type whose instances can run a series of work items in strict order. /// /// When a work item is scheduled on an instance of this type, it runs after any diff --git a/Tests/TestingTests/SwiftPMTests.swift b/Tests/TestingTests/SwiftPMTests.swift index 4668fbb25..2d7001e8f 100644 --- a/Tests/TestingTests/SwiftPMTests.swift +++ b/Tests/TestingTests/SwiftPMTests.swift @@ -59,6 +59,25 @@ struct SwiftPMTests { #expect(!configuration.isParallelizationEnabled) } + @Test("--experimental-maximum-parallelization-width argument") + func maximumParallelizationWidth() throws { + var configuration = try configurationForEntryPoint(withArguments: ["PATH", "--experimental-maximum-parallelization-width", "12345"]) + #expect(configuration.isParallelizationEnabled) + #expect(configuration.maximumParallelizationWidth == 12345) + + configuration = try configurationForEntryPoint(withArguments: ["PATH", "--experimental-maximum-parallelization-width", "1"]) + #expect(!configuration.isParallelizationEnabled) + #expect(configuration.maximumParallelizationWidth == 1) + + configuration = try configurationForEntryPoint(withArguments: ["PATH", "--experimental-maximum-parallelization-width", "\(Int.max)"]) + #expect(configuration.isParallelizationEnabled) + #expect(configuration.maximumParallelizationWidth == .max) + + #expect(throws: (any Error).self) { + _ = try configurationForEntryPoint(withArguments: ["PATH", "--experimental-maximum-parallelization-width", "0"]) + } + } + @Test("--symbolicate-backtraces argument", arguments: [ (String?.none, Backtrace.SymbolicationMode?.none), From e03b747ea46266dd08441c7ae14d1ee8c9bdd0a7 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 10 Nov 2025 17:08:00 -0500 Subject: [PATCH 09/11] Update comment --- Sources/Testing/Running/Configuration.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/Testing/Running/Configuration.swift b/Sources/Testing/Running/Configuration.swift index ebd9fca07..e0fe009ba 100644 --- a/Sources/Testing/Running/Configuration.swift +++ b/Sources/Testing/Running/Configuration.swift @@ -43,8 +43,8 @@ public struct Configuration: Sendable { /// not available. /// } /// - /// If the value of ``isParallelizationEnabled`` is `false`, this property has - /// no effect. + /// - Note: Setting the value of this property implicitly sets the value of + /// the ``isParallelizationEnabled`` property. @_spi(Experimental) public var maximumParallelizationWidth: Int = defaultParallelizationWidth From 253aebb5bd5b5e20a880f59aac37215376ab45ee Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 10 Nov 2025 17:19:38 -0500 Subject: [PATCH 10/11] --no-parallel and --experimental-maximum-parallelization-width conflict --- Sources/Testing/ABI/EntryPoints/EntryPoint.swift | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index 734a1bf74..52a59e7fa 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -488,9 +488,7 @@ func parseCommandLineArguments(from args: [String]) throws -> __CommandLineArgum // Parallelization (on by default) if args.contains("--no-parallel") { result.parallel = false - } - if let maximumParallelizationWidth = args.argumentValue(forLabel: "--experimental-maximum-parallelization-width").flatMap(Int.init) - ?? Environment.variable(named: "SWT_EXPERIMENTAL_MAXIMUM_PARALLELIZATION_WIDTH").flatMap(Int.init) { + } else if let maximumParallelizationWidth = Environment.variable(named: "SWT_EXPERIMENTAL_MAXIMUM_PARALLELIZATION_WIDTH").flatMap(Int.init) { // TODO: decide if we want to repurpose --num-workers for this use case? result.experimentalMaximumParallelizationWidth = maximumParallelizationWidth } From 781cc58a785a6bada6375bac0fe9904a37b5dea2 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 10 Nov 2025 17:30:03 -0500 Subject: [PATCH 11/11] Lower the environment variable check so that Xcode 26 (which always sets parallel = true/false) doesn't disable it unintentionally --- Sources/Testing/ABI/EntryPoints/EntryPoint.swift | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index 52a59e7fa..4ef541ba9 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -210,9 +210,6 @@ public struct __CommandLineArguments_v0: Sendable { /// The value of the `--parallel` or `--no-parallel` argument. public var parallel: Bool? - /// The maximum number of test tasks to run in parallel. - public var experimentalMaximumParallelizationWidth: Int? - /// The value of the `--symbolicate-backtraces` argument. public var symbolicateBacktraces: String? @@ -339,7 +336,6 @@ extension __CommandLineArguments_v0: Codable { enum CodingKeys: String, CodingKey { case listTests case parallel - case experimentalMaximumParallelizationWidth case symbolicateBacktraces case verbose case veryVerbose @@ -488,9 +484,6 @@ func parseCommandLineArguments(from args: [String]) throws -> __CommandLineArgum // Parallelization (on by default) if args.contains("--no-parallel") { result.parallel = false - } else if let maximumParallelizationWidth = Environment.variable(named: "SWT_EXPERIMENTAL_MAXIMUM_PARALLELIZATION_WIDTH").flatMap(Int.init) { - // TODO: decide if we want to repurpose --num-workers for this use case? - result.experimentalMaximumParallelizationWidth = maximumParallelizationWidth } // Whether or not to symbolicate backtraces in the event stream. @@ -552,8 +545,11 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr var configuration = Configuration() // Parallelization (on by default) - configuration.isParallelizationEnabled = args.parallel ?? true - if let maximumParallelizationWidth = args.experimentalMaximumParallelizationWidth { + if let parallel = args.parallel { + configuration.isParallelizationEnabled = parallel + } + if let maximumParallelizationWidth = Environment.variable(named: "SWT_EXPERIMENTAL_MAXIMUM_PARALLELIZATION_WIDTH").flatMap(Int.init) { + // TODO: decide if we want to repurpose --num-workers for this use case? if maximumParallelizationWidth < 1 { throw _EntryPointError.invalidArgument("--experimental-maximum-parallelization-width", value: String(describing: maximumParallelizationWidth)) }