Allow opting in to using the new SwiftPM BSP when opening a package

Author: owenv

Documentation/Configuration File.md

@@ -30,6 +30,7 @@ The structure of the file is currently not guaranteed to be stable. Options may
   - `linkerFlags: string[]`: Extra arguments passed to the linker. Equivalent to SwiftPM's `-Xlinker` option.
   - `buildToolsSwiftCompilerFlags: string[]`: Extra arguments passed to the compiler for Swift files or plugins. Equivalent to SwiftPM's `-Xbuild-tools-swiftc` option.
   - `disableSandbox: boolean`: Disables running subprocesses from SwiftPM in a sandbox. Equivalent to SwiftPM's `--disable-sandbox` option. Useful when running `sourcekit-lsp` in a sandbox because nested sandboxes are not supported.
+  - `buildSystem: "native"|"swiftbuild"`: Which SwiftPM build system should be used when opening a package.
 - `compilationDatabase`: Dictionary with the following keys, defining options for workspaces with a compilation database.
   - `searchPaths: string[]`: Additional paths to search for a compilation database, relative to a workspace root.
 - `fallbackBuildSystem`: Dictionary with the following keys, defining options for files that aren't managed by any build server.

Sources/BuildServerIntegration/BuildServerManager.swift

@@ -280,22 +280,43 @@ private extension BuildServerSpec {
         )
       }
     case .swiftPM:
-      #if !NO_SWIFTPM_DEPENDENCY
-      return await createBuiltInBuildServerAdapter(
-        messagesToSourceKitLSPHandler: messagesToSourceKitLSPHandler,
-        buildServerHooks: buildServerHooks
-      ) { connectionToSourceKitLSP in
-        try await SwiftPMBuildServer(
-          projectRoot: projectRoot,
-          toolchainRegistry: toolchainRegistry,
-          options: options,
-          connectionToSourceKitLSP: connectionToSourceKitLSP,
-          testHooks: buildServerHooks.swiftPMTestHooks
-        )
+      switch options.swiftPMOrDefault.buildSystem {
+      case .swiftbuild:
+        let buildServer = await orLog("Creating external SwiftPM build server") {
+          try await ExternalBuildServerAdapter(
+            projectRoot: projectRoot,
+            config: BuildServerConfig.forSwiftPMBuildServer(
+              projectRoot: projectRoot,
+              swiftPMOptions: options.swiftPMOrDefault,
+              toolchainRegistry: toolchainRegistry
+            ),
+            messagesToSourceKitLSPHandler: messagesToSourceKitLSPHandler
+          )
+        }
+        guard let buildServer else {
+          logger.log("Failed to create external SwiftPM build server at \(projectRoot)")
+          return nil
+        }
+        logger.log("Created external SwiftPM build server at \(projectRoot)")
+        return .external(buildServer)
+      case .native, nil:
+        #if !NO_SWIFTPM_DEPENDENCY
+        return await createBuiltInBuildServerAdapter(
+          messagesToSourceKitLSPHandler: messagesToSourceKitLSPHandler,
+          buildServerHooks: buildServerHooks
+        ) { connectionToSourceKitLSP in
+          try await SwiftPMBuildServer(
+            projectRoot: projectRoot,
+            toolchainRegistry: toolchainRegistry,
+            options: options,
+            connectionToSourceKitLSP: connectionToSourceKitLSP,
+            testHooks: buildServerHooks.swiftPMTestHooks
+          )
+        }
+        #else
+        return nil
+        #endif
       }
-      #else
-      return nil
-      #endif
     case .injected(let injector):
       let connectionToSourceKitLSP = LocalConnection(
         receiverName: "BuildServerManager for \(projectRoot.lastPathComponent)",

Sources/BuildServerIntegration/ExternalBuildServerAdapter.swift

@@ -19,6 +19,7 @@ import Foundation
 import SKOptions
 import SwiftExtensions
 import TSCExtensions
+import ToolchainRegistry
 
 import func TSCBasic.getEnvSearchPaths
 import var TSCBasic.localFileSystem
@@ -61,7 +62,7 @@ enum BuildServerNotFoundError: Error {
 /// BSP configuration
 ///
 /// See https://build-server-protocol.github.io/docs/overview/server-discovery#the-bsp-connection-details
-private struct BuildServerConfig: Codable {
+struct BuildServerConfig: Codable {
   /// The name of the build tool.
   let name: String
 
@@ -82,15 +83,92 @@ private struct BuildServerConfig: Codable {
     let fileData = try Data(contentsOf: path)
     return try decoder.decode(BuildServerConfig.self, from: fileData)
   }
+
+  static func forSwiftPMBuildServer(
+    projectRoot: URL,
+    swiftPMOptions: SourceKitLSPOptions.SwiftPMOptions,
+    toolchainRegistry: ToolchainRegistry
+  ) async throws -> BuildServerConfig {
+    let toolchain = await toolchainRegistry.preferredToolchain(containing: [\.swift])
+    guard let swiftPath = try toolchain?.swift?.filePath else {
+      throw ExecutableNotFoundError(executableName: "swift")
+    }
+    var args: [String] = [swiftPath, "package", "experimental-build-server"]
+    // The build server requires use of the Swift Build backend.
+    args.append(contentsOf: ["--build-system", "swiftbuild"])
+    // Explicitly specify the package path.
+    try args.append(contentsOf: ["--package-path", projectRoot.filePath])
+    // Map LSP SwiftPM options to build server flags
+    if let configuration = swiftPMOptions.configuration {
+      args.append(contentsOf: ["--configuration", configuration.rawValue])
+    }
+    if let scratchPath = swiftPMOptions.scratchPath {
+      args.append(contentsOf: ["--scratch-path", scratchPath])
+    }
+    if let swiftSDKsDirectory = swiftPMOptions.swiftSDKsDirectory {
+      args.append(contentsOf: ["--swift-sdks-path", swiftSDKsDirectory])
+    }
+    if let swiftSDK = swiftPMOptions.swiftSDK {
+      args.append(contentsOf: ["--swift-sdk", swiftSDK])
+    }
+    if let triple = swiftPMOptions.triple {
+      args.append(contentsOf: ["--triple", triple])
+    }
+    if let toolsets = swiftPMOptions.toolsets {
+      for toolset in toolsets {
+        args.append(contentsOf: ["--toolset", toolset])
+      }
+    }
+    if let traits = swiftPMOptions.traits {
+      args.append(contentsOf: ["--traits", traits.joined(separator: ",")])
+    }
+    if let cCompilerFlags = swiftPMOptions.cCompilerFlags {
+      for flag in cCompilerFlags {
+        args.append(contentsOf: ["-Xcc", flag])
+      }
+    }
+    if let cxxCompilerFlags = swiftPMOptions.cxxCompilerFlags {
+      for flag in cxxCompilerFlags {
+        args.append(contentsOf: ["-Xcxx", flag])
+      }
+    }
+    if let swiftCompilerFlags = swiftPMOptions.swiftCompilerFlags {
+      for flag in swiftCompilerFlags {
+        args.append(contentsOf: ["-Xswiftc", flag])
+      }
+    }
+    if let linkerFlags = swiftPMOptions.linkerFlags {
+      for flag in linkerFlags {
+        args.append(contentsOf: ["-Xlinker", flag])
+      }
+    }
+    if let buildToolsSwiftCompilerFlags = swiftPMOptions.buildToolsSwiftCompilerFlags {
+      for flag in buildToolsSwiftCompilerFlags {
+        args.append(contentsOf: ["-Xbuild-tools-swiftc", flag])
+      }
+    }
+    if swiftPMOptions.disableSandbox == true {
+      args.append("--disable-sandbox")
+    }
+    // The skipPlugins option isn't currently respected because the underlying build server does not support it.
+    // We may want to reconsider this in the future, or remove the option entirely.
+    return BuildServerConfig(
+      name: "SwiftPM Build Server",
+      version: "",
+      bspVersion: "2.2.0",
+      languages: [Language.c, .cpp, .objective_c, .objective_cpp, .swift].map(\.rawValue),
+      argv: args
+    )
+  }
 }
 
 /// Launches a subprocess that is a BSP server and manages the process's lifetime.
 actor ExternalBuildServerAdapter {
   /// The root folder of the project. Used to resolve relative server paths.
   private let projectRoot: URL
 
-  /// The file that specifies the configuration for this build server.
-  private let configPath: URL
+  /// The configuration for this build server.
+  private let serverConfig: BuildServerConfig
 
   /// The `BuildServerManager` that handles messages from the BSP server to SourceKit-LSP.
   var messagesToSourceKitLSPHandler: any MessageHandler
@@ -123,15 +201,28 @@ actor ExternalBuildServerAdapter {
 
   init(
     projectRoot: URL,
-    configPath: URL,
+    config: BuildServerConfig,
     messagesToSourceKitLSPHandler: any MessageHandler
   ) async throws {
     self.projectRoot = projectRoot
-    self.configPath = configPath
+    self.serverConfig = config
     self.messagesToSourceKitLSPHandler = messagesToSourceKitLSPHandler
     self.connectionToBuildServer = try await self.createConnectionToBspServer()
   }
 
+  init(
+    projectRoot: URL,
+    configPath: URL,
+    messagesToSourceKitLSPHandler: any MessageHandler
+  ) async throws {
+    let serverConfig = try BuildServerConfig.load(from: configPath)
+    try await self.init(
+      projectRoot: projectRoot,
+      config: serverConfig,
+      messagesToSourceKitLSPHandler: messagesToSourceKitLSPHandler
+    )
+  }
+
   /// Change the handler that handles messages from the build server.
   ///
   /// The intended use of this is to intercept messages from the build server by `LegacyBuildServer`.
@@ -165,7 +256,6 @@ actor ExternalBuildServerAdapter {
 
   /// Create a new JSONRPCConnection to the build server.
   private func createConnectionToBspServer() async throws -> JSONRPCConnection {
-    let serverConfig = try BuildServerConfig.load(from: configPath)
     var serverPath = URL(fileURLWithPath: serverConfig.argv[0], relativeTo: projectRoot.ensuringCorrectTrailingSlash)
     var serverArgs = Array(serverConfig.argv[1...])
 

Sources/SKOptions/CMakeLists.txt

@@ -3,6 +3,7 @@ add_library(SKOptions STATIC
   BuildConfiguration.swift
   ExperimentalFeatures.swift
   SourceKitLSPOptions.swift
+  SwiftPMBuildSystem.swift
   WorkspaceType.swift)
 set_target_properties(SKOptions PROPERTIES
   INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY})

Sources/SKOptions/SourceKitLSPOptions.swift

@@ -76,6 +76,9 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable {
     ///   background indexing.
     public var skipPlugins: Bool?
 
+    /// Which SwiftPM build system should be used when opening a package.
+    public var buildSystem: SwiftPMBuildSystem?
+
     public init(
       configuration: BuildConfiguration? = nil,
       scratchPath: String? = nil,
@@ -90,7 +93,8 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable {
       linkerFlags: [String]? = nil,
       buildToolsSwiftCompilerFlags: [String]? = nil,
       disableSandbox: Bool? = nil,
-      skipPlugins: Bool? = nil
+      skipPlugins: Bool? = nil,
+      buildSystem: SwiftPMBuildSystem? = nil
     ) {
       self.configuration = configuration
       self.scratchPath = scratchPath
@@ -105,6 +109,7 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable {
       self.linkerFlags = linkerFlags
       self.buildToolsSwiftCompilerFlags = buildToolsSwiftCompilerFlags
       self.disableSandbox = disableSandbox
+      self.buildSystem = buildSystem
     }
 
     static func merging(base: SwiftPMOptions, override: SwiftPMOptions?) -> SwiftPMOptions {
@@ -122,7 +127,8 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable {
         linkerFlags: override?.linkerFlags ?? base.linkerFlags,
         buildToolsSwiftCompilerFlags: override?.buildToolsSwiftCompilerFlags ?? base.buildToolsSwiftCompilerFlags,
         disableSandbox: override?.disableSandbox ?? base.disableSandbox,
-        skipPlugins: override?.skipPlugins ?? base.skipPlugins
+        skipPlugins: override?.skipPlugins ?? base.skipPlugins,
+        buildSystem: override?.buildSystem ?? base.buildSystem
       )
     }
   }

Sources/SKOptions/SwiftPMBuildSystem.swift

@@ -0,0 +1,16 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+public enum SwiftPMBuildSystem: String, Codable, Sendable {
+  case native
+  case swiftbuild
+}

Tests/BuildServerIntegrationTests/SwiftPMBuildServerTests.swift

@@ -47,6 +47,12 @@ private var hostTriple: Triple {
   }
 }
 
+fileprivate extension SourceKitLSPOptions {
+  static var forTestingExperimentalSwiftPMBuildServer: Self {
+    SourceKitLSPOptions(swiftPM: SwiftPMOptions(buildSystem: .swiftbuild))
+  }
+}
+
 @Suite(.serialized, .configureLogging)
 struct SwiftPMBuildServerTests {
   @Test
@@ -132,8 +138,8 @@ struct SwiftPMBuildServerTests {
     }
   }
 
-  @Test
-  func testBasicSwiftArgs() async throws {
+  @Test(arguments: [SourceKitLSPOptions(), .forTestingExperimentalSwiftPMBuildServer])
+  func testBasicSwiftArgs(options: SourceKitLSPOptions) async throws {
     try await withTestScratchDir { tempDir in
       try FileManager.default.createFiles(
         root: tempDir,
@@ -153,7 +159,7 @@ struct SwiftPMBuildServerTests {
       let buildServerManager = await BuildServerManager(
         buildServerSpec: .swiftpmSpec(for: packageRoot),
         toolchainRegistry: .forTesting,
-        options: SourceKitLSPOptions(),
+        options: options,
         connectionToClient: DummyBuildServerManagerConnectionToClient(),
         buildServerHooks: BuildServerHooks(),
         createMainFilesProvider: { _, _ in nil }
@@ -181,18 +187,34 @@ struct SwiftPMBuildServerTests {
       expectArgumentsContain("-target", arguments: arguments)  // Only one!
       #if os(macOS)
       let versionString = PackageModel.Platform.macOS.oldestSupportedVersion.versionString
+      if options.swiftPMOrDefault.buildSystem == .swiftbuild {
+        expectArgumentsContain(
+          "-target",
+          // Account for differences in macOS naming canonicalization
+          try await hostTriple.tripleString(forPlatformVersion: versionString).replacing("macosx", with: "macos"),
+          arguments: arguments
+        )
+      } else {
+        expectArgumentsContain(
+          "-target",
+          try await hostTriple.tripleString(forPlatformVersion: versionString),
+          arguments: arguments
+        )
+      }
       expectArgumentsContain(
-        "-target",
-        try await hostTriple.tripleString(forPlatformVersion: versionString),
-        arguments: arguments
+        "-sdk",
+        arguments: arguments,
+        allowMultiple: options.swiftPMOrDefault.buildSystem == .swiftbuild
       )
-      expectArgumentsContain("-sdk", arguments: arguments)
       expectArgumentsContain("-F", arguments: arguments, allowMultiple: true)
       #else
       expectArgumentsContain("-target", try await hostTriple.tripleString, arguments: arguments)
       #endif
 
-      expectArgumentsContain("-I", try build.appending(component: "Modules").filePath, arguments: arguments)
+      if options.swiftPMOrDefault.buildSystem != .swiftbuild {
+        // Swift Build and the native build system setup search paths differently. We deliberately avoid testing implementation details of Swift Build here.
+        expectArgumentsContain("-I", try build.appending(component: "Modules").filePath, arguments: arguments)
+      }
 
       expectArgumentsContain(try aswift.filePath, arguments: arguments)
     }

config.schema.json

@@ -205,6 +205,15 @@
       "description" : "Options for SwiftPM workspaces.",
       "markdownDescription" : "Options for SwiftPM workspaces.",
       "properties" : {
+        "buildSystem" : {
+          "description" : "Which SwiftPM build system should be used when opening a package.",
+          "enum" : [
+            "native",
+            "swiftbuild"
+          ],
+          "markdownDescription" : "Which SwiftPM build system should be used when opening a package.",
+          "type" : "string"
+        },
         "buildToolsSwiftCompilerFlags" : {
           "description" : "Extra arguments passed to the compiler for Swift files or plugins. Equivalent to SwiftPM's `-Xbuild-tools-swiftc` option.",
           "items" : {