Merge remote-tracking branch 'swiftlang/main' into release/6.1

Author: ahoppen

CMakeLists.txt

@@ -12,7 +12,7 @@ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 
-add_compile_options("$<$<COMPILE_LANGUAGE:Swift>:SHELL:-enable-upcoming-feature InternalImportsByDefault>")
+add_compile_options("$<$<COMPILE_LANGUAGE:Swift>:SHELL:-enable-upcoming-feature InternalImportsByDefault -enable-upcoming-feature MemberImportVisibility>")
 
 find_package(dispatch QUIET)
 find_package(Foundation QUIET)

Contributor Documentation/LSP Extensions.md

@@ -622,3 +622,10 @@ export interface GetReferenceDocumentResult {
   content: string;
 }
 ```
+
+## Languages
+
+Added a new language with the identifier `tutorial` to support the `*.tutorial` files that
+Swift DocC uses to define tutorials and tutorial overviews in its documentation catalogs.
+It is expected that editors send document events for `tutorial` and `markdown` files if
+they wish to request information about these files from SourceKit-LSP.

Documentation/Configuration File.md

@@ -15,7 +15,7 @@ The structure of the file is currently not guaranteed to be stable. Options may
 
 - `swiftPM`: Dictionary with the following keys, defining options for SwiftPM workspaces
   - `configuration: "debug"|"release"`: The configuration to build the project for during background indexing and the configuration whose build folder should be used for Swift modules if background indexing is disabled. Equivalent to SwiftPM's `--configuration` option.
-  - `scratchPath: string`: Build artifacts directory path. If nil, the build system may choose a default value. Equivalent to SwiftPM's `--scratch-path` option.
+  - `scratchPath: string`: Build artifacts directory path. If nil, the build system may choose a default value. This path can be specified as a relative path, which will be interpreted relative to the project root. Equivalent to SwiftPM's `--scratch-path` option.
   - `swiftSDKsDirectory: string`: Equivalent to SwiftPM's `--swift-sdks-path` option
   - `swiftSDK: string`: Equivalent to SwiftPM's `--swift-sdk` option
   - `triple: string`: Equivalent to SwiftPM's `--triple` option
@@ -51,5 +51,5 @@ The structure of the file is currently not guaranteed to be stable. Options may
   - `noLazy`: Prepare a target without generating object files but do not do lazy type checking and function body skipping
   - `enabled`: Prepare a target without generating object files and the like
 - `cancelTextDocumentRequestsOnEditAndClose: bool`: Whether sending a `textDocument/didChange` or `textDocument/didClose` notification for a document should cancel all pending requests for that document.
-- `experimentalFeatures: string[]`: Experimental features to enable
+- `experimentalFeatures: string[]`: Experimental features to enable. Available features: on-type-formatting
 - `swiftPublishDiagnosticsDebounceDuration: double`: The time that `SwiftLanguageService` should wait after an edit before starting to compute diagnostics and sending a `PublishDiagnosticsNotification`.

Package.swift

@@ -97,6 +97,7 @@ var targets: [Target] = [
       "SKTestSupport",
       "SourceKitLSP",
       "ToolchainRegistry",
+      "TSCExtensions",
     ],
     swiftSettings: globalSwiftSettings
   ),
@@ -348,7 +349,6 @@ var targets: [Target] = [
       "Csourcekitd",
       "SKLogging",
       "SwiftExtensions",
-      .product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core"),
     ],
     exclude: ["CMakeLists.txt", "sourcekitd_uids.swift.gyb"],
     swiftSettings: globalSwiftSettings
@@ -487,7 +487,6 @@ var targets: [Target] = [
       "SwiftExtensions",
       "TSCExtensions",
     ],
-    exclude: ["CMakeLists.txt"],
     swiftSettings: globalSwiftSettings
   ),
 ]

Sources/BuildSystemIntegration/BuildSystemManager.swift

@@ -13,7 +13,7 @@
 #if compiler(>=6)
 package import BuildServerProtocol
 import Dispatch
-import Foundation
+package import Foundation
 package import LanguageServerProtocol
 package import LanguageServerProtocolExtensions
 import SKLogging
@@ -23,8 +23,7 @@ package import SwiftExtensions
 package import ToolchainRegistry
 import TSCExtensions
 
-package import struct TSCBasic.AbsolutePath
-package import struct TSCBasic.RelativePath
+import struct TSCBasic.RelativePath
 #else
 import BuildServerProtocol
 import Dispatch
@@ -38,7 +37,6 @@ import SwiftExtensions
 import ToolchainRegistry
 import TSCExtensions
 
-import struct TSCBasic.AbsolutePath
 import struct TSCBasic.RelativePath
 #endif
 
@@ -138,31 +136,31 @@ private enum BuildSystemAdapter {
 
 private extension BuildSystemSpec {
   private static func createBuiltInBuildSystemAdapter(
-    projectRoot: AbsolutePath,
+    projectRoot: URL,
     messagesToSourceKitLSPHandler: any MessageHandler,
     buildSystemTestHooks: BuildSystemTestHooks,
     _ createBuildSystem: @Sendable (_ connectionToSourceKitLSP: any Connection) async throws -> BuiltInBuildSystem?
   ) async -> BuildSystemAdapter? {
     let connectionToSourceKitLSP = LocalConnection(
-      receiverName: "BuildSystemManager for \(projectRoot.asURL.lastPathComponent)"
+      receiverName: "BuildSystemManager for \(projectRoot.lastPathComponent)"
     )
     connectionToSourceKitLSP.start(handler: messagesToSourceKitLSPHandler)
 
     let buildSystem = await orLog("Creating build system") {
       try await createBuildSystem(connectionToSourceKitLSP)
     }
     guard let buildSystem else {
-      logger.log("Failed to create build system at \(projectRoot.pathString)")
+      logger.log("Failed to create build system at \(projectRoot)")
       return nil
     }
-    logger.log("Created \(type(of: buildSystem), privacy: .public) at \(projectRoot.pathString)")
+    logger.log("Created \(type(of: buildSystem), privacy: .public) at \(projectRoot)")
     let buildSystemAdapter = BuiltInBuildSystemAdapter(
       underlyingBuildSystem: buildSystem,
       connectionToSourceKitLSP: connectionToSourceKitLSP,
       buildSystemTestHooks: buildSystemTestHooks
     )
     let connectionToBuildSystem = LocalConnection(
-      receiverName: "\(type(of: buildSystem)) for \(projectRoot.asURL.lastPathComponent)"
+      receiverName: "\(type(of: buildSystem)) for \(projectRoot.lastPathComponent)"
     )
     connectionToBuildSystem.start(handler: buildSystemAdapter)
     return .builtIn(buildSystemAdapter, connectionToBuildSystem: connectionToBuildSystem)
@@ -185,10 +183,10 @@ private extension BuildSystemSpec {
         )
       }
       guard let buildSystem else {
-        logger.log("Failed to create external build system at \(projectRoot.pathString)")
+        logger.log("Failed to create external build system at \(projectRoot)")
         return nil
       }
-      logger.log("Created external build server at \(projectRoot.pathString)")
+      logger.log("Created external build server at \(projectRoot)")
       return .external(buildSystem)
     case .compilationDatabase:
       return await Self.createBuiltInBuildSystemAdapter(
@@ -245,7 +243,7 @@ package actor BuildSystemManager: QueueBasedMessageHandler {
   /// For compilation databases it is the root folder based on which the compilation database was found.
   ///
   /// `nil` if the `BuildSystemManager` does not have an underlying build system.
-  package let projectRoot: AbsolutePath?
+  package let projectRoot: URL?
 
   /// The files for which the delegate has requested change notifications, ie. the files for which the delegate wants to
   /// get `fileBuildSettingsChanged` and `filesDependenciesUpdated` callbacks.
@@ -340,7 +338,10 @@ package actor BuildSystemManager: QueueBasedMessageHandler {
     let files: [DocumentURI: SourceFileInfo]
 
     /// The source directories in the workspace, ie. all `SourceItem`s that have `kind == .directory`.
-    let directories: [DocumentURI: SourceFileInfo]
+    ///
+    /// `pathComponents` is the result of `key.fileURL?.pathComponents`. We frequently need these path components to
+    /// determine if a file is descendent of the directory and computing them from the `DocumentURI` is expensive.
+    let directories: [DocumentURI: (pathComponents: [String]?, info: SourceFileInfo)]
   }
 
   private let cachedSourceFilesAndDirectories = Cache<SourceFilesAndDirectoriesKey, SourceFilesAndDirectories>()
@@ -402,7 +403,7 @@ package actor BuildSystemManager: QueueBasedMessageHandler {
             displayName: "SourceKit-LSP",
             version: "",
             bspVersion: "2.2.0",
-            rootUri: URI(buildSystemSpec.projectRoot.asURL),
+            rootUri: URI(buildSystemSpec.projectRoot),
             capabilities: BuildClientCapabilities(languageIds: [.c, .cpp, .objective_c, .objective_cpp, .swift])
           )
         )
@@ -608,7 +609,7 @@ package actor BuildSystemManager: QueueBasedMessageHandler {
     in target: BuildTargetIdentifier?,
     language: Language
   ) async -> Toolchain? {
-    let toolchainPath = await orLog("Getting toolchain from build targets") { () -> AbsolutePath? in
+    let toolchainPath = await orLog("Getting toolchain from build targets") { () -> URL? in
       guard let target else {
         return nil
       }
@@ -624,7 +625,7 @@ package actor BuildSystemManager: QueueBasedMessageHandler {
         logger.error("Toolchain is not a file URL")
         return nil
       }
-      return try AbsolutePath(validating: toolchainUrl.filePath)
+      return toolchainUrl
     }
     if let toolchainPath {
       if let toolchain = await self.toolchainRegistry.toolchain(withPath: toolchainPath) {
@@ -634,7 +635,7 @@ package actor BuildSystemManager: QueueBasedMessageHandler {
     }
 
     switch language {
-    case .swift:
+    case .swift, .markdown, .tutorial:
       return await toolchainRegistry.preferredToolchain(containing: [\.sourcekitd, \.swift, \.swiftc])
     case .c, .cpp, .objective_c, .objective_cpp:
       return await toolchainRegistry.preferredToolchain(containing: [\.clang, \.clangd])
@@ -681,14 +682,12 @@ package actor BuildSystemManager: QueueBasedMessageHandler {
       if let targets = filesAndDirectories.files[document]?.targets {
         result.formUnion(targets)
       }
-      if !filesAndDirectories.directories.isEmpty,
-        let documentPath = AbsolutePath(validatingOrNil: try? document.fileURL?.filePath)
-      {
-        for (directory, info) in filesAndDirectories.directories {
-          guard let directoryPath = AbsolutePath(validatingOrNil: try? directory.fileURL?.filePath) else {
+      if !filesAndDirectories.directories.isEmpty, let documentPathComponents = document.fileURL?.pathComponents {
+        for (directory, (directoryPathComponents, info)) in filesAndDirectories.directories {
+          guard let directoryPathComponents, let directoryPath = directory.fileURL else {
             continue
           }
-          if documentPath.isDescendant(of: directoryPath) {
+          if isDescendant(documentPathComponents, of: directoryPathComponents) {
             result.formUnion(info.targets)
           }
         }
@@ -1013,19 +1012,21 @@ package actor BuildSystemManager: QueueBasedMessageHandler {
       return []
     }
 
+    let request = BuildTargetSourcesRequest(targets: targets.sorted { $0.uri.stringValue < $1.uri.stringValue })
+
     // If we have a cached request for a superset of the targets, serve the result from that cache entry.
     let fromSuperset = await orLog("Getting source files from superset request") {
-      try await cachedTargetSources.get(isolation: self) { request in
-        targets.isSubset(of: request.targets)
-      } transform: { response in
-        return BuildTargetSourcesResponse(items: response.items.filter { targets.contains($0.target) })
-      }
+      try await cachedTargetSources.getDerived(
+        isolation: self,
+        request,
+        canReuseKey: { targets.isSubset(of: $0.targets) },
+        transform: { BuildTargetSourcesResponse(items: $0.items.filter { targets.contains($0.target) }) }
+      )
     }
     if let fromSuperset {
       return fromSuperset.items
     }
 
-    let request = BuildTargetSourcesRequest(targets: targets.sorted { $0.uri.stringValue < $1.uri.stringValue })
     let response = try await cachedTargetSources.get(request, isolation: self) { request in
       try await buildSystemAdapter.send(request)
     }
@@ -1059,7 +1060,7 @@ package actor BuildSystemManager: QueueBasedMessageHandler {
 
     return try await cachedSourceFilesAndDirectories.get(key, isolation: self) { key in
       var files: [DocumentURI: SourceFileInfo] = [:]
-      var directories: [DocumentURI: SourceFileInfo] = [:]
+      var directories: [DocumentURI: (pathComponents: [String]?, info: SourceFileInfo)] = [:]
       for sourcesItem in key.sourcesItems {
         let target = targets[sourcesItem.target]?.target
         let isPartOfRootProject = !(target?.tags.contains(.dependency) ?? false)
@@ -1081,7 +1082,9 @@ package actor BuildSystemManager: QueueBasedMessageHandler {
           case .file:
             files[sourceItem.uri] = info.merging(files[sourceItem.uri])
           case .directory:
-            directories[sourceItem.uri] = info.merging(directories[sourceItem.uri])
+            directories[sourceItem.uri] = (
+              sourceItem.uri.fileURL?.pathComponents, info.merging(directories[sourceItem.uri]?.info)
+            )
           }
         }
       }
@@ -1230,3 +1233,12 @@ package actor BuildSystemManager: QueueBasedMessageHandler {
     }
   }
 }
+
+/// Returns `true` if the path components `selfPathComponents`, retrieved from `URL.pathComponents` are a descendent
+/// of the other path components.
+///
+/// This operates directly on path components instead of `URL`s because computing the path components of a URL is
+/// expensive and this allows us to cache the path components.
+private func isDescendant(_ selfPathComponents: [String], of otherPathComponents: [String]) -> Bool {
+  return selfPathComponents.dropLast().starts(with: otherPathComponents)
+}

Sources/BuildSystemIntegration/BuildSystemMessageDependencyTracker.swift

@@ -15,7 +15,7 @@ import BuildServerProtocol
 package import LanguageServerProtocol
 import LanguageServerProtocolExtensions
 import SKLogging
-import SwiftExtensions
+package import SwiftExtensions
 #else
 import BuildServerProtocol
 import LanguageServerProtocol
@@ -82,6 +82,10 @@ package enum BuildSystemMessageDependencyTracker: QueueBasedMessageHandlerDepend
     }
   }
 
+  package func dependencies(in pendingTasks: [PendingTask<Self>]) -> [PendingTask<Self>] {
+    return pendingTasks.filter { $0.metadata.isDependency(of: self) }
+  }
+
   package init(_ request: some RequestType) {
     switch request {
     case is BuildShutdownRequest:

Sources/BuildSystemIntegration/BuiltInBuildSystem.swift

@@ -12,22 +12,18 @@
 
 #if compiler(>=6)
 package import BuildServerProtocol
+package import Foundation
 package import LanguageServerProtocol
 import SKLogging
 import SKOptions
 import ToolchainRegistry
-
-package import struct TSCBasic.AbsolutePath
-package import struct TSCBasic.RelativePath
 #else
 import BuildServerProtocol
+import Foundation
 import LanguageServerProtocol
 import SKLogging
 import SKOptions
 import ToolchainRegistry
-
-import struct TSCBasic.AbsolutePath
-import struct TSCBasic.RelativePath
 #endif
 
 /// An error build systems can throw from `prepare` if they don't support preparation of targets.
@@ -42,16 +38,16 @@ package protocol BuiltInBuildSystem: AnyObject, Sendable {
   /// The root of the project that this build system manages. For example, for SwiftPM packages, this is the folder
   /// containing Package.swift. For compilation databases it is the root folder based on which the compilation database
   /// was found.
-  var projectRoot: AbsolutePath { get async }
+  var projectRoot: URL { get async }
 
   /// The files to watch for changes.
   var fileWatchers: [FileSystemWatcher] { get async }
 
   /// The path to the raw index store data, if any.
-  var indexStorePath: AbsolutePath? { get async }
+  var indexStorePath: URL? { get async }
 
   /// The path to put the index database, if any.
-  var indexDatabasePath: AbsolutePath? { get async }
+  var indexDatabasePath: URL? { get async }
 
   /// Whether the build system is capable of preparing a target for indexing, ie. if the `prepare` methods has been
   /// implemented.

Sources/BuildSystemIntegration/BuiltInBuildSystemAdapter.swift

@@ -11,7 +11,6 @@
 //===----------------------------------------------------------------------===//
 
 import BuildServerProtocol
-import Foundation
 import LanguageServerProtocol
 import LanguageServerProtocolExtensions
 import SKLogging
@@ -20,11 +19,9 @@ import SwiftExtensions
 import ToolchainRegistry
 
 #if compiler(>=6)
-package import struct TSCBasic.AbsolutePath
-package import struct TSCBasic.RelativePath
+package import Foundation
 #else
-import struct TSCBasic.AbsolutePath
-import struct TSCBasic.RelativePath
+import Foundation
 #endif
 
 /// The details necessary to create a `BuildSystemAdapter`.
@@ -38,9 +35,9 @@ package struct BuildSystemSpec {
 
   package var kind: Kind
 
-  package var projectRoot: AbsolutePath
+  package var projectRoot: URL
 
-  package init(kind: BuildSystemSpec.Kind, projectRoot: AbsolutePath) {
+  package init(kind: BuildSystemSpec.Kind, projectRoot: URL) {
     self.kind = kind
     self.projectRoot = projectRoot
   }
@@ -95,8 +92,12 @@ actor BuiltInBuildSystemAdapter: QueueBasedMessageHandler {
       capabilities: BuildServerCapabilities(),
       dataKind: .sourceKit,
       data: SourceKitInitializeBuildResponseData(
-        indexDatabasePath: await underlyingBuildSystem.indexDatabasePath?.pathString,
-        indexStorePath: await underlyingBuildSystem.indexStorePath?.pathString,
+        indexDatabasePath: await orLog("getting index database file path") {
+          try await underlyingBuildSystem.indexDatabasePath?.filePath
+        },
+        indexStorePath: await orLog("getting index store file path") {
+          try await underlyingBuildSystem.indexStorePath?.filePath
+        },
         watchers: await underlyingBuildSystem.fileWatchers,
         prepareProvider: underlyingBuildSystem.supportsPreparation,
         sourceKitOptionsProvider: true