Merge pull request #3849 from SwiftPackageIndex/processing-env-variable

Inject SPI_PROCESSING env variable

Author: finestructure

Sources/App/Core/Dependencies/ShellClient.swift

@@ -19,14 +19,14 @@ import ShellOut
 
 @DependencyClient
 struct ShellClient {
-    var run: @Sendable (ShellOutCommand, String) async throws -> String
+    var run: @Sendable (ShellOutCommand, String, [String: String]?) async throws -> String
 }
 
 
 extension ShellClient {
     @discardableResult
-    func run(command: ShellOutCommand, at path: String) async throws -> String {
-        try await run(command, path)
+    func run(command: ShellOutCommand, at path: String, environment: [String: String]? = nil) async throws -> String {
+        try await run(command, path, environment)
     }
 }
 
@@ -39,10 +39,15 @@ extension String {
 extension ShellClient: DependencyKey {
     static var liveValue: Self {
         .init(
-            run: { command, path in
+            run: { command, path, environment in
                 @Dependency(\.logger) var logger
                 do {
-                    let res = try await ShellOut.shellOut(to: command, at: path, logger: logger)
+                    let res = try await ShellOut.shellOut(
+                        to: command,
+                        at: path,
+                        logger: logger,
+                        environment: (environment ?? [:]).merging(["SPI_PROCESSING": "1"], uniquingKeysWith: { $1 })
+                    )
                     if !res.stderr.isEmpty {
                         logger.warning("stderr: \(res.stderr)")
                     }

Tests/AppTests/AnalyzeErrorTests.swift

@@ -67,7 +67,7 @@ extension AllTests.AnalyzeErrorTests {
                 $0.environment.loadSPIManifest = { _ in nil }
                 $0.fileManager.fileExists = { @Sendable _ in true }
                 $0.logger = .testLogger(capturingLogger)
-                $0.shell.run = { @Sendable cmd, path in
+                $0.shell.run = { @Sendable cmd, path, _ in
                     switch cmd {
                         case _ where cmd.description.contains("git clone https://github.com/foo/1"):
                             throw SimulatedError()
@@ -131,7 +131,7 @@ extension AllTests.AnalyzeErrorTests {
                 $0.environment.loadSPIManifest = { _ in nil }
                 $0.fileManager.fileExists = { @Sendable _ in true }
                 $0.logger = .testLogger(capturingLogger)
-                $0.shell.run = { @Sendable cmd, path in
+                $0.shell.run = { @Sendable cmd, path, _ in
                     switch cmd {
                         case .gitCheckout(branch: "main", quiet: true) where path.hasSuffix("foo-1"):
                             throw SimulatedError()
@@ -195,7 +195,7 @@ extension AllTests.AnalyzeErrorTests {
             $0.environment.allowSocialPosts = { true }
             $0.git = .analyzeErrorTestsMock
             $0.httpClient.mastodonPost = { @Sendable msg in socialPosts.withValue { $0.append(msg) } }
-            $0.shell.run = defaultShellRun(command:path:)
+            $0.shell.run = defaultShellRun(command:path:environment:)
         }
     }
 }
@@ -222,7 +222,7 @@ extension AllTests.AnalyzeErrorTests {
 }
 
 
-private func defaultShellRun(command: ShellOutCommand, path: String) throws -> String {
+private func defaultShellRun(command: ShellOutCommand, path: String, environment: [String: String]? = nil) throws -> String {
     switch command {
         case .swiftDumpPackage where path.hasSuffix("foo-1"):
             return packageSwift1

Tests/AppTests/AnalyzerTests.swift

@@ -59,7 +59,7 @@ extension AllTests.AnalyzerTests {
                 }
                 $0.git = .liveValue
                 $0.httpClient.mastodonPost = { @Sendable _ in }
-                $0.shell.run = { @Sendable cmd, path in
+                $0.shell.run = { @Sendable cmd, path, _ in
                     let trimmedPath = path.replacingOccurrences(of: checkoutDir.value!, with: ".")
                     commands.withValue {
                         $0.append(.init(command: cmd, path: trimmedPath)!)
@@ -249,7 +249,7 @@ extension AllTests.AnalyzerTests {
                 """
                 }
                 $0.httpClient.mastodonPost = { @Sendable _ in }
-                $0.shell.run = { @Sendable cmd, path in
+                $0.shell.run = { @Sendable cmd, path, _ in
                     if cmd.description.hasSuffix("package dump-package") {
                         return #"""
                         {
@@ -320,7 +320,7 @@ extension AllTests.AnalyzerTests {
                 $0.git.hasBranch = { @Sendable _, _ in false }  // simulate analysis error via branch mismatch
                 $0.git.lastCommitDate = { @Sendable _ in .t1 }
                 $0.git.shortlog = { @Sendable _ in "" }
-                $0.shell.run = { @Sendable _, _ in "" }
+                $0.shell.run = { @Sendable _, _, _ in "" }
             } operation: {
                 // setup
                 do {
@@ -374,7 +374,7 @@ extension AllTests.AnalyzerTests {
                  2\tPerson 2
                 """
                 }
-                $0.shell.run = { @Sendable cmd, path in
+                $0.shell.run = { @Sendable cmd, path, _ in
                     // first package fails
                     if cmd.description.hasSuffix("swift package dump-package") && path.hasSuffix("foo-1") {
                         return "bad data"
@@ -439,7 +439,7 @@ extension AllTests.AnalyzerTests {
                     return false
                 }
                 $0.git = .liveValue
-                $0.shell.run = { @Sendable cmd, path in
+                $0.shell.run = { @Sendable cmd, path, _ in
                     commands.withValue {
                         $0.append(.init(command: cmd, path: path)!)
                     }
@@ -500,7 +500,7 @@ extension AllTests.AnalyzerTests {
             let commands = QueueIsolated<[String]>([])
             try await withDependencies {
                 $0.fileManager.fileExists = { @Sendable _ in true }
-                $0.shell.run = { @Sendable cmd, path in
+                $0.shell.run = { @Sendable cmd, path, _ in
                     commands.withValue { $0.append(cmd.description) }
                     return ""
                 }
@@ -532,7 +532,7 @@ extension AllTests.AnalyzerTests {
                  2\tPerson 2
                 """
                 }
-                $0.shell.run = { @Sendable cmd, _ in throw TestError.unknownCommand }
+                $0.shell.run = { @Sendable cmd, _, _ in throw TestError.unknownCommand }
             } operation: {
                 // setup
                 let pkg = Package(id: .id0, url: "1".asGithubUrl.url)
@@ -641,7 +641,7 @@ extension AllTests.AnalyzerTests {
                     if ref == .tag(1, 2, 3) { return .init(commit: "sha.1.2.3", date: .t1) }
                     fatalError("unknown ref: \(ref)")
                 }
-                $0.shell.run = { @Sendable cmd, _ in throw TestError.unknownCommand }
+                $0.shell.run = { @Sendable cmd, _, _ in throw TestError.unknownCommand }
             } operation: {
                 //setup
                 let pkgId = UUID()
@@ -777,7 +777,7 @@ extension AllTests.AnalyzerTests {
             try await withDependencies {
                 $0.environment.loadSPIManifest = { _ in nil }
                 $0.fileManager.fileExists = { @Sendable _ in true }
-                $0.shell.run = { @Sendable cmd, _ in
+                $0.shell.run = { @Sendable cmd, _, _ in
                     commands.withValue {
                         $0.append(cmd.description)
                     }
@@ -938,7 +938,7 @@ extension AllTests.AnalyzerTests {
                  2\tPerson 2
                 """
                 }
-                $0.shell.run = { @Sendable cmd, path in
+                $0.shell.run = { @Sendable cmd, path, _ in
                     if cmd.description.hasSuffix("swift package dump-package") {
                         return #"""
                         {
@@ -995,7 +995,7 @@ extension AllTests.AnalyzerTests {
                 // claim every file exists, including our ficticious 'index.lock' for which
                 // we want to trigger the cleanup mechanism
                 $0.fileManager.fileExists = { @Sendable path in true }
-                $0.shell.run = { @Sendable cmd, path in
+                $0.shell.run = { @Sendable cmd, path, _ in
                     commands.withValue { $0.append(cmd.description) }
                     return ""
                 }
@@ -1027,7 +1027,7 @@ extension AllTests.AnalyzerTests {
                 // claim every file exists, including our ficticious 'index.lock' for which
                 // we want to trigger the cleanup mechanism
                 $0.fileManager.fileExists = { @Sendable path in true }
-                $0.shell.run = { @Sendable cmd, path in
+                $0.shell.run = { @Sendable cmd, path, _ in
                     commands.withValue { $0.append(cmd.description) }
                     if cmd == .gitCheckout(branch: "master") {
                         throw TestError.simulatedCheckoutError
@@ -1132,29 +1132,40 @@ extension AllTests.AnalyzerTests {
         try await withDependencies {
             $0.logger = .noop
         } operation: {
-        try await withTempDir { @Sendable tempDir in
-            let fixture = fixturesDirectory()
-                .appendingPathComponent("5.9-Package-swift").path
-            let fname = tempDir.appending("/Package.swift")
-            try await ShellOut.shellOut(to: .copyFile(from: fixture, to: fname))
-            var json = try await ShellClient.liveValue.run(command: .swiftDumpPackage, at: tempDir)
-            do {  // "root" references tempDir's absolute path - replace it to make the test stable
-                if var obj = try JSONSerialization.jsonObject(with: Data(json.utf8)) as? [String: Any],
-                   var packageKind = obj["packageKind"] as? [String: Any] {
-                    packageKind["root"] = ["<tempdir>"]
-                    obj["packageKind"] = packageKind
-                    let data = try JSONSerialization.data(withJSONObject: obj,
-                                                          options: [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes])
-                    json = String(decoding: data, as: UTF8.self)
+            try await withTempDir { @Sendable tempDir in
+                let fixture = fixturesDirectory()
+                    .appendingPathComponent("5.9-Package-swift").path
+                let fname = tempDir.appending("/Package.swift")
+                try await ShellOut.shellOut(to: .copyFile(from: fixture, to: fname))
+                var json = try await ShellClient.liveValue.run(command: .swiftDumpPackage, at: tempDir)
+                do {  // "root" references tempDir's absolute path - replace it to make the test stable
+                    if var obj = try JSONSerialization.jsonObject(with: Data(json.utf8)) as? [String: Any],
+                       var packageKind = obj["packageKind"] as? [String: Any] {
+                        packageKind["root"] = ["<tempdir>"]
+                        obj["packageKind"] = packageKind
+                        let data = try JSONSerialization.data(withJSONObject: obj,
+                                                              options: [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes])
+                        json = String(decoding: data, as: UTF8.self)
+                    }
                 }
-            }
 #if os(macOS)
-            assertSnapshot(of: json, as: .init(pathExtension: "json", diffing: .lines), named: "macos")
+                assertSnapshot(of: json, as: .init(pathExtension: "json", diffing: .lines), named: "macos")
 #elseif os(Linux)
-            assertSnapshot(of: json, as: .init(pathExtension: "json", diffing: .lines), named: "linux")
+                assertSnapshot(of: json, as: .init(pathExtension: "json", diffing: .lines), named: "linux")
 #endif
+            }
         }
     }
+
+    @Test func processingEnvironmentVariable() async throws {
+        try await withDependencies {
+            $0.logger = .noop
+            $0.shell = .liveValue
+        } operation: {
+            @Dependency(\.shell) var shell
+            let res = try await shell.run(command: .init(command: "printenv", arguments: ["SPI_PROCESSING"]), at: "/tmp")
+            #expect(res == "1")
+        }
     }
 
     @Test func issue_577() async throws {
@@ -1193,7 +1204,7 @@ extension AllTests.AnalyzerTests {
             let commands = QueueIsolated<[String]>([])
             try await withDependencies {
                 $0.fileManager.fileExists = { @Sendable _ in true }
-                $0.shell.run = { @Sendable cmd, _ in
+                $0.shell.run = { @Sendable cmd, _, _ in
                     commands.withValue { $0.append(cmd.description) }
                     if cmd == .gitFetchAndPruneTags { throw TestError.simulatedFetchError }
                     return ""
@@ -1297,7 +1308,7 @@ extension AllTests.AnalyzerTests {
                     if path.hasSuffix("github.com-foo-1") { return false }
                     return true
                 }
-                $0.shell.run = { @Sendable cmd, path in
+                $0.shell.run = { @Sendable cmd, path, _ in
                     if cmd == .gitClone(url: url, to: repoDir) {
                         struct ShellOutError: Error {}
                         throw ShellOutError()
@@ -1355,7 +1366,7 @@ extension AllTests.AnalyzerTests {
                 1\tPerson 2
                 """
                 }
-                $0.shell.run = { @Sendable _, _ in "" }
+                $0.shell.run = { @Sendable _, _, _ in "" }
             } operation: {
                 let pkgId = UUID()
                 let pkg = Package(id: pkgId, url: "1".asGithubUrl.url, processingStage: .ingestion)
@@ -1427,7 +1438,7 @@ extension AllTests.AnalyzerTests {
                 try await withDependencies {  // third scenario: everything throws
                     $0.git.getTags = { @Sendable _ in throw TestError.unspecifiedError }
                     $0.git.revisionInfo = { @Sendable _, _ in throw TestError.unspecifiedError }
-                    $0.shell.run = { @Sendable _, _ in throw TestError.unspecifiedError }
+                    $0.shell.run = { @Sendable _, _, _ in throw TestError.unspecifiedError }
                 } operation: {
                     // MUT
                     try await Analyze.analyze(client: app.client,
@@ -1464,7 +1475,7 @@ extension AllTests.AnalyzerTests {
                 """
                 }
                 $0.logger = .testLogger(capturingLogger)
-                $0.shell.run = { @Sendable _, _ in return "" }
+                $0.shell.run = { @Sendable _, _, _ in return "" }
             } operation: {
                 let pkgId = UUID()
                 let pkg = Package(id: pkgId, url: "1".asGithubUrl.url, processingStage: .ingestion)
@@ -1526,7 +1537,7 @@ extension AllTests.AnalyzerTests {
                                 throw Error()
                         }
                     }
-                    $0.shell.run = { @Sendable cmd, path in
+                    $0.shell.run = { @Sendable cmd, path, _ in
                         // simulate error in getPackageInfo by failing checkout
                         if cmd == .gitCheckout(branch: "main") {
                             throw Error()
@@ -1569,7 +1580,7 @@ extension AllTests.AnalyzerTests {
                 $0.git.lastCommitDate = { @Sendable _ in .t1 }
                 $0.git.revisionInfo = { @Sendable _, _ in .init(commit: "sha1", date: .t0) }
                 $0.git.shortlog = { @Sendable _ in "10\tPerson 1" }
-                $0.shell.run = { @Sendable cmd, path in
+                $0.shell.run = { @Sendable cmd, path, _ in
                     if cmd == .swiftDumpPackage { return .packageDump(name: "foo1") }
                     return ""
                 }

Tests/AppTests/ErrorReportingTests.swift

@@ -64,7 +64,7 @@ extension AllTests.ErrorReportingTests {
             try await withDependencies {
                 $0.fileManager.fileExists = { @Sendable _ in true }
                 $0.logger = .testLogger(capturingLogger)
-                $0.shell.run = { @Sendable cmd, _ in
+                $0.shell.run = { @Sendable cmd, _, _ in
                     if cmd.description == "git tag" { return "1.0.0" }
                     // returning a blank string will cause an exception when trying to
                     // decode it as the manifest result - we use this to simulate errors

Tests/AppTests/GitTests.swift

@@ -45,7 +45,7 @@ extension AllTests.GitTests {
 
     @Test func revInfo() async throws {
         try await withDependencies {
-            $0.shell.run = { @Sendable cmd, _ in
+            $0.shell.run = { @Sendable cmd, _, _ in
                 if cmd.description == #"git log -n1 --format=tformat:"%H-%ct" 2.2.1"# {
                     return "63c973f3c2e632a340936c285e94d59f9ffb01d5-1536799579"
                 }
@@ -62,7 +62,7 @@ extension AllTests.GitTests {
         // Ensure we look up by tag name and not semver
         // https://github.com/SwiftPackageIndex/SwiftPackageIndex-Server/issues/139
         try await withDependencies {
-            $0.shell.run = { @Sendable cmd, _ in
+            $0.shell.run = { @Sendable cmd, _, _ in
                 if cmd.description == #"git log -n1 --format=tformat:"%H-%ct" v2.2.1"# {
                     return "63c973f3c2e632a340936c285e94d59f9ffb01d5-1536799579"
                 }
@@ -83,8 +83,8 @@ private enum TestError: Error {
 }
 
 
-func mock(for command: String, _ result: String) -> @Sendable (ShellOutCommand, String) throws -> String {
-    { @Sendable cmd, path in
+func mock(for command: String, _ result: String) -> @Sendable (ShellOutCommand, String, [String: String]?) throws -> String {
+    { @Sendable cmd, path, _ in
         guard cmd.description == command else { throw TestError.unknownCommand }
         return result
     }

Tests/AppTests/IngestionTests.swift

@@ -473,7 +473,7 @@ extension AllTests.IngestionTests {
                 $0.git.lastCommitDate = { @Sendable _ in .t0 }
                 $0.git.revisionInfo = { @Sendable _, _ in .init(commit: "sha0", date: .t0) }
                 $0.git.shortlog = { @Sendable _ in "" }
-                $0.shell.run = { @Sendable cmd, _ in
+                $0.shell.run = { @Sendable cmd, _, _ in
                     if cmd.description.hasSuffix("package dump-package") {
                         return .packageDump(name: "foo")
                     }

Tests/AppTests/MastodonTests.swift

@@ -55,7 +55,7 @@ extension AllTests.MastodonTests {
                     Issue.record("message must only be set once")
                 }
             }
-            $0.shell.run = { @Sendable cmd, path in
+            $0.shell.run = { @Sendable cmd, path, _ in
                 if cmd.description.hasSuffix("swift package dump-package") {
                     return #"{ "name": "Mock", "products": [], "targets": [] }"#
                 }

Tests/AppTests/PackageController+routesTests.swift

@@ -1533,7 +1533,7 @@ extension AllTests.PackageController_routesTests {
             }
             $0.git.shortlog = { @Sendable _ in "2\tauthor" }
             $0.httpClient.fetchDocumentation = { @Sendable _ in .ok(body: .mockIndexHTML()) }
-            $0.shell.run = { @Sendable cmd, _ in
+            $0.shell.run = { @Sendable cmd, _, _ in
                 if cmd.description == "swift package dump-package" { return .mockManifest }
                 return ""
             }

Tests/AppTests/PackageTests.swift

@@ -338,7 +338,7 @@ extension AllTests.PackageTests {
             $0.packageListRepository.fetchPackageList = { @Sendable _ in [url.url] }
             $0.packageListRepository.fetchPackageDenyList = { @Sendable _ in [] }
             $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] }
-            $0.shell.run = { @Sendable cmd, path in
+            $0.shell.run = { @Sendable cmd, path, _ in
                 if cmd.description.hasSuffix("swift package dump-package") {
                     return #"{ "name": "Mock", "products": [] }"#
                 }

Tests/AppTests/PipelineTests.swift

@@ -198,7 +198,7 @@ extension AllTests.PipelineTests {
             $0.packageListRepository.fetchPackageDenyList = { @Sendable _ in [] }
             $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] }
             $0.packageListRepository.fetchCustomCollection = { @Sendable _, _ in [] }
-            $0.shell.run = { @Sendable cmd, path in
+            $0.shell.run = { @Sendable cmd, path, _ in
                 if cmd.description.hasSuffix("swift package dump-package") {
                     return #"{ "name": "Mock", "products": [], "targets": [] }"#
                 }