From 24200dd4917e6ad9a8f06b6532aa000ec54d95ba Mon Sep 17 00:00:00 2001 From: "Sven A. Schmidt" Date: Mon, 14 Oct 2024 13:55:12 +0200 Subject: [PATCH 1/3] Move Current.fetchPackageList to @Dependency --- Sources/App/Commands/Reconcile.swift | 14 +-- Sources/App/Core/AppEnvironment.swift | 2 - .../PackageListRepositoryClient.swift | 8 ++ Tests/AppTests/MastodonTests.swift | 2 +- Tests/AppTests/MetricsTests.swift | 4 +- .../AppTests/Mocks/AppEnvironment+mock.swift | 4 - Tests/AppTests/PackageTests.swift | 4 +- Tests/AppTests/PipelineTests.swift | 115 +++++++++--------- Tests/AppTests/ReconcilerTests.swift | 42 ++++--- 9 files changed, 99 insertions(+), 96 deletions(-) diff --git a/Sources/App/Commands/Reconcile.swift b/Sources/App/Commands/Reconcile.swift index 10500fc9e..930e0afeb 100644 --- a/Sources/App/Commands/Reconcile.swift +++ b/Sources/App/Commands/Reconcile.swift @@ -67,7 +67,9 @@ func reconcile(client: Client, database: Database) async throws { func reconcileMainPackageList(client: Client, database: Database) async throws -> [URL] { - async let sourcePackageList = try Current.fetchPackageList(client) + @Dependency(\.packageListRepository) var packageListRepository + + async let sourcePackageList = try packageListRepository.fetchPackageList(client: client) async let sourcePackageDenyList = try Current.fetchPackageDenyList(client) async let currentList = try fetchCurrentPackageList(database) @@ -82,15 +84,7 @@ func reconcileMainPackageList(client: Client, database: Database) async throws - } -func liveFetchPackageList(_ client: Client) async throws -> [URL] { - try await client - .get(Constants.packageListUri) - .content - .decode([String].self, using: JSONDecoder()) - .compactMap(URL.init(string:)) -} - - +@available(*, deprecated) func liveFetchPackageDenyList(_ client: Client) async throws -> [URL] { struct DeniedPackage: Decodable { var packageUrl: String diff --git a/Sources/App/Core/AppEnvironment.swift b/Sources/App/Core/AppEnvironment.swift index 6889c169e..b846a9994 100644 --- a/Sources/App/Core/AppEnvironment.swift +++ b/Sources/App/Core/AppEnvironment.swift @@ -43,7 +43,6 @@ struct AppEnvironment: Sendable { var environment: @Sendable () -> Environment var fetchDocumentation: @Sendable (_ client: Client, _ url: URI) async throws -> ClientResponse var fetchHTTPStatusCode: @Sendable (_ url: String) async throws -> HTTPStatus - var fetchPackageList: @Sendable (_ client: Client) async throws -> [URL] var fetchPackageDenyList: @Sendable (_ client: Client) async throws -> [URL] var fetchLicense: @Sendable (_ client: Client, _ owner: String, _ repository: String) async -> Github.License? var fetchMetadata: @Sendable (_ client: Client, _ owner: String, _ repository: String) async throws -> Github.Metadata @@ -165,7 +164,6 @@ extension AppEnvironment { environment: { (try? Environment.detect()) ?? .development }, fetchDocumentation: { client, url in try await client.get(url) }, fetchHTTPStatusCode: { url in try await Networking.fetchHTTPStatusCode(url) }, - fetchPackageList: { client in try await liveFetchPackageList(client) }, fetchPackageDenyList: { client in try await liveFetchPackageDenyList(client) }, fetchLicense: { client, owner, repo in await Github.fetchLicense(client:client, owner: owner, repository: repo) }, fetchMetadata: { client, owner, repo in try await Github.fetchMetadata(client:client, owner: owner, repository: repo) }, diff --git a/Sources/App/Core/Dependencies/PackageListRepositoryClient.swift b/Sources/App/Core/Dependencies/PackageListRepositoryClient.swift index 8b68a3f4b..371ade156 100644 --- a/Sources/App/Core/Dependencies/PackageListRepositoryClient.swift +++ b/Sources/App/Core/Dependencies/PackageListRepositoryClient.swift @@ -21,6 +21,7 @@ import Vapor struct PackageListRepositoryClient { var fetchCustomCollection: @Sendable (_ client: Client, _ url: URL) async throws -> [URL] var fetchCustomCollections: @Sendable (_ client: Client) async throws -> [CustomCollection.DTO] + var fetchPackageList: @Sendable (_ client: Client) async throws -> [URL] // TODO: move other package list dependencies here } @@ -39,6 +40,13 @@ extension PackageListRepositoryClient: DependencyKey { .get(Constants.customCollectionsUri) .content .decode([CustomCollection.DTO].self, using: JSONDecoder()) + }, + fetchPackageList: { client in + try await client + .get(Constants.packageListUri) + .content + .decode([String].self, using: JSONDecoder()) + .compactMap(URL.init(string:)) } ) } diff --git a/Tests/AppTests/MastodonTests.swift b/Tests/AppTests/MastodonTests.swift index 1f5b0ccf7..6b06357b2 100644 --- a/Tests/AppTests/MastodonTests.swift +++ b/Tests/AppTests/MastodonTests.swift @@ -34,7 +34,6 @@ final class MastodonTests: AppTestCase { let url = "https://github.com/foo/bar" Current.fetchMetadata = { _, owner, repository in .mock(owner: owner, repository: repository) } - Current.fetchPackageList = { _ in [url.url] } Current.git.commitCount = { @Sendable _ in 12 } Current.git.firstCommitDate = { @Sendable _ in .t0 } @@ -60,6 +59,7 @@ final class MastodonTests: AppTestCase { $0.date.now = .now $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] } $0.packageListRepository.fetchCustomCollection = { @Sendable _, _ in [] } + $0.packageListRepository.fetchPackageList = { @Sendable _ in [url.url] } } operation: { // run first two processing steps try await reconcile(client: app.client, database: app.db) diff --git a/Tests/AppTests/MetricsTests.swift b/Tests/AppTests/MetricsTests.swift index 49fe99ac7..7ee990a5a 100644 --- a/Tests/AppTests/MetricsTests.swift +++ b/Tests/AppTests/MetricsTests.swift @@ -101,10 +101,8 @@ class MetricsTests: AppTestCase { func test_reconcileDurationSeconds() async throws { try await withDependencies { $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] } + $0.packageListRepository.fetchPackageList = { @Sendable _ in ["1", "2", "3"].asURLs } } operation: { - // setup - Current.fetchPackageList = { _ in ["1", "2", "3"].asURLs } - // MUT try await reconcile(client: app.client, database: app.db) diff --git a/Tests/AppTests/Mocks/AppEnvironment+mock.swift b/Tests/AppTests/Mocks/AppEnvironment+mock.swift index 46bdd71b0..4d96b15ee 100644 --- a/Tests/AppTests/Mocks/AppEnvironment+mock.swift +++ b/Tests/AppTests/Mocks/AppEnvironment+mock.swift @@ -42,10 +42,6 @@ extension AppEnvironment { environment: { .development }, fetchDocumentation: { _, _ in .init(status: .ok) }, fetchHTTPStatusCode: { _ in .ok }, - fetchPackageList: { _ in - ["https://github.com/finestructure/Gala", - "https://github.com/SwiftPackageIndex/SwiftPackageIndex-Server"].asURLs - }, fetchPackageDenyList: { _ in ["https://github.com/daveverwer/LeftPad"].asURLs }, diff --git a/Tests/AppTests/PackageTests.swift b/Tests/AppTests/PackageTests.swift index 518a28cfa..ae43dfd8e 100644 --- a/Tests/AppTests/PackageTests.swift +++ b/Tests/AppTests/PackageTests.swift @@ -295,14 +295,14 @@ final class PackageTests: AppTestCase { } func test_isNew() async throws { + let url = "1".asGithubUrl try await withDependencies { $0.date.now = .now $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] } + $0.packageListRepository.fetchPackageList = { @Sendable _ in [url.url] } } operation: { // setup - let url = "1".asGithubUrl Current.fetchMetadata = { _, owner, repository in .mock(owner: owner, repository: repository) } - Current.fetchPackageList = { _ in [url.url] } Current.git.commitCount = { @Sendable _ in 12 } Current.git.firstCommitDate = { @Sendable _ in Date(timeIntervalSince1970: 0) } Current.git.getTags = { @Sendable _ in [] } diff --git a/Tests/AppTests/PipelineTests.swift b/Tests/AppTests/PipelineTests.swift index c66c3884e..ddf603ec1 100644 --- a/Tests/AppTests/PipelineTests.swift +++ b/Tests/AppTests/PipelineTests.swift @@ -158,17 +158,16 @@ class PipelineTests: AppTestCase { } func test_processing_pipeline() async throws { + let urls = ["1", "2", "3"].asGithubUrls try await withDependencies { $0.date.now = .now $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] } $0.packageListRepository.fetchCustomCollection = { @Sendable _, _ in [] } + $0.packageListRepository.fetchPackageList = { @Sendable _ in urls.asURLs } } operation: { // Test pipeline pick-up end to end // setup - let urls = ["1", "2", "3"].asGithubUrls Current.fetchMetadata = { _, owner, repository in .mock(owner: owner, repository: repository) } - Current.fetchPackageList = { _ in urls.asURLs } - Current.git.commitCount = { @Sendable _ in 12 } Current.git.firstCommitDate = { @Sendable _ in .t0 } Current.git.lastCommitDate = { @Sendable _ in .t1 } @@ -224,76 +223,78 @@ class PipelineTests: AppTestCase { XCTAssertEqual(packages.map(\.isNew), [false, false, false]) } - // Now we've got a new package and a deletion - Current.fetchPackageList = { _ in ["1", "3", "4"].asGithubUrls.asURLs } - - // MUT - reconcile again - try await reconcile(client: app.client, database: app.db) - - do { // validate - only new package moves to .reconciliation stage - let packages = try await Package.query(on: app.db).sort(\.$url).all() - XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls) - XCTAssertEqual(packages.map(\.status), [.ok, .ok, .new]) - XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .reconciliation]) - XCTAssertEqual(packages.map(\.isNew), [false, false, true]) - } - - // MUT - ingest again - try await ingest(client: app.client, database: app.db, mode: .limit(10)) - - do { // validate - only new package moves to .ingestion stage - let packages = try await Package.query(on: app.db).sort(\.$url).all() - XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls) - XCTAssertEqual(packages.map(\.status), [.ok, .ok, .new]) - XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .ingestion]) - XCTAssertEqual(packages.map(\.isNew), [false, false, true]) - } - - // MUT - analyze again - let lastAnalysis = Date.now - try await Analyze.analyze(client: app.client, - database: app.db, - mode: .limit(10)) - - do { // validate - only new package moves to .ingestion stage - let packages = try await Package.query(on: app.db).sort(\.$url).all() - XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls) - XCTAssertEqual(packages.map(\.status), [.ok, .ok, .ok]) - XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .analysis]) - XCTAssertEqual(packages.map { $0.updatedAt! > lastAnalysis }, [false, false, true]) - XCTAssertEqual(packages.map(\.isNew), [false, false, false]) - } - try await withDependencies { - // fast forward our clock by the deadtime interval - $0.date.now = .now.addingTimeInterval(Constants.reIngestionDeadtime) + // Now we've got a new package and a deletion + $0.packageListRepository.fetchPackageList = { @Sendable _ in ["1", "3", "4"].asGithubUrls.asURLs } } operation: { - // MUT - ingest yet again + // MUT - reconcile again + try await reconcile(client: app.client, database: app.db) + + do { // validate - only new package moves to .reconciliation stage + let packages = try await Package.query(on: app.db).sort(\.$url).all() + XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls) + XCTAssertEqual(packages.map(\.status), [.ok, .ok, .new]) + XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .reconciliation]) + XCTAssertEqual(packages.map(\.isNew), [false, false, true]) + } + + // MUT - ingest again try await ingest(client: app.client, database: app.db, mode: .limit(10)) - - do { // validate - now all three packages should have been updated + + do { // validate - only new package moves to .ingestion stage let packages = try await Package.query(on: app.db).sort(\.$url).all() XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls) - XCTAssertEqual(packages.map(\.status), [.ok, .ok, .ok]) - XCTAssertEqual(packages.map(\.processingStage), [.ingestion, .ingestion, .ingestion]) - XCTAssertEqual(packages.map(\.isNew), [false, false, false]) + XCTAssertEqual(packages.map(\.status), [.ok, .ok, .new]) + XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .ingestion]) + XCTAssertEqual(packages.map(\.isNew), [false, false, true]) } - - // MUT - re-run analysis to complete the sequence + + // MUT - analyze again + let lastAnalysis = Date.now try await Analyze.analyze(client: app.client, database: app.db, mode: .limit(10)) - + do { // validate - only new package moves to .ingestion stage let packages = try await Package.query(on: app.db).sort(\.$url).all() XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls) XCTAssertEqual(packages.map(\.status), [.ok, .ok, .ok]) XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .analysis]) + XCTAssertEqual(packages.map { $0.updatedAt! > lastAnalysis }, [false, false, true]) XCTAssertEqual(packages.map(\.isNew), [false, false, false]) } - - // at this point we've ensured that retriggering ingestion after the deadtime will - // refresh analysis as expected + + try await withDependencies { + // fast forward our clock by the deadtime interval + $0.date.now = .now.addingTimeInterval(Constants.reIngestionDeadtime) + } operation: { + // MUT - ingest yet again + try await ingest(client: app.client, database: app.db, mode: .limit(10)) + + do { // validate - now all three packages should have been updated + let packages = try await Package.query(on: app.db).sort(\.$url).all() + XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls) + XCTAssertEqual(packages.map(\.status), [.ok, .ok, .ok]) + XCTAssertEqual(packages.map(\.processingStage), [.ingestion, .ingestion, .ingestion]) + XCTAssertEqual(packages.map(\.isNew), [false, false, false]) + } + + // MUT - re-run analysis to complete the sequence + try await Analyze.analyze(client: app.client, + database: app.db, + mode: .limit(10)) + + do { // validate - only new package moves to .ingestion stage + let packages = try await Package.query(on: app.db).sort(\.$url).all() + XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls) + XCTAssertEqual(packages.map(\.status), [.ok, .ok, .ok]) + XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .analysis]) + XCTAssertEqual(packages.map(\.isNew), [false, false, false]) + } + + // at this point we've ensured that retriggering ingestion after the deadtime will + // refresh analysis as expected + } } } } diff --git a/Tests/AppTests/ReconcilerTests.swift b/Tests/AppTests/ReconcilerTests.swift index 0eb97a18d..07df798b4 100644 --- a/Tests/AppTests/ReconcilerTests.swift +++ b/Tests/AppTests/ReconcilerTests.swift @@ -36,12 +36,13 @@ class ReconcilerTests: AppTestCase { } func test_reconcileMainPackageList() async throws { - // setup let urls = ["1", "2", "3"] - Current.fetchPackageList = { _ in urls.asURLs } - - // MUT - _ = try await reconcileMainPackageList(client: app.client, database: app.db) + try await withDependencies { + $0.packageListRepository.fetchPackageList = { @Sendable _ in urls.asURLs } + } operation: { + // MUT + _ = try await reconcileMainPackageList(client: app.client, database: app.db) + } // validate let packages = try await Package.query(on: app.db).all() @@ -63,10 +64,13 @@ class ReconcilerTests: AppTestCase { // new package list drops 2, 3, adds 4, 5 let urls = ["1", "4", "5"] - Current.fetchPackageList = { _ in urls.asURLs } - // MUT - _ = try await reconcileMainPackageList(client: app.client, database: app.db) + try await withDependencies { + $0.packageListRepository.fetchPackageList = { @Sendable _ in urls.asURLs } + } operation: { + // MUT + _ = try await reconcileMainPackageList(client: app.client, database: app.db) + } // validate let packages = try await Package.query(on: app.db).all() @@ -81,14 +85,17 @@ class ReconcilerTests: AppTestCase { // New list adds two new packages 4, 5 let packageList = ["1", "2", "3", "4", "5"] - Current.fetchPackageList = { _ in packageList.asURLs } // Deny list denies 2 and 4 (one existing and one new) let packageDenyList = ["2", "4"] Current.fetchPackageDenyList = { _ in packageDenyList.asURLs } - // MUT - _ = try await reconcileMainPackageList(client: app.client, database: app.db) + try await withDependencies { + $0.packageListRepository.fetchPackageList = { @Sendable _ in packageList.asURLs } + } operation: { + // MUT + _ = try await reconcileMainPackageList(client: app.client, database: app.db) + } // validate let packages = try await Package.query(on: app.db).all() @@ -103,14 +110,17 @@ class ReconcilerTests: AppTestCase { // New list adds no new packages let packageList = ["https://example.com/one/one", "https://example.com/two/two"] - Current.fetchPackageList = { _ in packageList.asURLs } // Deny list denies one/one, but with incorrect casing. let packageDenyList = ["https://example.com/OnE/oNe"] Current.fetchPackageDenyList = { _ in packageDenyList.asURLs } - // MUT - _ = try await reconcileMainPackageList(client: app.client, database: app.db) + try await withDependencies { + $0.packageListRepository.fetchPackageList = { @Sendable _ in packageList.asURLs } + } operation: { + // MUT + _ = try await reconcileMainPackageList(client: app.client, database: app.db) + } // validate let packages = try await Package.query(on: app.db).all() @@ -221,10 +231,8 @@ class ReconcilerTests: AppTestCase { $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [.init(name: "List", url: "collectionURL")] } + $0.packageListRepository.fetchPackageList = { @Sendable _ in fullPackageList } } operation: { - // setup - Current.fetchPackageList = { _ in fullPackageList } - // MUT _ = try await reconcile(client: app.client, database: app.db) From 98c96a5521b6cafc193f44219701e19fb4b0af7d Mon Sep 17 00:00:00 2001 From: "Sven A. Schmidt" Date: Tue, 15 Oct 2024 08:20:07 +0200 Subject: [PATCH 2/3] Move Current.fetchPackageDenyList to @Dependency --- Sources/App/Commands/Reconcile.swift | 26 ++------------ Sources/App/Core/AppEnvironment.swift | 2 -- .../PackageListRepositoryClient.swift | 34 ++++++++++++++----- Tests/AppTests/MastodonTests.swift | 3 +- .../AppTests/Mocks/AppEnvironment+mock.swift | 3 -- Tests/AppTests/PipelineTests.swift | 3 +- Tests/AppTests/ReconcilerTests.swift | 9 +++-- 7 files changed, 38 insertions(+), 42 deletions(-) diff --git a/Sources/App/Commands/Reconcile.swift b/Sources/App/Commands/Reconcile.swift index 930e0afeb..072bfc15a 100644 --- a/Sources/App/Commands/Reconcile.swift +++ b/Sources/App/Commands/Reconcile.swift @@ -69,12 +69,11 @@ func reconcile(client: Client, database: Database) async throws { func reconcileMainPackageList(client: Client, database: Database) async throws -> [URL] { @Dependency(\.packageListRepository) var packageListRepository - async let sourcePackageList = try packageListRepository.fetchPackageList(client: client) - async let sourcePackageDenyList = try Current.fetchPackageDenyList(client) + let sourcePackageList = try await packageListRepository.fetchPackageList(client: client) + let sourcePackageDenyList = try await packageListRepository.fetchPackageDenyList(client: client) async let currentList = try fetchCurrentPackageList(database) - let packageList = processPackageDenyList(packageList: try await sourcePackageList, - denyList: try await sourcePackageDenyList) + let packageList = processPackageDenyList(packageList: sourcePackageList, denyList: sourcePackageDenyList) try await reconcileLists(db: database, source: packageList, @@ -84,25 +83,6 @@ func reconcileMainPackageList(client: Client, database: Database) async throws - } -@available(*, deprecated) -func liveFetchPackageDenyList(_ client: Client) async throws -> [URL] { - struct DeniedPackage: Decodable { - var packageUrl: String - - enum CodingKeys: String, CodingKey { - case packageUrl = "package_url" - } - } - - return try await client - .get(Constants.packageDenyListUri) - .content - .decode([DeniedPackage].self, using: JSONDecoder()) - .map(\.packageUrl) - .compactMap(URL.init(string:)) -} - - func fetchCurrentPackageList(_ db: Database) async throws -> [URL] { try await Package.query(on: db) .field(Package.self, \.$url) diff --git a/Sources/App/Core/AppEnvironment.swift b/Sources/App/Core/AppEnvironment.swift index b846a9994..54efd59c1 100644 --- a/Sources/App/Core/AppEnvironment.swift +++ b/Sources/App/Core/AppEnvironment.swift @@ -43,7 +43,6 @@ struct AppEnvironment: Sendable { var environment: @Sendable () -> Environment var fetchDocumentation: @Sendable (_ client: Client, _ url: URI) async throws -> ClientResponse var fetchHTTPStatusCode: @Sendable (_ url: String) async throws -> HTTPStatus - var fetchPackageDenyList: @Sendable (_ client: Client) async throws -> [URL] var fetchLicense: @Sendable (_ client: Client, _ owner: String, _ repository: String) async -> Github.License? var fetchMetadata: @Sendable (_ client: Client, _ owner: String, _ repository: String) async throws -> Github.Metadata var fetchReadme: @Sendable (_ client: Client, _ owner: String, _ repository: String) async -> Github.Readme? @@ -164,7 +163,6 @@ extension AppEnvironment { environment: { (try? Environment.detect()) ?? .development }, fetchDocumentation: { client, url in try await client.get(url) }, fetchHTTPStatusCode: { url in try await Networking.fetchHTTPStatusCode(url) }, - fetchPackageDenyList: { client in try await liveFetchPackageDenyList(client) }, fetchLicense: { client, owner, repo in await Github.fetchLicense(client:client, owner: owner, repository: repo) }, fetchMetadata: { client, owner, repo in try await Github.fetchMetadata(client:client, owner: owner, repository: repo) }, fetchReadme: { client, owner, repo in await Github.fetchReadme(client:client, owner: owner, repository: repo) }, diff --git a/Sources/App/Core/Dependencies/PackageListRepositoryClient.swift b/Sources/App/Core/Dependencies/PackageListRepositoryClient.swift index 371ade156..c92548545 100644 --- a/Sources/App/Core/Dependencies/PackageListRepositoryClient.swift +++ b/Sources/App/Core/Dependencies/PackageListRepositoryClient.swift @@ -19,16 +19,39 @@ import Vapor @DependencyClient struct PackageListRepositoryClient { + var fetchPackageList: @Sendable (_ client: Client) async throws -> [URL] + var fetchPackageDenyList: @Sendable (_ client: Client) async throws -> [URL] var fetchCustomCollection: @Sendable (_ client: Client, _ url: URL) async throws -> [URL] var fetchCustomCollections: @Sendable (_ client: Client) async throws -> [CustomCollection.DTO] - var fetchPackageList: @Sendable (_ client: Client) async throws -> [URL] - // TODO: move other package list dependencies here } extension PackageListRepositoryClient: DependencyKey { static var liveValue: PackageListRepositoryClient { .init( + fetchPackageList: { client in + try await client + .get(Constants.packageListUri) + .content + .decode([String].self, using: JSONDecoder()) + .compactMap(URL.init(string:)) + }, + fetchPackageDenyList: { client in + struct DeniedPackage: Decodable { + var packageUrl: String + + enum CodingKeys: String, CodingKey { + case packageUrl = "package_url" + } + } + + return try await client + .get(Constants.packageDenyListUri) + .content + .decode([DeniedPackage].self, using: JSONDecoder()) + .map(\.packageUrl) + .compactMap(URL.init(string:)) + }, fetchCustomCollection: { client, url in try await client .get(URI(string: url.absoluteString)) @@ -40,13 +63,6 @@ extension PackageListRepositoryClient: DependencyKey { .get(Constants.customCollectionsUri) .content .decode([CustomCollection.DTO].self, using: JSONDecoder()) - }, - fetchPackageList: { client in - try await client - .get(Constants.packageListUri) - .content - .decode([String].self, using: JSONDecoder()) - .compactMap(URL.init(string:)) } ) } diff --git a/Tests/AppTests/MastodonTests.swift b/Tests/AppTests/MastodonTests.swift index 6b06357b2..a161885e0 100644 --- a/Tests/AppTests/MastodonTests.swift +++ b/Tests/AppTests/MastodonTests.swift @@ -57,9 +57,10 @@ final class MastodonTests: AppTestCase { try await withDependencies { $0.date.now = .now + $0.packageListRepository.fetchPackageList = { @Sendable _ in [url.url] } + $0.packageListRepository.fetchPackageDenyList = { @Sendable _ in [] } $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] } $0.packageListRepository.fetchCustomCollection = { @Sendable _, _ in [] } - $0.packageListRepository.fetchPackageList = { @Sendable _ in [url.url] } } operation: { // run first two processing steps try await reconcile(client: app.client, database: app.db) diff --git a/Tests/AppTests/Mocks/AppEnvironment+mock.swift b/Tests/AppTests/Mocks/AppEnvironment+mock.swift index 4d96b15ee..e36ad2490 100644 --- a/Tests/AppTests/Mocks/AppEnvironment+mock.swift +++ b/Tests/AppTests/Mocks/AppEnvironment+mock.swift @@ -42,9 +42,6 @@ extension AppEnvironment { environment: { .development }, fetchDocumentation: { _, _ in .init(status: .ok) }, fetchHTTPStatusCode: { _ in .ok }, - fetchPackageDenyList: { _ in - ["https://github.com/daveverwer/LeftPad"].asURLs - }, fetchLicense: { _, _, _ in .init(htmlUrl: "https://github.com/foo/bar/blob/main/LICENSE") }, fetchMetadata: { _, _, _ in .mock }, fetchReadme: { _, _, _ in .init(html: "readme html", htmlUrl: "readme html url", imagesToCache: []) }, diff --git a/Tests/AppTests/PipelineTests.swift b/Tests/AppTests/PipelineTests.swift index ddf603ec1..0895f80d0 100644 --- a/Tests/AppTests/PipelineTests.swift +++ b/Tests/AppTests/PipelineTests.swift @@ -161,9 +161,10 @@ class PipelineTests: AppTestCase { let urls = ["1", "2", "3"].asGithubUrls try await withDependencies { $0.date.now = .now + $0.packageListRepository.fetchPackageList = { @Sendable _ in urls.asURLs } + $0.packageListRepository.fetchPackageDenyList = { @Sendable _ in [] } $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] } $0.packageListRepository.fetchCustomCollection = { @Sendable _, _ in [] } - $0.packageListRepository.fetchPackageList = { @Sendable _ in urls.asURLs } } operation: { // Test pipeline pick-up end to end // setup diff --git a/Tests/AppTests/ReconcilerTests.swift b/Tests/AppTests/ReconcilerTests.swift index 07df798b4..1a7de3bb5 100644 --- a/Tests/AppTests/ReconcilerTests.swift +++ b/Tests/AppTests/ReconcilerTests.swift @@ -39,6 +39,7 @@ class ReconcilerTests: AppTestCase { let urls = ["1", "2", "3"] try await withDependencies { $0.packageListRepository.fetchPackageList = { @Sendable _ in urls.asURLs } + $0.packageListRepository.fetchPackageDenyList = { @Sendable _ in [] } } operation: { // MUT _ = try await reconcileMainPackageList(client: app.client, database: app.db) @@ -67,6 +68,7 @@ class ReconcilerTests: AppTestCase { try await withDependencies { $0.packageListRepository.fetchPackageList = { @Sendable _ in urls.asURLs } + $0.packageListRepository.fetchPackageDenyList = { @Sendable _ in [] } } operation: { // MUT _ = try await reconcileMainPackageList(client: app.client, database: app.db) @@ -88,10 +90,10 @@ class ReconcilerTests: AppTestCase { // Deny list denies 2 and 4 (one existing and one new) let packageDenyList = ["2", "4"] - Current.fetchPackageDenyList = { _ in packageDenyList.asURLs } try await withDependencies { $0.packageListRepository.fetchPackageList = { @Sendable _ in packageList.asURLs } + $0.packageListRepository.fetchPackageDenyList = { @Sendable _ in packageDenyList.asURLs } } operation: { // MUT _ = try await reconcileMainPackageList(client: app.client, database: app.db) @@ -113,10 +115,10 @@ class ReconcilerTests: AppTestCase { // Deny list denies one/one, but with incorrect casing. let packageDenyList = ["https://example.com/OnE/oNe"] - Current.fetchPackageDenyList = { _ in packageDenyList.asURLs } try await withDependencies { $0.packageListRepository.fetchPackageList = { @Sendable _ in packageList.asURLs } + $0.packageListRepository.fetchPackageDenyList = { @Sendable _ in packageDenyList.asURLs } } operation: { // MUT _ = try await reconcileMainPackageList(client: app.client, database: app.db) @@ -221,6 +223,8 @@ class ReconcilerTests: AppTestCase { struct TestError: Error { var message: String } try await withDependencies { + $0.packageListRepository.fetchPackageList = { @Sendable _ in fullPackageList } + $0.packageListRepository.fetchPackageDenyList = { @Sendable _ in [] } $0.packageListRepository.fetchCustomCollection = { @Sendable _, url in if url == "collectionURL" { return [URL("2")] @@ -231,7 +235,6 @@ class ReconcilerTests: AppTestCase { $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [.init(name: "List", url: "collectionURL")] } - $0.packageListRepository.fetchPackageList = { @Sendable _ in fullPackageList } } operation: { // MUT _ = try await reconcile(client: app.client, database: app.db) From bbd6d68eb0f1e95e8f0f64fa94d7beb89ec2b18b Mon Sep 17 00:00:00 2001 From: "Sven A. Schmidt" Date: Tue, 15 Oct 2024 08:43:04 +0200 Subject: [PATCH 3/3] Fix tests --- Tests/AppTests/MetricsTests.swift | 3 ++- Tests/AppTests/PackageTests.swift | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Tests/AppTests/MetricsTests.swift b/Tests/AppTests/MetricsTests.swift index 7ee990a5a..3addc215b 100644 --- a/Tests/AppTests/MetricsTests.swift +++ b/Tests/AppTests/MetricsTests.swift @@ -100,8 +100,9 @@ class MetricsTests: AppTestCase { func test_reconcileDurationSeconds() async throws { try await withDependencies { - $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] } $0.packageListRepository.fetchPackageList = { @Sendable _ in ["1", "2", "3"].asURLs } + $0.packageListRepository.fetchPackageDenyList = { @Sendable _ in [] } + $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] } } operation: { // MUT try await reconcile(client: app.client, database: app.db) diff --git a/Tests/AppTests/PackageTests.swift b/Tests/AppTests/PackageTests.swift index ae43dfd8e..36269265b 100644 --- a/Tests/AppTests/PackageTests.swift +++ b/Tests/AppTests/PackageTests.swift @@ -298,8 +298,9 @@ final class PackageTests: AppTestCase { let url = "1".asGithubUrl try await withDependencies { $0.date.now = .now - $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] } $0.packageListRepository.fetchPackageList = { @Sendable _ in [url.url] } + $0.packageListRepository.fetchPackageDenyList = { @Sendable _ in [] } + $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] } } operation: { // setup Current.fetchMetadata = { _, owner, repository in .mock(owner: owner, repository: repository) }