Skip to content

Commit 6a58ebf

Browse files
finestructurefinestructure
committed
Convert QueryPerformanceTests
1 parent 64ff8e7 commit 6a58ebf

File tree

1 file changed

+152
-124
lines changed

1 file changed

+152
-124
lines changed

‎Tests/AppTests/QueryPerformanceTests.swift‎

Lines changed: 152 additions & 124 deletions
Original file line numberDiff line numberDiff line change
@@ -14,153 +14,172 @@
1414

1515
@testable import App
1616

17+
import Dependencies
1718
import SQLKit
19+
import Testing
1820
import Vapor
19-
import XCTest
2021

2122

22-
class QueryPerformanceTests: XCTestCase {
23-
var app: Application!
24-
23+
@Suite(
24+
.disabled(if: !runQueryPerformanceTests())
25+
)
26+
struct QueryPerformanceTests {
2527
// Set this to true when running locally to convert warnings to test failures for easier updating of values.
2628
static let failOnWarning = false
2729

28-
override func setUp() async throws {
29-
try await super.setUp()
30-
31-
try XCTSkipUnless(runQueryPerformanceTests)
32-
30+
func withStagingApp(_ test: (Application) async throws -> Void) async throws {
3331
// Update db settings for CI runs in
3432
// https://github.com/SwiftPackageIndex/SwiftPackageIndex-Server/settings/secrets/actions
3533
// or in `.env.staging` for local runs.
36-
self.app = try await Application.make(.staging)
37-
self.app.logger.logLevel = Environment.get("LOG_LEVEL")
34+
let app = try await Application.make(.staging)
35+
app.logger.logLevel = Environment.get("LOG_LEVEL")
3836
.flatMap(Logger.Level.init(rawValue:)) ?? .warning
3937
let host = try await configure(app)
4038

41-
XCTAssert(host.hasPrefix("spi-dev-db"), "was: \(host)")
42-
XCTAssert(host.hasSuffix("postgres.database.azure.com"), "was: \(host)")
39+
try #require(host.hasPrefix("spi-dev-db"), "was: \(host)")
40+
try #require(host.hasSuffix("postgres.database.azure.com"), "was: \(host)")
41+
42+
return try await run {
43+
try await test(app)
44+
} defer: {
45+
try await app.asyncShutdown()
46+
}
4347
}
4448

45-
func test_01_Search_packageMatchQuery() async throws {
46-
let query = Search.packageMatchQueryBuilder(on: app.db, terms: ["a"], filters: [])
47-
try await assertQueryPerformance(query, expectedCost: 1800, variation: 150)
49+
@Test func queryPerformance_01_Search_packageMatchQuery() async throws {
50+
try await withStagingApp { app in
51+
let query = Search.packageMatchQueryBuilder(on: app.db, terms: ["a"], filters: [])
52+
try await assertQueryPerformance(query, expectedCost: 1800, variation: 150)
53+
}
4854
}
4955

50-
func test_02_Search_keywordMatchQuery() async throws {
51-
let query = Search.keywordMatchQueryBuilder(on: app.db, terms: ["a"])
52-
try await assertQueryPerformance(query, expectedCost: 5900, variation: 200)
56+
@Test func queryPerformance_02_Search_keywordMatchQuery() async throws {
57+
try await withStagingApp { app in
58+
let query = Search.keywordMatchQueryBuilder(on: app.db, terms: ["a"])
59+
try await assertQueryPerformance(query, expectedCost: 5900, variation: 200)
60+
}
5361
}
5462

55-
func test_03_Search_authorMatchQuery() async throws {
56-
let query = Search.authorMatchQueryBuilder(on: app.db, terms: ["a"])
57-
try await assertQueryPerformance(query, expectedCost: 1100, variation: 50)
63+
@Test func queryPerformance_03_Search_authorMatchQuery() async throws {
64+
try await withStagingApp { app in
65+
let query = Search.authorMatchQueryBuilder(on: app.db, terms: ["a"])
66+
try await assertQueryPerformance(query, expectedCost: 1100, variation: 50)
67+
}
5868
}
5969

60-
func test_04_Search_query_noFilter() async throws {
61-
let query = try Search.query(app.db, ["a"], page: 1)
62-
.unwrap()
63-
try await assertQueryPerformance(query, expectedCost: 8100, variation: 200)
70+
@Test func queryPerformance_04_Search_query_noFilter() async throws {
71+
try await withStagingApp { app in
72+
let query = try Search.query(app.db, ["a"], page: 1).unwrap()
73+
try await assertQueryPerformance(query, expectedCost: 8100, variation: 200)
74+
}
6475
}
6576

66-
func test_05_Search_query_authorFilter() async throws {
67-
let filter = try AuthorSearchFilter(expression: .init(operator: .is, value: "apple"))
68-
let query = try Search.query(app.db, ["a"], filters: [filter], page: 1)
69-
.unwrap()
70-
try await assertQueryPerformance(query, expectedCost: 7700, variation: 200)
77+
@Test func queryPerformance_05_Search_query_authorFilter() async throws {
78+
try await withStagingApp { app in
79+
let filter = try AuthorSearchFilter(expression: .init(operator: .is, value: "apple"))
80+
let query = try Search.query(app.db, ["a"], filters: [filter], page: 1).unwrap()
81+
try await assertQueryPerformance(query, expectedCost: 7700, variation: 200)
82+
}
7183
}
7284

73-
func test_06_Search_query_keywordFilter() async throws {
74-
let filter = try KeywordSearchFilter(expression: .init(operator: .is, value: "apple"))
75-
let query = try Search.query(app.db, ["a"], filters: [filter], page: 1)
76-
.unwrap()
77-
try await assertQueryPerformance(query, expectedCost: 7800, variation: 200)
85+
@Test func queryPerformance_06_Search_query_keywordFilter() async throws {
86+
try await withStagingApp { app in
87+
let filter = try KeywordSearchFilter(expression: .init(operator: .is, value: "apple"))
88+
let query = try Search.query(app.db, ["a"], filters: [filter], page: 1).unwrap()
89+
try await assertQueryPerformance(query, expectedCost: 7800, variation: 200)
90+
}
7891
}
7992

80-
func test_07_Search_query_lastActicityFilter() async throws {
81-
let filter = try LastActivitySearchFilter(expression: .init(operator: .greaterThan, value: "2000-01-01"))
82-
let query = try Search.query(app.db, ["a"], filters: [filter], page: 1)
83-
.unwrap()
84-
try await assertQueryPerformance(query, expectedCost: 8100, variation: 200)
93+
@Test func queryPerformance_07_Search_query_lastActicityFilter() async throws {
94+
try await withStagingApp { app in
95+
let filter = try LastActivitySearchFilter(expression: .init(operator: .greaterThan, value: "2000-01-01"))
96+
let query = try Search.query(app.db, ["a"], filters: [filter], page: 1).unwrap()
97+
try await assertQueryPerformance(query, expectedCost: 8100, variation: 200)
98+
}
8599
}
86100

87-
func test_08_Search_query_licenseFilter() async throws {
88-
let filter = try LicenseSearchFilter(expression: .init(operator: .is, value: "mit"))
89-
let query = try Search.query(app.db, ["a"], filters: [filter], page: 1)
90-
.unwrap()
91-
try await assertQueryPerformance(query, expectedCost: 8000, variation: 200)
101+
@Test func queryPerformance_08_Search_query_licenseFilter() async throws {
102+
try await withStagingApp { app in
103+
let filter = try LicenseSearchFilter(expression: .init(operator: .is, value: "mit"))
104+
let query = try Search.query(app.db, ["a"], filters: [filter], page: 1).unwrap()
105+
try await assertQueryPerformance(query, expectedCost: 8000, variation: 200)
106+
}
92107
}
93108

94-
func test_09_Search_query_platformFilter() async throws {
95-
let filter = try PlatformSearchFilter(expression: .init(operator: .is, value: "macos,ios"))
96-
let query = try Search.query(app.db, ["a"], filters: [filter], page: 1)
97-
.unwrap()
98-
try await assertQueryPerformance(query, expectedCost: 7900, variation: 200)
109+
@Test func queryPerformance_09_Search_query_platformFilter() async throws {
110+
try await withStagingApp { app in
111+
let filter = try PlatformSearchFilter(expression: .init(operator: .is, value: "macos,ios"))
112+
let query = try Search.query(app.db, ["a"], filters: [filter], page: 1).unwrap()
113+
try await assertQueryPerformance(query, expectedCost: 7900, variation: 200)
114+
}
99115
}
100116

101-
func test_10_Search_query_productTypeFilter() async throws {
102-
let filter = try ProductTypeSearchFilter(expression: .init(operator: .is, value: "plugin"))
103-
let query = try Search.query(app.db, ["a"], filters: [filter], page: 1)
104-
.unwrap()
105-
try await assertQueryPerformance(query, expectedCost: 7700, variation: 200)
117+
@Test func queryPerformance_10_Search_query_productTypeFilter() async throws {
118+
try await withStagingApp { app in
119+
let filter = try ProductTypeSearchFilter(expression: .init(operator: .is, value: "plugin"))
120+
let query = try Search.query(app.db, ["a"], filters: [filter], page: 1).unwrap()
121+
try await assertQueryPerformance(query, expectedCost: 7700, variation: 200)
122+
}
106123
}
107124

108-
func test_11_Search_query_starsFilter() async throws {
109-
let filter = try StarsSearchFilter(expression: .init(operator: .greaterThan, value: "5"))
110-
let query = try Search.query(app.db, ["a"], filters: [filter], page: 1)
111-
.unwrap()
112-
try await assertQueryPerformance(query, expectedCost: 7900, variation: 300)
125+
@Test func queryPerformance_11_Search_query_starsFilter() async throws {
126+
try await withStagingApp { app in
127+
let filter = try StarsSearchFilter(expression: .init(operator: .greaterThan, value: "5"))
128+
let query = try Search.query(app.db, ["a"], filters: [filter], page: 1).unwrap()
129+
try await assertQueryPerformance(query, expectedCost: 7900, variation: 300)
130+
}
113131
}
114132

115-
func test_12_Search_refresh() async throws {
133+
@Test func queryPerformance_12_Search_refresh() async throws {
116134
// We can't "explain analyze" the refresh itself so we need to measure the underlying
117135
// query.
118136
// Unfortunately, this means it'll need to be kept in sync when updating the search
119137
// view.
120-
guard let db = app.db as? SQLDatabase else {
121-
XCTFail()
122-
return
138+
try await withStagingApp { app in
139+
guard let db = app.db as? SQLDatabase else {
140+
Issue.record()
141+
return
142+
}
143+
let query = db.raw("""
144+
-- v12
145+
SELECT
146+
p.id AS package_id,
147+
p.platform_compatibility,
148+
p.score,
149+
r.keywords,
150+
r.last_commit_date,
151+
r.license,
152+
r.name AS repo_name,
153+
r.owner AS repo_owner,
154+
r.stars,
155+
r.last_activity_at,
156+
r.summary,
157+
v.package_name,
158+
(
159+
ARRAY_LENGTH(doc_archives, 1) >= 1
160+
OR spi_manifest->'external_links'->'documentation' IS NOT NULL
161+
) AS has_docs,
162+
ARRAY(
163+
SELECT DISTINCT JSONB_OBJECT_KEYS(type) FROM products WHERE products.version_id = v.id
164+
UNION
165+
SELECT * FROM (
166+
SELECT DISTINCT JSONB_OBJECT_KEYS(type) AS "type" FROM targets
167+
WHERE targets.version_id = v.id) AS macro_targets
168+
WHERE type = 'macro'
169+
) AS product_types,
170+
ARRAY(SELECT DISTINCT name FROM products WHERE products.version_id = v.id) AS product_names,
171+
TO_TSVECTOR(CONCAT_WS(' ', COALESCE(v.package_name, ''), r.name, COALESCE(r.summary, ''), ARRAY_TO_STRING(r.keywords, ' '))) AS tsvector
172+
FROM packages p
173+
JOIN repositories r ON r.package_id = p.id
174+
JOIN versions v ON v.package_id = p.id
175+
WHERE v.reference ->> 'branch' = r.default_branch
176+
""")
177+
try await assertQueryPerformance(query, expectedCost: 132_000, variation: 5000)
123178
}
124-
let query = db.raw("""
125-
-- v12
126-
SELECT
127-
p.id AS package_id,
128-
p.platform_compatibility,
129-
p.score,
130-
r.keywords,
131-
r.last_commit_date,
132-
r.license,
133-
r.name AS repo_name,
134-
r.owner AS repo_owner,
135-
r.stars,
136-
r.last_activity_at,
137-
r.summary,
138-
v.package_name,
139-
(
140-
ARRAY_LENGTH(doc_archives, 1) >= 1
141-
OR spi_manifest->'external_links'->'documentation' IS NOT NULL
142-
) AS has_docs,
143-
ARRAY(
144-
SELECT DISTINCT JSONB_OBJECT_KEYS(type) FROM products WHERE products.version_id = v.id
145-
UNION
146-
SELECT * FROM (
147-
SELECT DISTINCT JSONB_OBJECT_KEYS(type) AS "type" FROM targets
148-
WHERE targets.version_id = v.id) AS macro_targets
149-
WHERE type = 'macro'
150-
) AS product_types,
151-
ARRAY(SELECT DISTINCT name FROM products WHERE products.version_id = v.id) AS product_names,
152-
TO_TSVECTOR(CONCAT_WS(' ', COALESCE(v.package_name, ''), r.name, COALESCE(r.summary, ''), ARRAY_TO_STRING(r.keywords, ' '))) AS tsvector
153-
FROM packages p
154-
JOIN repositories r ON r.package_id = p.id
155-
JOIN versions v ON v.package_id = p.id
156-
WHERE v.reference ->> 'branch' = r.default_branch
157-
""")
158-
try await assertQueryPerformance(query, expectedCost: 132_000, variation: 5000)
159179
}
160180

161181
}
162182

163-
164183
// MARK: - Query plan helpers
165184

166185

@@ -169,6 +188,11 @@ private extension Environment {
169188
}
170189

171190

191+
private func runQueryPerformanceTests() -> Bool {
192+
ProcessInfo.processInfo.environment.keys.contains("RUN_QUERY_PERFORMANCE_TESTS")
193+
}
194+
195+
172196
final class SQLQueryExplainer: SQLQueryFetcher {
173197
var query: any SQLExpression
174198
var database: any SQLDatabase
@@ -201,19 +225,21 @@ private extension QueryPerformanceTests {
201225
variation: Double = 0,
202226
filePath: StaticString = #filePath,
203227
lineNumber: UInt = #line,
204-
testName: String = #function) async throws {
228+
testName: String = #function,
229+
sourceLocation: Testing.SourceLocation = #_sourceLocation) async throws {
205230
let queryPlan = try await query.explain()
206231
let parsedPlan = try QueryPlan(queryPlan)
207232
print("ℹ️ TEST: \(testName)")
208233
if parsedPlan.cost.total <= expectedCost {
209234
print("ℹ️ COST: \(parsedPlan.cost.total)")
210235
} else {
211236
if Self.failOnWarning {
212-
XCTFail("""
213-
Total cost of \(parsedPlan.cost.total) above the expected cost of \(expectedCost)
214-
""",
215-
file: filePath,
216-
line: lineNumber)
237+
Issue.record(
238+
"""
239+
Total cost of \(parsedPlan.cost.total) above the expected cost of \(expectedCost)
240+
""",
241+
sourceLocation: sourceLocation
242+
)
217243
} else {
218244
print("⚠️ COST: \(parsedPlan.cost.total)")
219245
}
@@ -223,35 +249,37 @@ private extension QueryPerformanceTests {
223249

224250
switch parsedPlan.cost.total {
225251
case ..<10.0:
226-
if isRunningInCI {
252+
if isRunningInCI() {
227253
print("::error file=\(filePath),line=\(lineNumber),title=\(testName)::Cost very low \(parsedPlan.cost.total) - did you run the query against an empty database?")
228254
}
229-
XCTFail("""
230-
Cost very low \(parsedPlan.cost.total) - did you run the query against an empty database?
231-
232-
\(queryPlan)
233-
""",
234-
file: filePath,
235-
line: lineNumber)
255+
Issue.record(
256+
"""
257+
Cost very low \(parsedPlan.cost.total) - did you run the query against an empty database?
258+
259+
\(queryPlan)
260+
""",
261+
sourceLocation: sourceLocation
262+
)
236263
case ..<expectedCost:
237264
break
238265
case ..<(expectedCost + variation):
239-
if isRunningInCI {
266+
if isRunningInCI() {
240267
print("::warning file=\(filePath),line=\(lineNumber),title=\(testName)::Total cost of \(parsedPlan.cost.total) close to the threshold of \(expectedCost + variation)")
241268
}
242269
default:
243-
if isRunningInCI {
270+
if isRunningInCI() {
244271
print("::error file=\(filePath),line=\(lineNumber),title=\(testName)::Total cost of \(parsedPlan.cost.total) above the threshold of \(expectedCost + variation)")
245272
}
246-
XCTFail("""
247-
Total cost of \(parsedPlan.cost.total) above the threshold of \(expectedCost + variation)
273+
Issue.record(
274+
"""
275+
Total cost of \(parsedPlan.cost.total) above the threshold of \(expectedCost + variation)
248276
249-
Query plan:
277+
Query plan:
250278
251-
\(queryPlan)
252-
""",
253-
file: filePath,
254-
line: lineNumber)
279+
\(queryPlan)
280+
""",
281+
sourceLocation: sourceLocation
282+
)
255283
}
256284
}
257285

0 commit comments

Comments
 (0)