Skip to content

Commit cfb5885

Browse files
hkandalahkandala
committed
[yugabyte#9365] Platform: Optimise CertificateInfo.getAll by populating universe details in batch rather than individually
Summary: CertificateInfo has two transient parameters inUse and universeDetails whose values are fetched through their corresponding getters. But the getter fetches all universes for the given customer and search through it. This is very inefficient when fetching all certificates at once. Modified the getter to cache the result and perform the universe search only if the value is not present. Also made changes in getAll such that inUse and universeDetails are set efficiently by fetching universe details only once from DB. Test Plan: Loaded entire dev portal data on local using pg_dump and compared API performance before and after the change Before change: 504 Timeout After change: 800ms to 1s average time Note: Dev portal contains 55 universes and 467 certificates at the time of testing Reviewers: hsu, spotachev Reviewed By: spotachev Subscribers: jenkins-bot, yugaware Differential Revision: https://phabricator.dev.yugabyte.com/D12365
1 parent 3fd0736 commit cfb5885

File tree

2 files changed

+227
-4
lines changed

2 files changed

+227
-4
lines changed

‎managed/src/main/java/com/yugabyte/yw/models/CertificateInfo.java‎

Lines changed: 74 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,19 @@
2222
import java.io.IOException;
2323
import java.security.NoSuchAlgorithmException;
2424
import java.util.Date;
25+
import java.util.HashMap;
26+
import java.util.HashSet;
2527
import java.util.List;
28+
import java.util.Map;
2629
import java.util.Set;
2730
import java.util.UUID;
31+
import java.util.stream.Collectors;
2832
import javax.persistence.Column;
2933
import javax.persistence.Entity;
3034
import javax.persistence.EnumType;
3135
import javax.persistence.Enumerated;
3236
import javax.persistence.Id;
37+
import javax.persistence.Transient;
3338
import org.slf4j.Logger;
3439
import org.slf4j.LoggerFactory;
3540
import play.data.validation.Constraints;
@@ -279,7 +284,10 @@ public static List<CertificateInfo> getAllNoChecksum() {
279284
}
280285

281286
public static List<CertificateInfo> getAll(UUID customerUUID) {
282-
return find.query().where().eq("customer_uuid", customerUUID).findList();
287+
List<CertificateInfo> certificateInfoList =
288+
find.query().where().eq("customer_uuid", customerUUID).findList();
289+
populateUniverseData(customerUUID, certificateInfoList);
290+
return certificateInfoList;
283291
}
284292

285293
public static boolean isCertificateValid(UUID certUUID) {
@@ -297,20 +305,82 @@ public static boolean isCertificateValid(UUID certUUID) {
297305
return true;
298306
}
299307

308+
@Transient private Boolean inUse = null;
309+
300310
@ApiModelProperty(
301311
value = "Indicates whether the Certificate is in use or not",
302312
accessMode = READ_ONLY)
303313
// Returns if there is an in use reference to the object.
304314
public boolean getInUse() {
305-
return Universe.existsCertificate(this.uuid, this.customerUUID);
315+
if (inUse == null) {
316+
return Universe.existsCertificate(this.uuid, this.customerUUID);
317+
} else {
318+
return inUse;
319+
}
306320
}
307321

322+
public void setInUse(boolean inUse) {
323+
this.inUse = inUse;
324+
}
325+
326+
@Transient private ArrayNode universeDetails = null;
327+
308328
@ApiModelProperty(
309329
value = "Associated universe details of the Certificate",
310330
accessMode = READ_ONLY)
311331
public ArrayNode getUniverseDetails() {
312-
Set<Universe> universes = Universe.universeDetailsIfCertsExists(this.uuid, this.customerUUID);
313-
return Util.getUniverseDetails(universes);
332+
if (universeDetails == null) {
333+
Set<Universe> universes = Universe.universeDetailsIfCertsExists(this.uuid, this.customerUUID);
334+
return Util.getUniverseDetails(universes);
335+
} else {
336+
return universeDetails;
337+
}
338+
}
339+
340+
public void setUniverseDetails(ArrayNode universeDetails) {
341+
this.universeDetails = universeDetails;
342+
}
343+
344+
public static void populateUniverseData(
345+
UUID customerUUID, List<CertificateInfo> certificateInfoList) {
346+
Set<Universe> universes = Customer.get(customerUUID).getUniverses();
347+
Set<UUID> certificateInfoSet =
348+
certificateInfoList.stream().map(e -> e.uuid).collect(Collectors.toSet());
349+
350+
Map<UUID, Set<Universe>> certificateUniverseMap = new HashMap<>();
351+
universes.forEach(
352+
universe -> {
353+
UUID rootCA = universe.getUniverseDetails().rootCA;
354+
UUID clientRootCA = universe.getUniverseDetails().clientRootCA;
355+
if (rootCA != null) {
356+
if (certificateInfoSet.contains(rootCA)) {
357+
certificateUniverseMap.putIfAbsent(rootCA, new HashSet<>());
358+
certificateUniverseMap.get(rootCA).add(universe);
359+
} else {
360+
LOG.error("Universe: {} has unknown rootCA: {}", universe.universeUUID, rootCA);
361+
}
362+
}
363+
if (clientRootCA != null && !clientRootCA.equals(rootCA)) {
364+
if (certificateInfoSet.contains(clientRootCA)) {
365+
certificateUniverseMap.putIfAbsent(clientRootCA, new HashSet<>());
366+
certificateUniverseMap.get(clientRootCA).add(universe);
367+
} else {
368+
LOG.error("Universe: {} has unknown clientRootCA: {}", universe.universeUUID, rootCA);
369+
}
370+
}
371+
});
372+
373+
certificateInfoList.forEach(
374+
certificateInfo -> {
375+
if (certificateUniverseMap.containsKey(certificateInfo.uuid)) {
376+
certificateInfo.setInUse(true);
377+
certificateInfo.setUniverseDetails(
378+
Util.getUniverseDetails(certificateUniverseMap.get(certificateInfo.uuid)));
379+
} else {
380+
certificateInfo.setInUse(false);
381+
certificateInfo.setUniverseDetails(Json.newArray());
382+
}
383+
});
314384
}
315385

316386
public static void delete(UUID certUUID, UUID customerUUID) {

‎managed/src/test/java/com/yugabyte/yw/models/CertificateInfoTest.java‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
// Copyright (c) YugaByte, Inc.
2+
3+
package com.yugabyte.yw.models;
4+
5+
import static com.yugabyte.yw.common.ModelFactory.createUniverse;
6+
import static org.junit.Assert.assertEquals;
7+
import static org.junit.Assert.assertFalse;
8+
import static org.junit.Assert.assertNotEquals;
9+
import static org.junit.Assert.assertNotNull;
10+
import static org.junit.Assert.assertTrue;
11+
12+
import com.fasterxml.jackson.databind.node.ArrayNode;
13+
import com.yugabyte.yw.commissioner.Common;
14+
import com.yugabyte.yw.common.CertificateHelper;
15+
import com.yugabyte.yw.common.FakeDBApplication;
16+
import com.yugabyte.yw.common.ModelFactory;
17+
import java.io.File;
18+
import java.io.IOException;
19+
import java.lang.reflect.Field;
20+
import java.util.ArrayList;
21+
import java.util.Arrays;
22+
import java.util.List;
23+
import java.util.UUID;
24+
import org.apache.commons.io.FileUtils;
25+
import org.junit.After;
26+
import org.junit.Before;
27+
import org.junit.Test;
28+
import org.junit.runner.RunWith;
29+
import org.mockito.junit.MockitoJUnitRunner;
30+
31+
@RunWith(MockitoJUnitRunner.class)
32+
public class CertificateInfoTest extends FakeDBApplication {
33+
34+
private Customer customer;
35+
36+
private final List<String> certList = Arrays.asList("test_cert1", "test_cert2", "test_cert3");
37+
private final List<UUID> certIdList = new ArrayList<>();
38+
39+
@Before
40+
public void setUp() {
41+
customer = ModelFactory.testCustomer();
42+
for (String cert : certList) {
43+
certIdList.add(CertificateHelper.createRootCA(cert, customer.uuid, "/tmp/certs"));
44+
}
45+
}
46+
47+
@After
48+
public void tearDown() throws IOException {
49+
FileUtils.deleteDirectory(new File("/tmp/certs"));
50+
}
51+
52+
@Test
53+
public void testGetAllWithNoUniverses() {
54+
List<CertificateInfo> certificateInfoList = CertificateInfo.getAll(customer.uuid);
55+
assertEquals(3, certificateInfoList.size());
56+
for (CertificateInfo cert : certificateInfoList) {
57+
assertFalse(cert.getInUse());
58+
assertEquals(0, cert.getUniverseDetails().size());
59+
}
60+
}
61+
62+
@Test
63+
public void testGetAllWithMultipleUniverses() {
64+
Universe universe1 =
65+
createUniverse(
66+
"Test Universe 1",
67+
UUID.randomUUID(),
68+
customer.getCustomerId(),
69+
Common.CloudType.aws,
70+
null,
71+
certIdList.get(0));
72+
createUniverse(
73+
"Test Universe 2",
74+
UUID.randomUUID(),
75+
customer.getCustomerId(),
76+
Common.CloudType.aws,
77+
null,
78+
certIdList.get(1));
79+
createUniverse(
80+
"Test Universe 3",
81+
UUID.randomUUID(),
82+
customer.getCustomerId(),
83+
Common.CloudType.aws,
84+
null,
85+
certIdList.get(1));
86+
87+
List<CertificateInfo> certificateInfoList = CertificateInfo.getAll(customer.uuid);
88+
assertEquals(3, certificateInfoList.size());
89+
for (CertificateInfo cert : certificateInfoList) {
90+
if (cert.uuid.equals(certIdList.get(0))) {
91+
assertTrue(cert.getInUse());
92+
assertEquals(
93+
universe1.universeUUID.toString(),
94+
cert.getUniverseDetails().get(0).get("uuid").asText());
95+
} else if (cert.uuid.equals(certIdList.get(1))) {
96+
assertTrue(cert.getInUse());
97+
assertEquals(2, cert.getUniverseDetails().size());
98+
assertNotEquals(
99+
universe1.universeUUID.toString(),
100+
cert.getUniverseDetails().get(0).get("uuid").asText());
101+
assertNotEquals(
102+
universe1.universeUUID.toString(),
103+
cert.getUniverseDetails().get(1).get("uuid").asText());
104+
} else {
105+
assertFalse(cert.getInUse());
106+
assertEquals(0, cert.getUniverseDetails().size());
107+
}
108+
}
109+
}
110+
111+
@Test
112+
public void testGetAllUniverseDetailsInvocation()
113+
throws NoSuchFieldException, IllegalAccessException {
114+
createUniverse(
115+
"Test Universe 1",
116+
UUID.randomUUID(),
117+
customer.getCustomerId(),
118+
Common.CloudType.aws,
119+
null,
120+
certIdList.get(0));
121+
createUniverse(
122+
"Test Universe 2",
123+
UUID.randomUUID(),
124+
customer.getCustomerId(),
125+
Common.CloudType.aws,
126+
null,
127+
certIdList.get(1));
128+
createUniverse(
129+
"Test Universe 3",
130+
UUID.randomUUID(),
131+
customer.getCustomerId(),
132+
Common.CloudType.aws,
133+
null,
134+
certIdList.get(1));
135+
136+
List<CertificateInfo> certificateInfoList = CertificateInfo.getAll(customer.uuid);
137+
assertEquals(3, certificateInfoList.size());
138+
139+
Field inUseField = CertificateInfo.class.getDeclaredField("inUse");
140+
Field universeDetailsField = CertificateInfo.class.getDeclaredField("universeDetails");
141+
inUseField.setAccessible(true);
142+
universeDetailsField.setAccessible(true);
143+
144+
for (CertificateInfo cert : certificateInfoList) {
145+
Boolean inUse = (Boolean) inUseField.get(cert);
146+
ArrayNode universeDetails = (ArrayNode) universeDetailsField.get(cert);
147+
// If the private fields inUse and universeDetails are not null then
148+
// universeDetails are already populated and won't lead to individual universe data fetch
149+
assertNotNull(inUse);
150+
assertNotNull(universeDetails);
151+
}
152+
}
153+
}

0 commit comments

Comments
 (0)