[PLAT-13287] Add a local provider retry test for StartMaster

yorq · yorq · commit 82d5a82cdeaa · 2024-10-23T13:23:04.000+03:00
Summary: Added retry test in local provider framework. Test Plan: sbt test Reviewers: nsingh Reviewed By: nsingh Subscribers: svc_phabricator, yugaware Differential Revision: https://phorge.dev.yugabyte.com/D37979
diff --git a/managed/src/main/java/com/yugabyte/yw/common/LocalNodeManager.java b/managed/src/main/java/com/yugabyte/yw/common/LocalNodeManager.java
@@ -141,7 +141,7 @@ public void shutdown() {
                     killPostMasterProcess(baseDir);
                   }
                   log.debug("Destroying {}", process.pid());
-                  killProcess(process.pid());
+                  killProcess(process.pid(), false);
                 } catch (Exception e) {
                   log.error("Failed to destroy process " + process, e);
                 }
@@ -202,16 +202,16 @@ private void killPostMasterProcess(String path) {
     }
   }
 
-  private void killProcess(long pid) throws IOException, InterruptedException {
+  private void killProcess(long pid, boolean throwIfAbsent) {
     try {
-      terminateProcessAndSubprocesses(pid);
+      terminateProcessAndSubprocesses(pid, throwIfAbsent);
     } catch (SecurityException | IllegalArgumentException e) {
       System.err.println("Error occurred while terminating process: " + e.getMessage());
       e.printStackTrace();
     }
   }
 
-  private void terminateProcessAndSubprocesses(long pid) {
+  private void terminateProcessAndSubprocesses(long pid, boolean throwIfAbsent) {
     ProcessHandle.of(pid)
         .ifPresentOrElse(
             process -> {
@@ -221,7 +221,9 @@ private void terminateProcessAndSubprocesses(long pid) {
               process.destroy();
             },
             () -> {
-              throw new IllegalArgumentException("No such process with PID: " + pid);
+              if (throwIfAbsent) {
+                throw new IllegalArgumentException("No such process with PID: " + pid);
+              }
             });
   }
 
@@ -233,7 +235,7 @@ public void killProcess(String nodeName, UniverseTaskBase.ServerType serverType)
       Process process = nodeInfo.processMap.get(serverType);
       if (process != null) {
         log.debug("Destroying process with pid {} for {}", process.pid(), nodeInfo.ip);
-        killProcess(process.pid());
+        killProcess(process.pid(), true);
       }
     }
   }
@@ -702,6 +704,13 @@ public void startProcessForNode(
       UniverseDefinitionTaskParams.UserIntent userIntent,
       UniverseTaskBase.ServerType serverType,
       NodeInfo nodeInfo) {
+    if (nodeInfo.processMap.get(serverType) != null) {
+      Process process = nodeInfo.processMap.get(serverType);
+      if (process.isAlive()) {
+        log.debug("Already have started process");
+        return;
+      }
+    }
     List<String> args = new ArrayList<>();
     String executable;
     LocalCloudInfo localCloudInfo = getCloudInfo(userIntent);
@@ -756,12 +765,7 @@ private void stopProcessForNode(
       throw new IllegalStateException("No process of type " + serverType + " for " + nodeInfo.name);
     }
     log.debug("Killing process {}", process.pid());
-    try {
-      killProcess(process.pid());
-    } catch (IOException | InterruptedException e) {
-      System.err.println("Error occurred while terminating process: " + e.getMessage());
-      e.printStackTrace();
-    }
+    killProcess(process.pid(), true);
   }
 
   private static List<String> readProcessIdsFromFile(String filePath) {
diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/local/GFlagsUpgradeLocalTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/local/GFlagsUpgradeLocalTest.java
@@ -551,7 +551,7 @@ public void testNodesAreSafeToTakeDownFails() throws InterruptedException, IOExc
     assertNull(universe.getUniverseDetails().placementModificationTaskUuid);
 
     // Revert setting
-    RuntimeConfigEntry.upsertGlobal(UniverseConfKeys.followerLagMaxThreshold.getKey(), "30s");
+    RuntimeConfigEntry.upsertGlobal(UniverseConfKeys.followerLagMaxThreshold.getKey(), "60s");
 
     // Now it should be successful
     doGflagsUpgrade(
diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/local/LocalProviderUniverseTestBase.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/local/LocalProviderUniverseTestBase.java
@@ -26,12 +26,12 @@
 import com.yugabyte.yw.commissioner.tasks.subtasks.CheckClusterConsistency;
 import com.yugabyte.yw.common.ApiUtils;
 import com.yugabyte.yw.common.ConfigHelper;
+import com.yugabyte.yw.common.CustomerTaskManager;
 import com.yugabyte.yw.common.LocalNodeManager;
 import com.yugabyte.yw.common.LocalNodeUniverseManager;
 import com.yugabyte.yw.common.ModelFactory;
 import com.yugabyte.yw.common.NodeUIApiHelper;
 import com.yugabyte.yw.common.PlacementInfoUtil;
-import com.yugabyte.yw.common.PlatformGuiceApplicationBaseTest;
 import com.yugabyte.yw.common.ReleaseManager;
 import com.yugabyte.yw.common.RetryTaskUntilCondition;
 import com.yugabyte.yw.common.ShellResponse;
@@ -126,7 +126,7 @@
 import play.libs.Json;
 
 @Slf4j
-public abstract class LocalProviderUniverseTestBase extends PlatformGuiceApplicationBaseTest {
+public abstract class LocalProviderUniverseTestBase extends CommissionerBaseTest {
   private static final boolean IS_LINUX = System.getProperty("os.name").equalsIgnoreCase("linux");
   private static final Set<String> CONTROL_FILES =
       Set.of(LocalNodeManager.MASTER_EXECUTABLE, LocalNodeManager.TSERVER_EXECUTABLE);
@@ -218,7 +218,6 @@ public Map<String, String> getYbcGFlags(UniverseDefinitionTaskParams.UserIntent
   protected YcqlQueryExecutor ycqlQueryExecutor;
   protected UniverseTableHandler tableHandler;
   protected CertificateHelper certificateHelper;
-  protected Commissioner commissioner;
   protected SettableRuntimeConfigFactory settableRuntimeConfigFactory;
   protected RuntimeConfService runtimeConfService;
   protected JobScheduler jobScheduler;
@@ -427,13 +426,21 @@ private void injectDependencies() {
     runtimeConfService = app.injector().instanceOf(RuntimeConfService.class);
     jobScheduler = app.injector().instanceOf(JobScheduler.class);
     autoMasterFailoverScheduler = app.injector().instanceOf(AutoMasterFailoverScheduler.class);
+    customerTaskManager = app.injector().instanceOf(CustomerTaskManager.class);
   }
 
   @Before
   public void setUp() {
     injectDependencies();
 
     settableRuntimeConfigFactory.globalRuntimeConf().setValue("yb.releases.use_redesign", "false");
+    settableRuntimeConfigFactory
+        .globalRuntimeConf()
+        .setValue(GlobalConfKeys.startMasterOnRemoveNode.getKey(), "true");
+    settableRuntimeConfigFactory
+        .globalRuntimeConf()
+        .setValue(GlobalConfKeys.startMasterOnStopNode.getKey(), "true");
+
     settableRuntimeConfigFactory
         .globalRuntimeConf()
         .setValue("yb.universe.consistency_check.enabled", "true");
diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/local/NodeOperationsLocalTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/local/NodeOperationsLocalTest.java
@@ -10,16 +10,21 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.yugabyte.yw.commissioner.tasks.CommissionerBaseTest;
 import com.yugabyte.yw.commissioner.tasks.UniverseTaskBase;
+import com.yugabyte.yw.commissioner.tasks.params.NodeTaskParams;
 import com.yugabyte.yw.common.FakeApiHelper;
 import com.yugabyte.yw.common.LocalNodeManager;
 import com.yugabyte.yw.common.NodeActionType;
+import com.yugabyte.yw.common.PlacementInfoUtil;
 import com.yugabyte.yw.common.gflags.SpecificGFlags;
 import com.yugabyte.yw.common.utils.Pair;
+import com.yugabyte.yw.controllers.UniverseControllerRequestBinder;
 import com.yugabyte.yw.forms.NodeActionFormData;
 import com.yugabyte.yw.forms.UniverseDefinitionTaskParams;
+import com.yugabyte.yw.models.CustomerTask;
 import com.yugabyte.yw.models.TaskInfo;
 import com.yugabyte.yw.models.Universe;
 import com.yugabyte.yw.models.helpers.NodeDetails;
+import com.yugabyte.yw.models.helpers.TaskType;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
@@ -110,6 +115,7 @@ public void testMasterNodeRemoval() throws InterruptedException {
     String nodeName = nodeDetails.nodeName;
     NodeActionFormData formData = new NodeActionFormData();
     formData.nodeAction = NodeActionType.STOP;
+    NodeActionFormData.startMasterOnStopNode = true;
     Result result = nodeOperationInUniverse(universe.getUniverseUUID(), nodeName, formData);
     checkAndWaitForTask(result);
     universe = Universe.getOrBadRequest(universe.getUniverseUUID());
@@ -175,7 +181,53 @@ public void testRemoveReleaseNodeFromUniverse() throws InterruptedException {
     verifyUniverseState(universe);
   }
 
-  // @Test
+  @Test
+  public void testStartMasterRetries() throws InterruptedException {
+    Universe universe = createUniverse(4, 3);
+    Map<UUID, Integer> zoneToCount =
+        PlacementInfoUtil.getAzUuidToNumNodes(
+            universe.getUniverseDetails().getPrimaryCluster().placementInfo);
+
+    NodeDetails nodeWithMaster =
+        universe.getUniverseDetails().nodeDetailsSet.stream()
+            .filter(n -> n.isMaster && zoneToCount.get(n.azUuid) == 2)
+            .findFirst()
+            .get();
+    NodeActionFormData.startMasterOnStopNode = false;
+    NodeTaskParams taskParams =
+        UniverseControllerRequestBinder.deepCopy(
+            universe.getUniverseDetails(), NodeTaskParams.class);
+    NodeActionType nodeActionType = NodeActionType.STOP;
+    taskParams.nodeName = nodeWithMaster.getNodeName();
+    UUID taskUUID = commissioner.submit(nodeActionType.getCommissionerTask(), taskParams);
+    TaskInfo taskInfo = CommissionerBaseTest.waitForTask(taskUUID, 500);
+    assertEquals(TaskInfo.State.Success, taskInfo.getTaskState());
+    universe = Universe.getOrBadRequest(universe.getUniverseUUID());
+    assertEquals(2, universe.getMasters().size());
+    NodeDetails nodeWithoutMaster =
+        universe.getUniverseDetails().nodeDetailsSet.stream()
+            .filter(n -> !n.isMaster && !n.getNodeName().equals(nodeWithMaster.getNodeName()))
+            .findFirst()
+            .get();
+
+    taskParams =
+        UniverseControllerRequestBinder.deepCopy(
+            universe.getUniverseDetails(), NodeTaskParams.class);
+    taskParams.nodeName = nodeWithoutMaster.getNodeName();
+
+    super.verifyTaskRetries(
+        customer,
+        CustomerTask.TaskType.StartMaster,
+        CustomerTask.TargetType.Universe,
+        universe.getUniverseUUID(),
+        TaskType.StartMasterOnNode,
+        taskParams,
+        false);
+    universe = Universe.getOrBadRequest(universe.getUniverseUUID());
+    verifyUniverseState(universe);
+  }
+
+  @Test
   public void testStartAlreadyStarted() throws InterruptedException {
     Universe universe = createUniverse(3, 1);
     NodeDetails nodeDetails =
