From 0c07630f239f7ea927cd7463b057e72a5a52fece Mon Sep 17 00:00:00 2001 From: David Kyle Date: Wed, 8 Oct 2025 12:11:42 +0100 Subject: [PATCH 1/4] [ML] Fix and unmute Transform upgrade test (#132995) (cherry picked from commit 208ae578e089b6f40ba4339cab883d72631e40eb) # Conflicts: # muted-tests.yml # x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java --- .../TransportUpgradeTransformsAction.java | 3 +-- .../IndexBasedTransformConfigManager.java | 19 +++++++++++-------- .../upgrades/MLModelDeploymentsUpgradeIT.java | 4 ++-- .../upgrades/TransformSurvivesUpgradeIT.java | 17 ++++++++++------- 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java index b91cf6d3bb5b6..f8d6c399f9fda 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java @@ -29,7 +29,6 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackSettings; -import org.elasticsearch.xpack.core.ml.utils.TransportVersionUtils; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.transform.TransformMetadata; import org.elasticsearch.xpack.core.transform.action.UpgradeTransformsAction; @@ -107,7 +106,7 @@ protected void masterOperation(Task ignoredTask, Request request, ClusterState s } // do not allow in mixed clusters - if (TransportVersionUtils.isMinTransportVersionSameAsCurrent(state) == false) { + if (state.nodes().isMixedVersionCluster()) { listener.onFailure( new ElasticsearchStatusException("Cannot upgrade transforms while cluster upgrade is in progress.", RestStatus.CONFLICT) ); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java index 8e579c9357621..586afc405f787 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java @@ -295,16 +295,19 @@ public void deleteOldIndices(ActionListener listener) { // use the transform context as we access system indexes try (ThreadContext.StoredContext ctx = client.threadPool().getThreadContext().stashWithOrigin(TRANSFORM_ORIGIN)) { - indicesToDelete.addAll( - Arrays.asList( - indexNameExpressionResolver.concreteIndexNames( - state, - IndicesOptions.lenientExpandHidden(), - TransformInternalIndexConstants.INDEX_NAME_PATTERN - ) - ) + var matchingIndexes = indexNameExpressionResolver.concreteIndices( + state, + IndicesOptions.lenientExpandHidden(), + TransformInternalIndexConstants.INDEX_NAME_PATTERN ); + for (var index : matchingIndexes) { + var meta = state.getMetadata().indexMetadata(index); + if (meta.isSystem() == false) { // ignore system indices as these are automatically managed + indicesToDelete.add(meta.getIndex().getName()); + } + } + indicesToDelete.addAll( Arrays.asList( indexNameExpressionResolver.concreteIndexNames( diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java index adeaeae31be45..06a7f9f4047fe 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java @@ -73,10 +73,10 @@ public void setUpLogging() throws IOException { request.setJsonEntity(""" { "persistent": { - "logger.org.elasticsearch.xpack.ml.inference": "TRACE", + "logger.org.elasticsearch.xpack.ml.inference": "DEBUG", "logger.org.elasticsearch.xpack.ml.inference.assignments": "DEBUG", "logger.org.elasticsearch.xpack.ml.process": "DEBUG", - "logger.org.elasticsearch.xpack.ml.action": "TRACE" + "logger.org.elasticsearch.xpack.ml.action": "DEBUG" } } """); diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java index 87a9911bd80b6..2e98ebc57cc0f 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java @@ -9,6 +9,7 @@ import org.apache.http.HttpHost; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; +import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; @@ -131,11 +132,11 @@ private void createAndStartContinuousTransform() throws Exception { assertBusy(() -> { var stateAndStats = getTransformStats(CONTINUOUS_TRANSFORM_ID); + assertThat((Integer) XContentMapValues.extractValue("stats.documents_indexed", stateAndStats), equalTo(ENTITIES.size())); assertThat( - ((Integer) XContentMapValues.extractValue("stats.documents_indexed", stateAndStats)).longValue(), - equalTo(ENTITIES.size()) + ((Integer) XContentMapValues.extractValue("stats.documents_processed", stateAndStats)).longValue(), + equalTo(totalDocsWritten) ); - assertThat((Integer) XContentMapValues.extractValue("stats.documents_processed", stateAndStats), equalTo(totalDocsWritten)); // Even if we get back to started, we may periodically get set back to `indexing` when triggered. // Though short lived due to no changes on the source indices, it could result in flaky test behavior assertThat(stateAndStats.get("state"), oneOf("started", "indexing")); @@ -237,10 +238,12 @@ private void verifyUpgradeFailsIfMixedCluster() { if (isOriginalClusterCurrent()) { return; } - final Request upgradeTransformRequest = new Request("POST", getTransformEndpoint() + "_upgrade"); - - Exception ex = expectThrows(Exception.class, () -> client().performRequest(upgradeTransformRequest)); - assertThat(ex.getMessage(), containsString("All nodes must be the same version")); + var oldestVersion = Version.fromString(UPGRADE_FROM_VERSION); + if (oldestVersion.onOrAfter(Version.V_9_3_0)) { + final Request upgradeTransformRequest = new Request("POST", getTransformEndpoint() + "_upgrade"); + Exception ex = expectThrows(Exception.class, () -> client().performRequest(upgradeTransformRequest)); + assertThat(ex.getMessage(), containsString("Cannot upgrade transforms while cluster upgrade is in progress")); + } } private void verifyUpgrade() throws IOException { From 1330fb867a00d02aff8cae86692064a82f52ec46 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Fri, 10 Oct 2025 13:29:49 +0100 Subject: [PATCH 2/4] Update TransformSurvivesUpgradeIT.java --- .../upgrades/TransformSurvivesUpgradeIT.java | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java index 2e98ebc57cc0f..cc5af60982a0b 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java @@ -96,8 +96,7 @@ public void testTransformRollingUpgrade() throws Exception { if (Booleans.parseBoolean(System.getProperty("tests.first_round")) == false) { lastCheckpoint = 2; } - verifyContinuousTransformHandlesData(lastCheckpoint); - verifyUpgradeFailsIfMixedCluster(); + verifyContinuousTransformHandlesData(lastCheckpoint); } case UPGRADED -> { client().performRequest(waitForYellow); @@ -233,19 +232,6 @@ private void verifyContinuousTransformHandlesData(long expectedLastCheckpoint) t }); } - private void verifyUpgradeFailsIfMixedCluster() { - // upgrade tests by design are also executed with the same version, this check must be skipped in this case, see gh#39102. - if (isOriginalClusterCurrent()) { - return; - } - var oldestVersion = Version.fromString(UPGRADE_FROM_VERSION); - if (oldestVersion.onOrAfter(Version.V_9_3_0)) { - final Request upgradeTransformRequest = new Request("POST", getTransformEndpoint() + "_upgrade"); - Exception ex = expectThrows(Exception.class, () -> client().performRequest(upgradeTransformRequest)); - assertThat(ex.getMessage(), containsString("Cannot upgrade transforms while cluster upgrade is in progress")); - } - } - private void verifyUpgrade() throws IOException { final Request upgradeTransformRequest = new Request("POST", getTransformEndpoint() + "_upgrade"); Response response = client().performRequest(upgradeTransformRequest); From 1c61a7028e6216c549ba74303b74de3af3f467f8 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 10 Oct 2025 12:37:28 +0000 Subject: [PATCH 3/4] [CI] Auto commit changes from spotless --- .../elasticsearch/upgrades/TransformSurvivesUpgradeIT.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java index cc5af60982a0b..ab26223bce387 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java @@ -9,7 +9,6 @@ import org.apache.http.HttpHost; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; -import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; @@ -36,7 +35,6 @@ import static org.elasticsearch.xpack.test.rest.XPackRestTestConstants.TRANSFORM_INTERNAL_INDEX_PREFIX; import static org.elasticsearch.xpack.test.rest.XPackRestTestConstants.TRANSFORM_INTERNAL_INDEX_PREFIX_DEPRECATED; import static org.elasticsearch.xpack.test.rest.XPackRestTestConstants.TRANSFORM_TASK_NAME; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -96,7 +94,7 @@ public void testTransformRollingUpgrade() throws Exception { if (Booleans.parseBoolean(System.getProperty("tests.first_round")) == false) { lastCheckpoint = 2; } - verifyContinuousTransformHandlesData(lastCheckpoint); + verifyContinuousTransformHandlesData(lastCheckpoint); } case UPGRADED -> { client().performRequest(waitForYellow); From 04d0d3ab9b1181838f7ff7d57732f31c2efc3c65 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Fri, 10 Oct 2025 14:14:50 +0100 Subject: [PATCH 4/4] fix --- .../transform/persistence/IndexBasedTransformConfigManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java index 586afc405f787..5aefc2ebcc355 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java @@ -302,7 +302,7 @@ public void deleteOldIndices(ActionListener listener) { ); for (var index : matchingIndexes) { - var meta = state.getMetadata().indexMetadata(index); + var meta = state.getMetadata().getIndexSafe(index); if (meta.isSystem() == false) { // ignore system indices as these are automatically managed indicesToDelete.add(meta.getIndex().getName()); }