Speed up internal cluster tests by optimizing #awaitGlobalNettyThreadsFinish (#93890)

Using the #awaitInactivity call costs almost 1s every time it's called
since that's the hard coded quiet period of the executor and its thread
is mostly started at the end of a test (before #awaitGlobalNettyThreadsFinish) as
a result of shutting down the nodes (the global pool is used for
resolving pool shutdown listeners).
By simply asserting that we don't have any tasks queued on it we get a
stronger assertion and save a second per test suite almost.
This is particularly helpful when running tests on a laptop with less
parallelism. This partically saves over a minute when running
`server:check` on my laptop.

Author: original-brownbear

test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java

@@ -2269,21 +2269,12 @@ public static void afterClass() throws Exception {
     }
 
     /**
-     *  After the cluster is stopped, there are a few netty threads that can linger, so we wait for them to finish otherwise these
-     *  lingering threads can intermittently trigger the thread leak detector.
+     *  After the cluster is stopped, there are a few netty threads that can linger, so we make sure we don't leak any tasks on them.
      */
-    static void awaitGlobalNettyThreadsFinish() {
-        try {
-            GlobalEventExecutor.INSTANCE.awaitInactivity(5, TimeUnit.SECONDS);
-        } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-        } catch (IllegalStateException e) {
-            if (e.getMessage().equals("thread was not started") == false) {
-                throw e;
-            }
-            // ignore since the thread was never started
-        }
-
+    static void awaitGlobalNettyThreadsFinish() throws Exception {
+        // Don't use GlobalEventExecutor#awaitInactivity. It will waste up to 1s for every call and we expect no tasks queued for it
+        // except for the odd scheduled shutdown task.
+        assertBusy(() -> assertEquals(0, GlobalEventExecutor.INSTANCE.pendingTasks()));
         try {
             ThreadDeathWatcher.awaitInactivity(5, TimeUnit.SECONDS);
         } catch (InterruptedException e) {

test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java

@@ -182,7 +182,7 @@
 @ThreadLeakScope(Scope.SUITE)
 @ThreadLeakLingering(linger = 5000) // 5 sec lingering
 @TimeoutSuite(millis = 20 * TimeUnits.MINUTE)
-@ThreadLeakFilters(filters = { GraalVMThreadsFilter.class })
+@ThreadLeakFilters(filters = { GraalVMThreadsFilter.class, NettyGlobalThreadsFilter.class })
 @LuceneTestCase.SuppressSysoutChecks(bugUrl = "we log a lot on purpose")
 // we suppress pretty much all the lucene codecs for now, except asserting
 // assertingcodec is the winner for a codec here: it finds bugs and gives clear exceptions.

test/framework/src/main/java/org/elasticsearch/test/NettyGlobalThreadsFilter.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test;
+
+import com.carrotsearch.randomizedtesting.ThreadFilter;
+
+/**
+ * Excludes threads started by {@link io.netty.util.concurrent.GlobalEventExecutor} which are static per-JVM and reused across test suits.
+ * We make sure we don't leak any tasks on them in {@link ESIntegTestCase#awaitGlobalNettyThreadsFinish()}.
+ */
+public class NettyGlobalThreadsFilter implements ThreadFilter {
+    @Override
+    public boolean reject(Thread t) {
+        return t.getName().startsWith("globalEventExecutor");
+    }
+}