Ensure Test Cluster in Azure Discovery Tests Closes (#43057) (#43372)

* Use `internalCluster().close()` to force all nodes (and not just the datanodes) to shut down even if one fails to shut down in time
* Force closing httpServer to get cleaner logs if nodes still hang on shut down
* Relates #43048

plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java

@@ -25,7 +25,6 @@
 import com.sun.net.httpserver.HttpsConfigurator;
 import com.sun.net.httpserver.HttpsServer;
 import org.apache.logging.log4j.LogManager;
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.bootstrap.JavaVersion;
 import org.elasticsearch.cloud.azure.classic.management.AzureComputeService;
 import org.elasticsearch.common.SuppressForbidden;
@@ -71,7 +70,6 @@
 import java.util.UUID;
 import java.util.concurrent.ExecutionException;
 
-@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/43048")
 @ESIntegTestCase.ClusterScope(numDataNodes = 2, numClientNodes = 0)
 @SuppressForbidden(reason = "use http server")
 // TODO this should be a IT but currently all ITs in this project run against a real cluster
@@ -294,13 +292,14 @@ private static String getProtocol() {
 
     @AfterClass
     public static void stopHttpd() throws IOException {
-        for (int i = 0; i < internalCluster().size(); i++) {
+        try {
             // shut them all down otherwise we get spammed with connection refused exceptions
-            internalCluster().stopRandomDataNode();
+            internalCluster().close();
+        } finally {
+            httpsServer.stop(0);
+            httpsServer = null;
+            logDir = null;
         }
-        httpsServer.stop(0);
-        httpsServer = null;
-        logDir = null;
     }
 
     public void testJoin() throws ExecutionException, InterruptedException {