Migrate client transports to Apache HttpClient / Core 5.x

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

CHANGELOG.md

@@ -15,6 +15,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Add support to parse sub-aggregations from filter/nested aggregations ([#234](https://github.com/opensearch-project/opensearch-java/pull/234))
 - Add timeout and throttle to the jenkins workflows ([#231](https://github.com/opensearch-project/opensearch-java/pull/231)) 
 - Updating maintainers, admins and documentation ([#248](https://github.com/opensearch-project/opensearch-java/pull/248))
+- Migrate client transports to Apache HttpClient / Core 5.x ([#246](https://github.com/opensearch-project/opensearch-java/pull/246))
 
 ### Deprecated
 

gradle.properties

@@ -1 +1 @@
-systemProp.version = 2.1.1
+systemProp.version = 3.0.0-SNAPSHOT

java-client/build.gradle.kts

@@ -137,7 +137,7 @@ val integrationTest = task<Test>("integrationTest") {
 
 dependencies {
 
-    val opensearchVersion = "2.3.0"
+    val opensearchVersion = "3.0.0-SNAPSHOT"
     val jacksonVersion = "2.13.4"
     val jacksonDatabindVersion = "2.13.4.2"
 
@@ -172,7 +172,8 @@ dependencies {
     testImplementation("software.amazon.awssdk","aws-crt-client","[2.15,3.0)")
     testImplementation("software.amazon.awssdk","apache-client","[2.15,3.0)")
     testImplementation("software.amazon.awssdk","sts","[2.15,3.0)")
-
+    testImplementation("org.apache.logging.log4j", "log4j-api","[2.17.1,3.0)")
+    testImplementation("org.apache.logging.log4j", "log4j-core","[2.17.1,3.0)")
     // EPL-2.0 OR BSD-3-Clause
     // https://eclipse-ee4j.github.io/yasson/
     implementation("org.eclipse", "yasson", "2.0.2")

java-client/src/main/java/org/opensearch/client/transport/endpoints/SimpleEndpoint.java

@@ -33,9 +33,9 @@
 package org.opensearch.client.transport.endpoints;
 
 import org.opensearch.client.opensearch._types.ErrorResponse;
+import org.apache.hc.core5.net.URLEncodedUtils;
 import org.opensearch.client.json.JsonpDeserializer;
 import org.opensearch.client.transport.JsonEndpoint;
-import org.apache.http.client.utils.URLEncodedUtils;
 
 import java.util.Collections;
 import java.util.Map;

java-client/src/main/java/org/opensearch/client/transport/rest_client/RestClientTransport.java

@@ -48,11 +48,12 @@
 import org.opensearch.client.util.MissingRequiredPropertyException;
 import jakarta.json.stream.JsonGenerator;
 import jakarta.json.stream.JsonParser;
-import org.apache.http.HttpEntity;
-import org.apache.http.entity.BufferedHttpEntity;
-import org.apache.http.entity.ByteArrayEntity;
-import org.apache.http.entity.ContentType;
-import org.apache.http.util.EntityUtils;
+
+import org.apache.hc.core5.http.ContentType;
+import org.apache.hc.core5.http.HttpEntity;
+import org.apache.hc.core5.http.io.entity.BufferedHttpEntity;
+import org.apache.hc.core5.http.io.entity.ByteArrayEntity;
+import org.apache.hc.core5.http.io.entity.EntityUtils;
 import org.opensearch.client.Cancellable;
 import org.opensearch.client.RequestOptions;
 import org.opensearch.client.Response;

java-client/src/test/java/org/opensearch/client/documentation/ConnectingTest.java

@@ -38,8 +38,8 @@
 import org.opensearch.client.json.jackson.JacksonJsonpMapper;
 import org.opensearch.client.transport.OpenSearchTransport;
 import org.opensearch.client.transport.rest_client.RestClientTransport;
-import org.apache.http.HttpHost;
 import org.opensearch.client.RestClient;
+import org.apache.hc.core5.http.HttpHost;
 import org.junit.Ignore;
 import org.junit.Test;
 

java-client/src/test/java/org/opensearch/client/documentation/MigrateHlrcTest.java

@@ -36,9 +36,9 @@
 import org.opensearch.client.json.jackson.JacksonJsonpMapper;
 import org.opensearch.client.transport.OpenSearchTransport;
 import org.opensearch.client.transport.rest_client.RestClientTransport;
-import org.apache.http.HttpHost;
 import org.opensearch.client.RestClient;
 import org.opensearch.client.RestClientBuilder;
+import org.apache.hc.core5.http.HttpHost;
 import org.junit.Test;
 
 public class MigrateHlrcTest {

java-client/src/test/java/org/opensearch/client/opensearch/integTest/OpenSearchRestHighLevelClientTestCase.java

@@ -9,13 +9,18 @@
 package org.opensearch.client.opensearch.integTest;
 
 
-import org.apache.http.HttpHost;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.CredentialsProvider;
-import org.apache.http.conn.ssl.NoopHostnameVerifier;
-import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.ssl.SSLContextBuilder;
+import org.apache.hc.client5.http.auth.AuthScope;
+import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
+import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
+import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager;
+import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder;
+import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder;
+import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
+import org.apache.hc.core5.function.Factory;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.http.nio.ssl.TlsStrategy;
+import org.apache.hc.core5.reactor.ssl.TlsDetails;
+import org.apache.hc.core5.ssl.SSLContextBuilder;
 import org.junit.After;
 import org.opensearch.client.Request;
 import org.opensearch.client.Response;
@@ -41,6 +46,7 @@
 import java.util.Optional;
 import java.util.stream.Collectors;
 
+import javax.net.ssl.SSLEngine;
 
 public abstract class OpenSearchRestHighLevelClientTestCase extends OpenSearchRestTestCase {
 
@@ -80,25 +86,43 @@ protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOE
         RestClientBuilder builder = RestClient.builder(hosts);
         builder.setStrictDeprecationMode(true);
         if (isHttps()) {
-            configureHttpsClient(builder);
+            configureHttpsClient(builder, hosts);
         }
         return builder.build();
     }
 
-    private void configureHttpsClient(RestClientBuilder builder) {
+    private void configureHttpsClient(RestClientBuilder builder, HttpHost[] hosts) {
         builder.setHttpClientConfigCallback(httpClientBuilder -> {
             String userName = Optional.ofNullable(System.getProperty("user")).orElse("admin");
             String password = Optional.ofNullable(System.getProperty("password")).orElse("admin");
 
-            final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
-            credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, password));
+            final BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+            for (final HttpHost host: hosts) {
+                credentialsProvider.setCredentials(new AuthScope(host), new UsernamePasswordCredentials(userName, password.toCharArray()));
+            }
 
             try {
+                final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder
+                    .create()
+                    .setSslContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build())
+                    // disable the certificate since our testing cluster just uses the default security configuration
+                    .setHostnameVerifier(NoopHostnameVerifier.INSTANCE)
+                    // See please https://issues.apache.org/jira/browse/HTTPCLIENT-2219
+                    .setTlsDetailsFactory(new Factory<SSLEngine, TlsDetails>() {
+                        @Override
+                        public TlsDetails create(final SSLEngine sslEngine) {
+                            return new TlsDetails(sslEngine.getSession(), sslEngine.getApplicationProtocol());
+                        }
+                    })
+                    .build();
+
+                final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder.create()
+                    .setTlsStrategy(tlsStrategy)
+                    .build();
+
                 return httpClientBuilder
-                        .setDefaultCredentialsProvider(credentialsProvider)
-                        // disable the certificate since our testing cluster just uses the default security configuration
-                        .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE)
-                        .setSSLContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build());
+                    .setDefaultCredentialsProvider(credentialsProvider)
+                    .setConnectionManager(connectionManager);
             } catch (Exception e) {
                 throw new RuntimeException(e);
             }
@@ -128,7 +152,7 @@ public static void cleanupClient() throws IOException {
     @After
     protected void wipeAllOSIndices() throws IOException {
         Response response = adminClient().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all"));
-        XContentType xContentType = XContentType.fromMediaType(response.getEntity().getContentType().getValue());
+        XContentType xContentType = XContentType.fromMediaType(response.getEntity().getContentType());
         XContentParser parser = xContentType.xContent().createParser(
                 NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
                 response.getEntity().getContent());

java-client/src/test/java/org/opensearch/client/opensearch/integTest/PingAndInfoIT.java

@@ -8,7 +8,7 @@
 
 package org.opensearch.client.opensearch.integTest;
 
-import org.apache.http.client.methods.HttpGet;
+import org.apache.hc.client5.http.classic.methods.HttpGet;
 import org.opensearch.client.Request;
 import org.opensearch.client.opensearch.OpenSearchClient;
 import org.opensearch.client.opensearch.core.InfoResponse;

java-client/src/test/java/org/opensearch/client/opensearch/model/EndpointTest.java

@@ -51,10 +51,10 @@ public void testArrayPathParameter() {
         assertEquals("/a/_refresh", RefreshRequest._ENDPOINT.requestUrl(req));
 
         req = RefreshRequest.of(b -> b.index("a", "b"));
-        assertEquals("/a,b/_refresh", RefreshRequest._ENDPOINT.requestUrl(req));
+        assertEquals("/a%2Cb/_refresh", RefreshRequest._ENDPOINT.requestUrl(req));
 
         req = RefreshRequest.of(b -> b.index("a", "b", "c"));
-        assertEquals("/a,b,c/_refresh", RefreshRequest._ENDPOINT.requestUrl(req));
+        assertEquals("/a%2Cb%2Cc/_refresh", RefreshRequest._ENDPOINT.requestUrl(req));
     }
 
     @Test
@@ -65,7 +65,7 @@ public void testPathEncoding() {
         assertEquals("/a%2Fb/_refresh", RefreshRequest._ENDPOINT.requestUrl(req));
 
         req = RefreshRequest.of(b -> b.index("a/b", "c/d"));
-        assertEquals("/a%2Fb,c%2Fd/_refresh", RefreshRequest._ENDPOINT.requestUrl(req));
+        assertEquals("/a%2Fb%2Cc%2Fd/_refresh", RefreshRequest._ENDPOINT.requestUrl(req));
 
     }
 

java-client/src/test/java/org/opensearch/client/transport/RequestOptionsTest.java

@@ -36,11 +36,11 @@
 import org.opensearch.client.json.jsonb.JsonbJsonpMapper;
 import org.opensearch.client.transport.rest_client.RestClientTransport;
 import com.sun.net.httpserver.HttpServer;
-import org.apache.http.HttpHost;
-import org.apache.http.NameValuePair;
-import org.apache.http.client.utils.URLEncodedUtils;
 import org.opensearch.client.ResponseException;
 import org.opensearch.client.RestClient;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.http.NameValuePair;
+import org.apache.hc.core5.net.URLEncodedUtils;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -96,7 +96,7 @@ public void classSetup() throws IOException {
 
         httpServer.start();
         InetSocketAddress address = httpServer.getAddress();
-        restClient = RestClient.builder(new HttpHost(address.getHostString(), address.getPort(), "http"))
+        restClient = RestClient.builder(new HttpHost("http", address.getHostString(), address.getPort()))
             .build();
     }