Wide range of integration tests.

Signed-off-by: Lukasz Soszynski <lukasz.soszynski@eliatra.com>

build.gradle

@@ -430,6 +430,9 @@ dependencies {
     integrationTestImplementation 'org.hamcrest:hamcrest:2.2'
     integrationTestImplementation "org.bouncycastle:bcpkix-jdk15on:${versions.bouncycastle}"
     integrationTestImplementation "org.bouncycastle:bcutil-jdk15on:${versions.bouncycastle}"
+    integrationTestImplementation('org.awaitility:awaitility:4.2.0') {
+        exclude(group: 'org.hamcrest', module: 'hamcrest')
+    }
 }
 
 jar {

src/integrationTest/java/org/opensearch/security/SnapshotSteps.java

@@ -0,0 +1,87 @@
+/*
+* Copyright OpenSearch Contributors
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*
+*/
+package org.opensearch.security;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.awaitility.Awaitility;
+
+import org.opensearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest;
+import org.opensearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
+import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
+import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
+import org.opensearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
+import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
+import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
+import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
+import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
+import org.opensearch.client.RestHighLevelClient;
+import org.opensearch.client.SnapshotClient;
+import org.opensearch.snapshots.SnapshotInfo;
+import org.opensearch.snapshots.SnapshotState;
+
+import static java.util.Objects.requireNonNull;
+import static org.opensearch.client.RequestOptions.DEFAULT;
+
+class SnapshotSteps {
+
+	private final SnapshotClient snapshotClient;
+
+	public SnapshotSteps(RestHighLevelClient restHighLevelClient) {
+		this.snapshotClient = requireNonNull(restHighLevelClient, "Rest high level client is required.").snapshot();
+	}
+
+	//CS-SUPPRESS-ALL: It is not possible to use phrase "cluster manager" instead of master here
+	public org.opensearch.action.support.master.AcknowledgedResponse createSnapshotRepository(String repositoryName, String snapshotDirPath, String type)
+	//CS-ENFORCE-ALL
+		throws IOException {
+		PutRepositoryRequest createRepositoryRequest = new PutRepositoryRequest().name(repositoryName).type(type)
+			.settings(Map.of("location", snapshotDirPath));
+		return snapshotClient.createRepository(createRepositoryRequest, DEFAULT);
+	}
+
+	public CreateSnapshotResponse createSnapshot(String repositoryName, String snapshotName, String...indices) throws IOException {
+		CreateSnapshotRequest createSnapshotRequest = new CreateSnapshotRequest(repositoryName, snapshotName)
+			.indices(indices);
+		return snapshotClient.create(createSnapshotRequest, DEFAULT);
+	}
+
+	public void waitForSnapshotCreation(String repositoryName, String snapshotName) {
+		GetSnapshotsRequest getSnapshotsRequest = new GetSnapshotsRequest(repositoryName, new String[] { snapshotName });
+		Awaitility.await().until(() -> {
+			GetSnapshotsResponse snapshotsResponse = snapshotClient.get(getSnapshotsRequest, DEFAULT);
+			SnapshotInfo snapshotInfo = snapshotsResponse.getSnapshots().get(0);
+			return SnapshotState.SUCCESS.equals(snapshotInfo.state());
+		});
+	}
+
+	//CS-SUPPRESS-ALL: It is not possible to use phrase "cluster manager" instead of master here
+	public org.opensearch.action.support.master.AcknowledgedResponse deleteSnapshotRepository(String repositoryName) throws IOException {
+	//CS-ENFORCE-ALL
+		DeleteRepositoryRequest request = new DeleteRepositoryRequest(repositoryName);
+		return snapshotClient.deleteRepository(request, DEFAULT);
+	}
+
+	//CS-SUPPRESS-ALL: It is not possible to use phrase "cluster manager" instead of master here
+	public org.opensearch.action.support.master.AcknowledgedResponse deleteSnapshot(String repositoryName, String snapshotName) throws IOException {
+	//CS-ENFORCE-ALL
+		return snapshotClient.delete(new DeleteSnapshotRequest(repositoryName, snapshotName), DEFAULT);
+	}
+
+	public RestoreSnapshotResponse restoreSnapshot(
+		String repositoryName, String snapshotName, String renamePattern,
+		String renameReplacement) throws IOException {
+		RestoreSnapshotRequest restoreSnapshotRequest = new RestoreSnapshotRequest(repositoryName, snapshotName)
+			.renamePattern(renamePattern)
+			.renameReplacement(renameReplacement);
+		return snapshotClient.restore(restoreSnapshotRequest, DEFAULT);
+	}
+}

src/integrationTest/java/org/opensearch/security/Song.java

@@ -0,0 +1,44 @@
+/*
+* Copyright OpenSearch Contributors
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*
+*/
+package org.opensearch.security;
+
+
+class Song {
+
+	static final String FIELD_TITLE = "title";
+	static final String FIELD_ARTIST = "artist";
+	static final String FIELD_LYRICS = "lyrics";
+
+	static final String FIELD_STARS = "stars";
+	static final String ARTIST_FIRST = "First artist";
+	static final String ARTIST_STRING = "String";
+	static final String ARTIST_TWINS = "Twins";
+	static final String TITLE_MAGNUM_OPUS = "Magnum Opus";
+	static final String TITLE_SONG_1_PLUS_1 = "Song 1+1";
+	static final String TITLE_NEXT_SONG = "Next song";
+	static final String ARTIST_NO = "No!";
+	static final String TITLE_POISON = "Poison";
+
+	public static final String LYRICS_1 = "Very deep subject";
+	public static final String LYRICS_2 = "Once upon a time";
+	public static final String LYRICS_3 = "giant nonsense";
+	public static final String LYRICS_4 = "Much too much";
+
+	static final String QUERY_TITLE_NEXT_SONG = FIELD_TITLE + ":" + "\"" + TITLE_NEXT_SONG + "\"";
+	static final String QUERY_TITLE_POISON = FIELD_TITLE + ":" + TITLE_POISON;
+	static final String QUERY_TITLE_MAGNUM_OPUS = FIELD_TITLE + ":" + TITLE_MAGNUM_OPUS;
+
+	static final Object[][] SONGS = {
+		{FIELD_ARTIST, ARTIST_FIRST, FIELD_TITLE, TITLE_MAGNUM_OPUS ,FIELD_LYRICS, LYRICS_1, FIELD_STARS, 1},
+		{FIELD_ARTIST, ARTIST_STRING, FIELD_TITLE, TITLE_SONG_1_PLUS_1, FIELD_LYRICS, LYRICS_2, FIELD_STARS, 2},
+		{FIELD_ARTIST, ARTIST_TWINS, FIELD_TITLE, TITLE_NEXT_SONG, FIELD_LYRICS, LYRICS_3, FIELD_STARS, 3},
+		{FIELD_ARTIST, ARTIST_NO, FIELD_TITLE, TITLE_POISON, FIELD_LYRICS, LYRICS_4, FIELD_STARS, 4}
+	};
+}

src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java

@@ -98,6 +98,10 @@ private LocalCluster(String clusterName, TestSecurityConfig testSgConfig, Settin
 		this.testIndices = testIndices;
 	}
 
+	public String getSnapshotDirPath() {
+		return localOpenSearchCluster.getSnapshotDirPath();
+	}
+
 	@Override
 	public void before() throws Throwable {
 		if (localOpenSearchCluster == null) {

src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java

@@ -102,6 +102,8 @@ public class LocalOpenSearchCluster {
 	private boolean started;
 	private Random random = new Random();
 
+	private File snapshotDir;
+
 	public LocalOpenSearchCluster(String clusterName, ClusterManager clusterManager, NodeSettingsSupplier nodeSettingsSupplier,
 						List<Class<? extends Plugin>> additionalPlugins, TestCertificates testCertificates) {
 		this.clusterName = clusterName;
@@ -110,12 +112,23 @@ public LocalOpenSearchCluster(String clusterName, ClusterManager clusterManager,
 		this.additionalPlugins = additionalPlugins;
 		this.testCertificates = testCertificates;
 		try {
-			this.clusterHomeDir = Files.createTempDirectory("local_cluster_" + clusterName).toFile();
+			createClusterDirectory(clusterName);
 		} catch (IOException e) {
 			throw new IllegalStateException(e);
 		}        
 	}
 
+	public String getSnapshotDirPath() {
+		return snapshotDir.getAbsolutePath();
+	}
+
+	private void createClusterDirectory(String clusterName) throws IOException {
+		this.clusterHomeDir = Files.createTempDirectory("local_cluster_" + clusterName).toFile();
+		log.debug("Cluster home directory '{}'.", clusterHomeDir.getAbsolutePath());
+		this.snapshotDir = new File(this.clusterHomeDir, "snapshots");
+		this.snapshotDir.mkdir();
+	}
+
 	private List<Node> getNodesByType(NodeType nodeType) {
 		return nodes.stream()
 			.filter(currentNode -> currentNode.hasAssignedType(nodeType))
@@ -230,8 +243,7 @@ private void retry() throws Exception {
 		this.nodes.clear();
 		this.seedHosts = null;
 		this.initialClusterManagerHosts = null;
-		this.clusterHomeDir = Files.createTempDirectory("local_cluster_" + clusterName + "_retry_" + retry).toFile();
-
+		createClusterDirectory("local_cluster_" + clusterName + "_retry_" + retry);
 		start();
 	}
 
@@ -458,13 +470,15 @@ public InetSocketAddress getTransportAddress() {
 		}
 
 		private Settings getOpenSearchSettings() {
-			Settings settings = getMinimalOpenSearchSettings();
+			Settings settings = Settings.builder()
+				.put(getMinimalOpenSearchSettings())
+				.putList("path.repo", List.of(snapshotDir.getAbsolutePath()))
+				.build();
 
 			if (nodeSettingsSupplier != null) {
 				// TODO node number
 				return Settings.builder().put(settings).put(nodeSettingsSupplier.get(0)).build();
 			}
-
 			return settings;
 		}
 

src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java

@@ -44,8 +44,18 @@
 import javax.net.ssl.TrustManagerFactory;
 
 import org.apache.http.Header;
+import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.message.BasicHeader;
+import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
 
+import org.opensearch.client.RestClient;
+import org.opensearch.client.RestClientBuilder;
+import org.opensearch.client.RestHighLevelClient;
 import org.opensearch.security.support.PemKeyReader;
 import org.opensearch.test.framework.certificate.TestCertificates;
 
@@ -82,6 +92,25 @@ default TestRestClient getRestClient(UserCredentialsHolder user, Header... heade
 		return getRestClient(user.getName(), user.getPassword(), headers);
 	}
 
+	default RestHighLevelClient getRestHighLevelClient(UserCredentialsHolder user) {
+		InetSocketAddress httpAddress = getHttpAddress();
+		CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+		credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(user.getName(), user.getPassword()));
+
+		RestClientBuilder.HttpClientConfigCallback configCallback = httpClientBuilder -> {
+			httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider).setSSLStrategy(
+				new SSLIOSessionStrategy(getSSLContext(), null, null, NoopHostnameVerifier.INSTANCE));
+
+			return httpClientBuilder;
+		};
+
+		RestClientBuilder builder = RestClient.builder(new HttpHost(httpAddress.getHostString(), httpAddress.getPort(), "https"))
+			.setHttpClientConfigCallback(configCallback);
+
+
+		return new RestHighLevelClient(builder);
+	}
+
 	/**
 	* Returns a REST client that sends requests with basic authentication for the specified user name and password. Optionally,
 	* additional HTTP headers can be specified which will be sent with each request.

src/integrationTest/java/org/opensearch/test/framework/cluster/SearchRequestFactory.java

@@ -0,0 +1,75 @@
+/*
+* Copyright OpenSearch Contributors
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*
+*/
+package org.opensearch.test.framework.cluster;
+
+import org.opensearch.action.search.SearchRequest;
+import org.opensearch.action.search.SearchResponse;
+import org.opensearch.action.search.SearchScrollRequest;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.search.aggregations.AggregationBuilders;
+import org.opensearch.search.builder.SearchSourceBuilder;
+import org.opensearch.search.sort.FieldSortBuilder;
+import org.opensearch.search.sort.SortOrder;
+
+import static java.util.concurrent.TimeUnit.MINUTES;
+
+public class SearchRequestFactory {
+	public static SearchRequest queryStringQueryRequest(String indexName, String queryString) {
+		SearchRequest searchRequest = new SearchRequest(indexName);
+		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+		searchSourceBuilder.query(QueryBuilders.queryStringQuery(queryString));
+		searchRequest.source(searchSourceBuilder);
+		return searchRequest;
+	}
+
+	public static SearchRequest queryStringQueryRequest(String queryString) {
+		SearchRequest searchRequest = new SearchRequest();
+		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+		searchSourceBuilder.query(QueryBuilders.queryStringQuery(queryString));
+		searchRequest.source(searchSourceBuilder);
+		return searchRequest;
+	}
+
+	public static SearchRequest searchRequestWithScroll(String indexName, int pageSize) {
+		SearchRequest searchRequest = new SearchRequest(indexName);
+		searchRequest.scroll(new TimeValue(1, MINUTES));
+		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+		searchSourceBuilder.query(QueryBuilders.matchAllQuery());
+		searchSourceBuilder.sort(new FieldSortBuilder("_id").order(SortOrder.ASC));
+		searchSourceBuilder.size(pageSize);
+		searchRequest.source(searchSourceBuilder);
+		return searchRequest;
+	}
+
+	public static SearchScrollRequest getSearchScrollRequest(SearchResponse searchResponse) {
+		SearchScrollRequest scrollRequest = new SearchScrollRequest(searchResponse.getScrollId());
+		scrollRequest.scroll(new TimeValue(1, MINUTES));
+		return scrollRequest;
+	}
+
+	public static SearchRequest averageAggregationRequest(String indexName, String aggregationName, String fieldName) {
+		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+		searchSourceBuilder.aggregation(AggregationBuilders.avg(aggregationName).field(fieldName));
+		searchSourceBuilder.size(0);
+		SearchRequest searchRequest = new SearchRequest(indexName);
+		searchRequest.source(searchSourceBuilder);
+		return searchRequest;
+	}
+
+	public static SearchRequest statsAggregationRequest(String indexName, String aggregationName, String fieldName) {
+		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+		searchSourceBuilder.aggregation(AggregationBuilders.stats(aggregationName).field(fieldName));
+		searchSourceBuilder.size(0);
+		SearchRequest searchRequest = new SearchRequest(indexName);
+		searchRequest.source(searchSourceBuilder);
+		return searchRequest;
+	}
+}

src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java

@@ -32,6 +32,8 @@
 import java.io.UnsupportedEncodingException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -45,6 +47,7 @@
 import org.apache.commons.io.IOUtils;
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
+import org.apache.http.NameValuePair;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpDelete;
@@ -55,6 +58,7 @@
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.config.SocketConfig;
 import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
@@ -95,8 +99,17 @@ public TestRestClient(InetSocketAddress nodeHttpAddress, List<Header> headers, S
 		this.sslContext = sslContext;
 	}
 
+	public HttpResponse get(String path, List<NameValuePair> queryParameters, Header... headers) {
+		try {
+			URI uri = new URIBuilder(getHttpServerUri()).setPath(path).addParameters(queryParameters).build();
+			return executeRequest(new HttpGet(uri), headers);
+		} catch (URISyntaxException ex) {
+			throw new RuntimeException("Incorrect URI syntax", ex);
+		}
+	}
+
 	public HttpResponse get(String path, Header... headers) {
-		return executeRequest(new HttpGet(getHttpServerUri() + "/" + path), headers);
+		return get(path, Collections.emptyList(), headers);
 	}
 
 	public HttpResponse getAuthInfo( Header... headers) {
@@ -321,6 +334,13 @@ public String toString() {
 					+ ", statusReason=" + statusReason + "]";
 		}
 
+		public <T> T getBodyAs(Class<T> authInfoClass) {
+			try {
+				return DefaultObjectMapper.readValue(getBody(), authInfoClass);
+			} catch (IOException e) {
+				throw new RuntimeException("Cannot parse response body", e);
+			}
+		}
 	}
 
 	@Override