Skip to content

Commit

Permalink
local fix for gcp plugin branch 2.11.1
Browse files Browse the repository at this point in the history
Signed-off-by: fahadshamiinsta <fshami@netapp.com>
  • Loading branch information
fahadshamiinsta committed Dec 6, 2023
1 parent 6b1986e commit ea23130
Show file tree
Hide file tree
Showing 3 changed files with 132 additions and 29 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.repositories.gcs;

import com.google.auth.oauth2.GoogleCredentials;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.io.IOException;

public class GoogleApplicationDefaultCredentials {
private static final Logger logger = LogManager.getLogger(GoogleApplicationDefaultCredentials.class);

public GoogleCredentials getApplicationDefaultCredentials() {
GoogleCredentials credentials = null;
try {
credentials = SocketAccess.doPrivilegedIOException(GoogleCredentials::getApplicationDefault);
} catch (IOException e) {
logger.error("Failed to retrieve \"Application Default Credentials\"",e);
}
return credentials;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.auth.oauth2.ServiceAccountCredentials;
import com.google.cloud.ServiceOptions;
import com.google.cloud.http.HttpTransportOptions;
Expand Down Expand Up @@ -70,6 +71,15 @@ public class GoogleCloudStorageService {
*/
private volatile Map<String, Storage> clientCache = emptyMap();

final private GoogleApplicationDefaultCredentials googleApplicationDefaultCredentials;

public GoogleCloudStorageService() {
this.googleApplicationDefaultCredentials = new GoogleApplicationDefaultCredentials();
}
public GoogleCloudStorageService(GoogleApplicationDefaultCredentials googleApplicationDefaultCredentials) {
this.googleApplicationDefaultCredentials = googleApplicationDefaultCredentials;
}

/**
* Refreshes the client settings and clears the client cache. Subsequent calls to
* {@code GoogleCloudStorageService#client} will return new clients constructed
Expand Down Expand Up @@ -213,10 +223,11 @@ StorageOptions createStorageOptions(
storageOptionsBuilder.setProjectId(clientSettings.getProjectId());
}
if (clientSettings.getCredential() == null) {
logger.warn(
"\"Application Default Credentials\" are not supported out of the box."
+ " Additional file system permissions have to be granted to the plugin."
);
logger.info("\"Application Default Credentials\" will be in use");
final GoogleCredentials credentials = googleApplicationDefaultCredentials.getApplicationDefaultCredentials();
if (credentials != null) {
storageOptionsBuilder.setCredentials(credentials);
}
} else {
ServiceAccountCredentials serviceAccountCredentials = clientSettings.getCredential();
// override token server URI
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,12 @@
package org.opensearch.repositories.gcs;

import com.google.auth.Credentials;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.cloud.http.HttpTransportOptions;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageOptions;
import org.hamcrest.MatcherAssert;
import org.mockito.Mockito;
import org.opensearch.common.settings.MockSecureSettings;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
Expand All @@ -44,6 +48,10 @@
import org.opensearch.test.OpenSearchTestCase;
import org.hamcrest.Matchers;

import java.io.IOException;
import java.net.Proxy;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.KeyPair;
import java.security.KeyPairGenerator;
import java.util.Base64;
Expand All @@ -56,16 +64,17 @@

public class GoogleCloudStorageServiceTests extends OpenSearchTestCase {

final TimeValue connectTimeValue = TimeValue.timeValueNanos(randomIntBetween(0, 2000000));
final TimeValue readTimeValue = TimeValue.timeValueNanos(randomIntBetween(0, 2000000));
final String applicationName = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
final String endpoint = randomFrom("http://", "https://")
+ randomFrom("www.opensearch.org", "www.googleapis.com", "localhost/api", "google.com/oauth")
+ ":"
+ randomIntBetween(1, 65535);
final String projectIdName = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);

public void testClientInitializer() throws Exception {
final String clientName = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
final TimeValue connectTimeValue = TimeValue.timeValueNanos(randomIntBetween(0, 2000000));
final TimeValue readTimeValue = TimeValue.timeValueNanos(randomIntBetween(0, 2000000));
final String applicationName = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
final String endpoint = randomFrom("http://", "https://")
+ randomFrom("www.opensearch.org", "www.googleapis.com", "localhost/api", "google.com/oauth")
+ ":"
+ randomIntBetween(1, 65535);
final String projectIdName = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
final Settings settings = Settings.builder()
.put(
GoogleCloudStorageClientSettings.CONNECT_TIMEOUT_SETTING.getConcreteSettingForNamespace(clientName).getKey(),
Expand All @@ -82,31 +91,35 @@ public void testClientInitializer() throws Exception {
.put(GoogleCloudStorageClientSettings.ENDPOINT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), endpoint)
.put(GoogleCloudStorageClientSettings.PROJECT_ID_SETTING.getConcreteSettingForNamespace(clientName).getKey(), projectIdName)
.build();
final GoogleCloudStorageService service = new GoogleCloudStorageService();
GoogleCredentials mockGoogleCredentials = Mockito.mock(GoogleCredentials.class);
GoogleApplicationDefaultCredentials mockDefaultCredentials = Mockito.mock(GoogleApplicationDefaultCredentials.class);
Mockito.when(mockDefaultCredentials.getApplicationDefaultCredentials()).thenReturn(mockGoogleCredentials);

final GoogleCloudStorageService service = new GoogleCloudStorageService(mockDefaultCredentials);
service.refreshAndClearCache(GoogleCloudStorageClientSettings.load(settings));
GoogleCloudStorageOperationsStats statsCollector = new GoogleCloudStorageOperationsStats("bucket");
final IllegalArgumentException e = expectThrows(
IllegalArgumentException.class,
() -> service.client("another_client", "repo", statsCollector)
);
assertThat(e.getMessage(), Matchers.startsWith("Unknown client name"));
MatcherAssert.assertThat(e.getMessage(), Matchers.startsWith("Unknown client name"));
assertSettingDeprecationsAndWarnings(
new Setting<?>[] { GoogleCloudStorageClientSettings.APPLICATION_NAME_SETTING.getConcreteSettingForNamespace(clientName) }
);
final Storage storage = service.client(clientName, "repo", statsCollector);
assertThat(storage.getOptions().getApplicationName(), Matchers.containsString(applicationName));
assertThat(storage.getOptions().getHost(), Matchers.is(endpoint));
assertThat(storage.getOptions().getProjectId(), Matchers.is(projectIdName));
assertThat(storage.getOptions().getTransportOptions(), Matchers.instanceOf(HttpTransportOptions.class));
assertThat(
MatcherAssert.assertThat(storage.getOptions().getApplicationName(), Matchers.containsString(applicationName));
MatcherAssert.assertThat(storage.getOptions().getHost(), Matchers.is(endpoint));
MatcherAssert.assertThat(storage.getOptions().getProjectId(), Matchers.is(projectIdName));
MatcherAssert.assertThat(storage.getOptions().getTransportOptions(), Matchers.instanceOf(HttpTransportOptions.class));
MatcherAssert.assertThat(
((HttpTransportOptions) storage.getOptions().getTransportOptions()).getConnectTimeout(),
Matchers.is((int) connectTimeValue.millis())
);
assertThat(
MatcherAssert.assertThat(
((HttpTransportOptions) storage.getOptions().getTransportOptions()).getReadTimeout(),
Matchers.is((int) readTimeValue.millis())
);
assertThat(storage.getOptions().getCredentials(), Matchers.nullValue(Credentials.class));
MatcherAssert.assertThat(storage.getOptions().getCredentials(), Matchers.instanceOf(Credentials.class));
}

public void testReinitClientSettings() throws Exception {
Expand All @@ -122,33 +135,33 @@ public void testReinitClientSettings() throws Exception {
final GoogleCloudStorageService storageService = plugin.storageService;
GoogleCloudStorageOperationsStats statsCollector = new GoogleCloudStorageOperationsStats("bucket");
final Storage client11 = storageService.client("gcs1", "repo1", statsCollector);
assertThat(client11.getOptions().getProjectId(), equalTo("project_gcs11"));
MatcherAssert.assertThat(client11.getOptions().getProjectId(), equalTo("project_gcs11"));
final Storage client12 = storageService.client("gcs2", "repo2", statsCollector);
assertThat(client12.getOptions().getProjectId(), equalTo("project_gcs12"));
MatcherAssert.assertThat(client12.getOptions().getProjectId(), equalTo("project_gcs12"));
// client 3 is missing
final IllegalArgumentException e1 = expectThrows(
IllegalArgumentException.class,
() -> storageService.client("gcs3", "repo3", statsCollector)
);
assertThat(e1.getMessage(), containsString("Unknown client name [gcs3]."));
MatcherAssert.assertThat(e1.getMessage(), containsString("Unknown client name [gcs3]."));
// update client settings
plugin.reload(settings2);
// old client 1 not changed
assertThat(client11.getOptions().getProjectId(), equalTo("project_gcs11"));
MatcherAssert.assertThat(client11.getOptions().getProjectId(), equalTo("project_gcs11"));
// new client 1 is changed
final Storage client21 = storageService.client("gcs1", "repo1", statsCollector);
assertThat(client21.getOptions().getProjectId(), equalTo("project_gcs21"));
MatcherAssert.assertThat(client21.getOptions().getProjectId(), equalTo("project_gcs21"));
// old client 2 not changed
assertThat(client12.getOptions().getProjectId(), equalTo("project_gcs12"));
MatcherAssert.assertThat(client12.getOptions().getProjectId(), equalTo("project_gcs12"));
// new client2 is gone
final IllegalArgumentException e2 = expectThrows(
IllegalArgumentException.class,
() -> storageService.client("gcs2", "repo2", statsCollector)
);
assertThat(e2.getMessage(), containsString("Unknown client name [gcs2]."));
MatcherAssert.assertThat(e2.getMessage(), containsString("Unknown client name [gcs2]."));
// client 3 emerged
final Storage client23 = storageService.client("gcs3", "repo3", statsCollector);
assertThat(client23.getOptions().getProjectId(), equalTo("project_gcs23"));
MatcherAssert.assertThat(client23.getOptions().getProjectId(), equalTo("project_gcs23"));
}
}

Expand Down Expand Up @@ -193,4 +206,53 @@ public void testToTimeout() {
assertEquals(-1, GoogleCloudStorageService.toTimeout(TimeValue.ZERO).intValue());
assertEquals(0, GoogleCloudStorageService.toTimeout(TimeValue.MINUS_ONE).intValue());
}

/*
The following method test the Google Application Default Credential instead of
using service account file.
Considered use of Junit Mocking due to static method GoogleCredentials.getApplicationDefault
and avoiding environment variables to set which later use GCE.
*/
public void testApplicationDefaultCredential() throws Exception {
GoogleCloudStorageClientSettings settings = getGCSClientSettingsWithoutCredentials();
GoogleCredentials mockGoogleCredentials = Mockito.mock(GoogleCredentials.class);
HttpTransportOptions mockHttpTransportOptions = Mockito.mock(HttpTransportOptions.class);
GoogleApplicationDefaultCredentials mockDefaultCredentials = Mockito.mock(GoogleApplicationDefaultCredentials.class);
Mockito.when(mockDefaultCredentials.getApplicationDefaultCredentials()).thenReturn(mockGoogleCredentials);

GoogleCloudStorageService service = new GoogleCloudStorageService(mockDefaultCredentials);
StorageOptions storageOptions = service.createStorageOptions(settings,mockHttpTransportOptions);
assertNotNull(storageOptions);
assertEquals(storageOptions.getCredentials().toString(),mockGoogleCredentials.toString());
}

/**
* The application default credential throws exception when there are no
* Environment Variables provided or Google Compute Engine is not running
* @throws Exception
*/
public void applicationDefaultCredentialThrowsErrorWhenNotAvailable() throws Exception {
GoogleCloudStorageClientSettings settings = getGCSClientSettingsWithoutCredentials();
HttpTransportOptions mockHttpTransportOptions = Mockito.mock(HttpTransportOptions.class);
GoogleCloudStorageService service = new GoogleCloudStorageService();
StorageOptions storageOptions = service.createStorageOptions(settings,mockHttpTransportOptions);

Exception exception = assertThrows(IOException.class, GoogleCredentials::getApplicationDefault);
assertNotNull(storageOptions);
assertNull(storageOptions.getCredentials());
MatcherAssert.assertThat(exception.getMessage(),containsString("The Application Default Credentials are not available"));
}

private GoogleCloudStorageClientSettings getGCSClientSettingsWithoutCredentials() throws URISyntaxException {
return new GoogleCloudStorageClientSettings(
null,
endpoint,
projectIdName,
connectTimeValue,
readTimeValue,
applicationName,
new URI(""),
new ProxySettings(Proxy.Type.DIRECT, null, 0, null, null));
}

}

0 comments on commit ea23130

Please sign in to comment.