feat: enable gRPC client open telemetry metrics reporting (#2590)

Co-authored-by: Frank Natividad <franknatividad@google.com>
Co-authored-by: BenWhitehead <BenWhitehead@users.noreply.github.com>

google-cloud-storage/pom.xml

@@ -124,6 +124,53 @@
       <groupId>com.google.api.grpc</groupId>
       <artifactId>gapic-google-cloud-storage-v2</artifactId>
     </dependency>
+    <!-- Open Telemetry dependencies -->
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-sdk</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.grpc</groupId>
+      <artifactId>grpc-opentelemetry</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-api</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-sdk-metrics</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-sdk-common</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-sdk-extension-autoconfigure-spi</artifactId>
+    </dependency>
+
+    <dependency>
+    <groupId>io.opentelemetry.semconv</groupId>
+    <artifactId>opentelemetry-semconv</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>com.google.cloud.opentelemetry</groupId>
+      <artifactId>exporter-metrics</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>io.opentelemetry.contrib</groupId>
+      <artifactId>opentelemetry-gcp-resources</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.checkerframework</groupId>
+      <artifactId>checker-qual</artifactId>
+    </dependency>
 
     <!-- Access to exception for retry handling -->
     <dependency>
@@ -159,10 +206,7 @@
       <artifactId>grpc-googleapis</artifactId>
       <scope>runtime</scope>
     </dependency>
-    <dependency>
-      <groupId>org.checkerframework</groupId>
-      <artifactId>checker-qual</artifactId>
-    </dependency>
+
     <!--
     We're not using this directly, however there appears to be some resolution
     issues when we don't list this dependency. Specifically, the check to ensure the flattened
@@ -184,6 +228,7 @@
       <version>0.140.0</version>
       <scope>test</scope>
     </dependency>
+
     <dependency>
       <groupId>com.google.cloud</groupId>
       <artifactId>google-cloud-kms</artifactId>
@@ -307,6 +352,12 @@
               Add in vintage engine so that both JUnit4 and JUnit5 tests are run by the JUnit5 runner.
               -->
               <dependency>org.junit.vintage:junit-vintage-engine</dependency>
+
+              <!--
+              These are needed at runtime, but it causes unused declared dependency errors
+              -->
+              <dependency>io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi</dependency>
+              <dependency>io.opentelemetry.semconv:opentelemetry-semconv</dependency>
             </ignoredDependencies>
           </configuration>
         </plugin>

google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageOptions.java

@@ -116,6 +116,9 @@ public final class GrpcStorageOptions extends StorageOptions
   private final GrpcRetryAlgorithmManager retryAlgorithmManager;
   private final Duration terminationAwaitDuration;
   private final boolean attemptDirectPath;
+  private final boolean enableGrpcClientMetrics;
+
+  private final boolean grpcClientMetricsManuallyEnabled;
   private final GrpcInterceptorProvider grpcInterceptorProvider;
   private final BlobWriteSessionConfig blobWriteSessionConfig;
 
@@ -129,6 +132,8 @@ private GrpcStorageOptions(Builder builder, GrpcStorageDefaults serviceDefaults)
         MoreObjects.firstNonNull(
             builder.terminationAwaitDuration, serviceDefaults.getTerminationAwaitDuration());
     this.attemptDirectPath = builder.attemptDirectPath;
+    this.enableGrpcClientMetrics = builder.enableGrpcClientMetrics;
+    this.grpcClientMetricsManuallyEnabled = builder.grpcMetricsManuallyEnabled;
     this.grpcInterceptorProvider = builder.grpcInterceptorProvider;
     this.blobWriteSessionConfig = builder.blobWriteSessionConfig;
   }
@@ -287,6 +292,16 @@ private Tuple<StorageSettings, Opts<UserProject>> resolveSettingsAndOpts() throw
     if (scheme.equals("http")) {
       channelProviderBuilder.setChannelConfigurator(ManagedChannelBuilder::usePlaintext);
     }
+
+    if (enableGrpcClientMetrics) {
+      OpenTelemetryBootstrappingUtils.enableGrpcMetrics(
+          channelProviderBuilder,
+          endpoint,
+          this.getProjectId(),
+          this.getUniverseDomain(),
+          !grpcClientMetricsManuallyEnabled);
+    }
+
     builder.setTransportChannelProvider(channelProviderBuilder.build());
     RetrySettings baseRetrySettings = getRetrySettings();
     RetrySettings readRetrySettings =
@@ -350,6 +365,7 @@ public int hashCode() {
         retryAlgorithmManager,
         terminationAwaitDuration,
         attemptDirectPath,
+        enableGrpcClientMetrics,
         grpcInterceptorProvider,
         blobWriteSessionConfig,
         baseHashCode());
@@ -365,6 +381,7 @@ public boolean equals(Object o) {
     }
     GrpcStorageOptions that = (GrpcStorageOptions) o;
     return attemptDirectPath == that.attemptDirectPath
+        && enableGrpcClientMetrics == that.enableGrpcClientMetrics
         && Objects.equals(retryAlgorithmManager, that.retryAlgorithmManager)
         && Objects.equals(terminationAwaitDuration, that.terminationAwaitDuration)
         && Objects.equals(grpcInterceptorProvider, that.grpcInterceptorProvider)
@@ -408,11 +425,15 @@ public static final class Builder extends StorageOptions.Builder {
     private StorageRetryStrategy storageRetryStrategy;
     private Duration terminationAwaitDuration;
     private boolean attemptDirectPath = GrpcStorageDefaults.INSTANCE.isAttemptDirectPath();
+    private boolean enableGrpcClientMetrics =
+        GrpcStorageDefaults.INSTANCE.isEnableGrpcClientMetrics();
     private GrpcInterceptorProvider grpcInterceptorProvider =
         GrpcStorageDefaults.INSTANCE.grpcInterceptorProvider();
     private BlobWriteSessionConfig blobWriteSessionConfig =
         GrpcStorageDefaults.INSTANCE.getDefaultStorageWriterConfig();
 
+    private boolean grpcMetricsManuallyEnabled = false;
+
     Builder() {}
 
     Builder(StorageOptions options) {
@@ -421,6 +442,7 @@ public static final class Builder extends StorageOptions.Builder {
       this.storageRetryStrategy = gso.getRetryAlgorithmManager().retryStrategy;
       this.terminationAwaitDuration = gso.getTerminationAwaitDuration();
       this.attemptDirectPath = gso.attemptDirectPath;
+      this.enableGrpcClientMetrics = gso.enableGrpcClientMetrics;
       this.grpcInterceptorProvider = gso.grpcInterceptorProvider;
       this.blobWriteSessionConfig = gso.blobWriteSessionConfig;
     }
@@ -454,6 +476,21 @@ public GrpcStorageOptions.Builder setAttemptDirectPath(boolean attemptDirectPath
       this.attemptDirectPath = attemptDirectPath;
       return this;
     }
+    /**
+     * Option for whether this client should emit internal gRPC client internal metrics to Cloud
+     * Monitoring. To disable metric reporting, set this to false. True by default. Emitting metrics
+     * is free and requires minimal CPU and memory.
+     *
+     * @since 2.41.0 This new api is in preview and is subject to breaking changes.
+     */
+    @BetaApi
+    public GrpcStorageOptions.Builder setEnableGrpcClientMetrics(boolean enableGrpcClientMetrics) {
+      this.enableGrpcClientMetrics = enableGrpcClientMetrics;
+      if (enableGrpcClientMetrics) {
+        grpcMetricsManuallyEnabled = true;
+      }
+      return this;
+    }
 
     /** @since 2.14.0 This new api is in preview and is subject to breaking changes. */
     @BetaApi
@@ -660,6 +697,12 @@ public boolean isAttemptDirectPath() {
       return false;
     }
 
+    /** @since 2.41.0 This new api is in preview and is subject to breaking changes. */
+    @BetaApi
+    public boolean isEnableGrpcClientMetrics() {
+      return true;
+    }
+
     /** @since 2.22.3 This new api is in preview and is subject to breaking changes. */
     @BetaApi
     public GrpcInterceptorProvider grpcInterceptorProvider() {