Add base class for implementing retry strategies in otlp exporter

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/OtlpGrpcTraceExportClient.cs

@@ -14,7 +14,6 @@
 // limitations under the License.
 // </copyright>
 
-using Grpc.Core;
 using OtlpCollector = OpenTelemetry.Proto.Collector.Trace.V1;
 
 namespace OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.ExportClient;
@@ -43,16 +42,7 @@ public override bool SendExportRequest(OtlpCollector.ExportTraceServiceRequest r
     {
         var deadline = DateTime.UtcNow.AddMilliseconds(this.TimeoutMilliseconds);
 
-        try
-        {
-            this.traceClient.Export(request, headers: this.Headers, deadline: deadline, cancellationToken: cancellationToken);
-        }
-        catch (RpcException ex)
-        {
-            OpenTelemetryProtocolExporterEventSource.Log.FailedToReachCollector(this.Endpoint, ex);
-
-            return false;
-        }
+        this.traceClient.Export(request, headers: this.Headers, deadline: deadline, cancellationToken: cancellationToken);
 
         return true;
     }

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/OpenTelemetryProtocolExporterEventSource.cs

@@ -14,6 +14,9 @@
 // limitations under the License.
 // </copyright>
 
+#if NET6_0_OR_GREATER
+using System.Diagnostics.CodeAnalysis;
+#endif
 using System.Diagnostics.Tracing;
 using OpenTelemetry.Internal;
 
@@ -35,11 +38,11 @@ public void FailedToReachCollector(Uri collectorUri, Exception ex)
     }
 
     [NonEvent]
-    public void ExportMethodException(Exception ex)
+    public void ExportMethodException(Exception ex, bool isRetry = false)
     {
         if (Log.IsEnabled(EventLevel.Error, EventKeywords.All))
         {
-            this.ExportMethodException(ex.ToInvariantString());
+            this.ExportMethodException(ex.ToInvariantString(), isRetry);
         }
     }
 
@@ -55,10 +58,13 @@ public void CouldNotTranslateActivity(string className, string methodName)
         this.WriteEvent(3, className, methodName);
     }
 
-    [Event(4, Message = "Unknown error in export method: {0}", Level = EventLevel.Error)]
-    public void ExportMethodException(string ex)
+#if NET6_0_OR_GREATER
+    [UnconditionalSuppressMessage("ReflectionAnalysis", "IL2026:RequiresUnreferencedCode", Justification = "Parameters to this method are primitive and are trimmer safe.")]
+#endif
+    [Event(4, Message = "Unknown error in export method. Message: '{0}'. IsRetry: {1}", Level = EventLevel.Error)]
+    public void ExportMethodException(string ex, bool isRetry)
     {
-        this.WriteEvent(4, ex);
+        this.WriteEvent(4, ex, isRetry);
     }
 
     [Event(5, Message = "Could not translate metric from class '{0}' and method '{1}', metric will not be recorded.", Level = EventLevel.Informational)]

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/retry/OtlpExporterTransmissionHandler.cs

@@ -0,0 +1,89 @@
+// <copyright file="OtlpExporterTransmissionHandler.cs" company="OpenTelemetry Authors">
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// </copyright>
+
+using OpenTelemetry.Exporter;
+using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation;
+using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.ExportClient;
+
+namespace OpenTelemetry.ExporterOpenTelemetryProtocol.Implementation.Retry;
+
+internal class OtlpExporterTransmissionHandler<T>
+{
+    internal IExportClient<T> ExportClient;
+
+    public OtlpExporterOptions Options { get; internal set; }
+
+    /// <summary>
+    /// Sends export request to the server.
+    /// </summary>
+    /// <param name="request">The request to send to the server.</param>
+    /// <returns>True if the request is sent successfully or else false.</returns>
+    public virtual bool SubmitRequest(T request)
+    {
+        try
+        {
+            return this.ExportClient.SendExportRequest(request);
+        }
+        catch (Exception ex)
+        {
+            OpenTelemetryProtocolExporterEventSource.Log.ExportMethodException(ex);
+            return this.OnSubmitRequestExceptionThrown(request, ex);
+        }
+    }
+
+    /// <summary>
+    /// Retries sending request to the server.
+    /// </summary>
+    /// <param name="request">The request to send to the server.</param>
+    /// <param name="exception">Exception encountered when trying to send request.</param>
+    /// <returns>True if the request is sent successfully or else false.</returns>
+    protected virtual bool RetryRequest(T request, out Exception exception)
+    {
+        try
+        {
+            var result = this.ExportClient.SendExportRequest(request);
+            exception = null;
+            return result;
+        }
+        catch (Exception ex)
+        {
+            OpenTelemetryProtocolExporterEventSource.Log.ExportMethodException(ex, isRetry: true);
+            exception = ex;
+            return false;
+        }
+    }
+
+    /// <summary>
+    /// Callback to call when encountered exception while sending request to server.
+    /// </summary>
+    /// <param name="request">The request that was attempted to send to the server.</param>
+    /// <param name="exception">Exception that was encountered during request processing.</param>
+    /// <returns>True or False, based on the implementation of handling errors.</returns>
+    protected virtual bool OnSubmitRequestExceptionThrown(T request, Exception exception)
+    {
+        return this.OnHandleDroppedRequest(request);
+    }
+
+    /// <summary>
+    /// Action to take when dropping request.
+    /// </summary>
+    /// <param name="request">The request that was attempted to send to the server.</param>
+    /// <returns>True or False, based on the implementation.</returns>
+    protected virtual bool OnHandleDroppedRequest(T request)
+    {
+        return false;
+    }
+}

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpTraceExporter.cs

@@ -17,6 +17,7 @@
 using System.Diagnostics;
 using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation;
 using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.ExportClient;
+using OpenTelemetry.ExporterOpenTelemetryProtocol.Implementation.Retry;
 using OpenTelemetry.Internal;
 using OtlpCollector = OpenTelemetry.Proto.Collector.Trace.V1;
 using OtlpResource = OpenTelemetry.Proto.Resource.V1;
@@ -31,6 +32,7 @@ public class OtlpTraceExporter : BaseExporter<Activity>
 {
     private readonly SdkLimitOptions sdkLimitOptions;
     private readonly IExportClient<OtlpCollector.ExportTraceServiceRequest> exportClient;
+    private readonly OtlpExporterTransmissionHandler<OtlpCollector.ExportTraceServiceRequest> transmissionHandler;
 
     private OtlpResource.Resource processResource;
 
@@ -39,7 +41,7 @@ public class OtlpTraceExporter : BaseExporter<Activity>
     /// </summary>
     /// <param name="options">Configuration options for the export.</param>
     public OtlpTraceExporter(OtlpExporterOptions options)
-        : this(options, new(), null)
+        : this(options, new(), null, null)
     {
     }
 
@@ -49,10 +51,12 @@ public OtlpTraceExporter(OtlpExporterOptions options)
     /// <param name="exporterOptions"><see cref="OtlpExporterOptions"/>.</param>
     /// <param name="sdkLimitOptions"><see cref="SdkLimitOptions"/>.</param>
     /// <param name="exportClient">Client used for sending export request.</param>
+    /// <param name="transmissionHandler">Transmission handler for retrying failed requests.</param>
     internal OtlpTraceExporter(
         OtlpExporterOptions exporterOptions,
         SdkLimitOptions sdkLimitOptions,
-        IExportClient<OtlpCollector.ExportTraceServiceRequest> exportClient = null)
+        IExportClient<OtlpCollector.ExportTraceServiceRequest> exportClient = null,
+        OtlpExporterTransmissionHandler<OtlpCollector.ExportTraceServiceRequest> transmissionHandler = null)
     {
         Debug.Assert(exporterOptions != null, "exporterOptions was null");
         Debug.Assert(sdkLimitOptions != null, "sdkLimitOptions was null");
@@ -77,6 +81,13 @@ internal OtlpTraceExporter(
         {
             this.exportClient = exporterOptions.GetTraceExportClient();
         }
+
+        transmissionHandler ??= new OtlpExporterTransmissionHandler<OtlpCollector.ExportTraceServiceRequest>();
+
+        transmissionHandler.ExportClient = this.exportClient;
+        transmissionHandler.Options = exporterOptions;
+
+        this.transmissionHandler = transmissionHandler;
     }
 
     internal OtlpResource.Resource ProcessResource => this.processResource ??= this.ParentProvider.GetResource().ToOtlpResource();
@@ -93,7 +104,7 @@ public override ExportResult Export(in Batch<Activity> activityBatch)
         {
             request.AddBatch(this.sdkLimitOptions, this.ProcessResource, activityBatch);
 
-            if (!this.exportClient.SendExportRequest(request))
+            if (!this.transmissionHandler.SubmitRequest(request))
             {
                 return ExportResult.Failure;
             }
@@ -114,6 +125,6 @@ public override ExportResult Export(in Batch<Activity> activityBatch)
     /// <inheritdoc />
     protected override bool OnShutdown(int timeoutMilliseconds)
     {
-        return this.exportClient?.Shutdown(timeoutMilliseconds) ?? true;
+        return this.transmissionHandler.ExportClient?.Shutdown(timeoutMilliseconds) ?? true;
     }
 }

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpTraceExporterHelperExtensions.cs

@@ -16,10 +16,13 @@
 
 using System.Diagnostics;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection.Extensions;
 using Microsoft.Extensions.Options;
 using OpenTelemetry.Exporter;
 using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation;
+using OpenTelemetry.ExporterOpenTelemetryProtocol.Implementation.Retry;
 using OpenTelemetry.Internal;
+using OtlpCollector = OpenTelemetry.Proto.Collector.Trace.V1;
 
 namespace OpenTelemetry.Trace;
 
@@ -72,6 +75,7 @@ public static TracerProviderBuilder AddOtlpExporter(
 
             OtlpExporterOptions.RegisterOtlpExporterOptionsFactory(services);
             services.RegisterOptionsFactory(configuration => new SdkLimitOptions(configuration));
+            services.TryAddSingleton<OtlpExporterTransmissionHandler<OtlpCollector.ExportTraceServiceRequest>>();
         });
 
         return builder.AddProcessor(sp =>
@@ -103,6 +107,8 @@ public static TracerProviderBuilder AddOtlpExporter(
             // instance.
             var sdkOptionsManager = sp.GetRequiredService<IOptionsMonitor<SdkLimitOptions>>().CurrentValue;
 
+            var transmissionmanager = sp.GetRequiredService<OtlpExporterTransmissionHandler<OtlpCollector.ExportTraceServiceRequest>>();
+
             return BuildOtlpExporterProcessor(exporterOptions, sdkOptionsManager, sp);
         });
     }
@@ -111,11 +117,15 @@ internal static BaseProcessor<Activity> BuildOtlpExporterProcessor(
         OtlpExporterOptions exporterOptions,
         SdkLimitOptions sdkLimitOptions,
         IServiceProvider serviceProvider,
+        OtlpExporterTransmissionHandler<OtlpCollector.ExportTraceServiceRequest> transmissionHandler = null,
         Func<BaseExporter<Activity>, BaseExporter<Activity>> configureExporterInstance = null)
     {
         exporterOptions.TryEnableIHttpClientFactoryIntegration(serviceProvider, "OtlpTraceExporter");
 
-        BaseExporter<Activity> otlpExporter = new OtlpTraceExporter(exporterOptions, sdkLimitOptions);
+        BaseExporter<Activity> otlpExporter = new OtlpTraceExporter(
+            exporterOptions,
+            sdkLimitOptions,
+            transmissionHandler: transmissionHandler ?? serviceProvider.GetRequiredService<OtlpExporterTransmissionHandler<OtlpCollector.ExportTraceServiceRequest>>());
 
         if (configureExporterInstance != null)
         {

test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/IntegrationTest/IntegrationTests.cs

@@ -86,6 +86,7 @@ public void TraceExportResultIsSuccess(OtlpExportProtocol protocol, string endpo
             exporterOptions,
             DefaultSdkLimitOptions,
             serviceProvider: null,
+            new ExporterOpenTelemetryProtocol.Implementation.Retry.OtlpExporterTransmissionHandler<Proto.Collector.Trace.V1.ExportTraceServiceRequest>(),
             configureExporterInstance: otlpExporter =>
             {
                 delegatingExporter = new DelegatingExporter<Activity>